ia64/xen-unstable

view linux-2.4.26-xen-sparse/mm/vmalloc.c @ 1527:a815a43920c0

bitkeeper revision 1.994 (40d6ed9ePUmxTwjKFv1vprN2-xFpmQ)

Install mkernel odules with 'make install'
author iap10@labyrinth.cl.cam.ac.uk
date Mon Jun 21 14:15:58 2004 +0000 (2004-06-21)
parents f3123052268f
children
line source
1 /*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 */
9 #include <linux/config.h>
10 #include <linux/slab.h>
11 #include <linux/vmalloc.h>
12 #include <linux/spinlock.h>
13 #include <linux/highmem.h>
14 #include <linux/smp_lock.h>
16 #include <asm/uaccess.h>
17 #include <asm/pgalloc.h>
19 rwlock_t vmlist_lock = RW_LOCK_UNLOCKED;
20 struct vm_struct * vmlist;
22 static inline void free_area_pte(pmd_t * pmd, unsigned long address, unsigned long size)
23 {
24 pte_t * pte;
25 unsigned long end;
27 if (pmd_none(*pmd))
28 return;
29 if (pmd_bad(*pmd)) {
30 pmd_ERROR(*pmd);
31 pmd_clear(pmd);
32 return;
33 }
34 pte = pte_offset(pmd, address);
35 address &= ~PMD_MASK;
36 end = address + size;
37 if (end > PMD_SIZE)
38 end = PMD_SIZE;
39 do {
40 pte_t page;
41 page = ptep_get_and_clear(pte);
42 address += PAGE_SIZE;
43 pte++;
44 if (pte_none(page))
45 continue;
46 if (pte_present(page)) {
47 struct page *ptpage = pte_page(page);
48 #if defined(CONFIG_XEN_PRIVILEGED_GUEST)
49 if (pte_io(page))
50 continue;
51 #endif
52 if (VALID_PAGE(ptpage) && (!PageReserved(ptpage)))
53 __free_page(ptpage);
54 continue;
55 }
56 printk(KERN_CRIT "Whee.. Swapped out page in kernel page table\n");
57 } while (address < end);
58 }
60 static inline void free_area_pmd(pgd_t * dir, unsigned long address, unsigned long size)
61 {
62 pmd_t * pmd;
63 unsigned long end;
65 if (pgd_none(*dir))
66 return;
67 if (pgd_bad(*dir)) {
68 pgd_ERROR(*dir);
69 pgd_clear(dir);
70 return;
71 }
72 pmd = pmd_offset(dir, address);
73 address &= ~PGDIR_MASK;
74 end = address + size;
75 if (end > PGDIR_SIZE)
76 end = PGDIR_SIZE;
77 do {
78 free_area_pte(pmd, address, end - address);
79 address = (address + PMD_SIZE) & PMD_MASK;
80 pmd++;
81 } while (address < end);
82 }
84 void vmfree_area_pages(unsigned long address, unsigned long size)
85 {
86 pgd_t * dir;
87 unsigned long end = address + size;
89 dir = pgd_offset_k(address);
90 flush_cache_all();
91 do {
92 free_area_pmd(dir, address, end - address);
93 address = (address + PGDIR_SIZE) & PGDIR_MASK;
94 dir++;
95 } while (address && (address < end));
96 flush_tlb_all();
97 }
99 static inline int alloc_area_pte (pte_t * pte, unsigned long address,
100 unsigned long size, int gfp_mask,
101 pgprot_t prot, struct page ***pages)
102 {
103 unsigned long end;
105 address &= ~PMD_MASK;
106 end = address + size;
107 if (end > PMD_SIZE)
108 end = PMD_SIZE;
109 do {
110 struct page * page;
112 if (!pages) {
113 spin_unlock(&init_mm.page_table_lock);
114 page = alloc_page(gfp_mask);
115 spin_lock(&init_mm.page_table_lock);
116 } else {
117 page = (**pages);
118 (*pages)++;
120 /* Add a reference to the page so we can free later */
121 if (page)
122 atomic_inc(&page->count);
124 }
125 if (!pte_none(*pte))
126 printk(KERN_ERR "alloc_area_pte: page already exists\n");
127 if (!page)
128 return -ENOMEM;
129 set_pte(pte, mk_pte(page, prot));
130 address += PAGE_SIZE;
131 pte++;
132 } while (address < end);
133 return 0;
134 }
136 static inline int alloc_area_pmd(pmd_t * pmd, unsigned long address,
137 unsigned long size, int gfp_mask,
138 pgprot_t prot, struct page ***pages)
139 {
140 unsigned long end;
142 address &= ~PGDIR_MASK;
143 end = address + size;
144 if (end > PGDIR_SIZE)
145 end = PGDIR_SIZE;
146 do {
147 pte_t * pte = pte_alloc(&init_mm, pmd, address);
148 if (!pte)
149 return -ENOMEM;
150 if (alloc_area_pte(pte, address, end - address,
151 gfp_mask, prot, pages))
152 return -ENOMEM;
153 address = (address + PMD_SIZE) & PMD_MASK;
154 pmd++;
155 } while (address < end);
156 return 0;
157 }
159 static inline int __vmalloc_area_pages (unsigned long address,
160 unsigned long size,
161 int gfp_mask,
162 pgprot_t prot,
163 struct page ***pages)
164 {
165 pgd_t * dir;
166 unsigned long start = address;
167 unsigned long end = address + size;
169 dir = pgd_offset_k(address);
170 spin_lock(&init_mm.page_table_lock);
171 do {
172 pmd_t *pmd;
174 pmd = pmd_alloc(&init_mm, dir, address);
175 if (!pmd)
176 goto err;
178 if (alloc_area_pmd(pmd, address, end - address, gfp_mask, prot, pages))
179 goto err; // The kernel NEVER reclaims pmds, so no need to undo pmd_alloc() here
181 address = (address + PGDIR_SIZE) & PGDIR_MASK;
182 dir++;
183 } while (address && (address < end));
184 spin_unlock(&init_mm.page_table_lock);
185 flush_cache_all();
186 return 0;
187 err:
188 spin_unlock(&init_mm.page_table_lock);
189 flush_cache_all();
190 if (address > start)
191 vmfree_area_pages(start, address - start);
192 return -ENOMEM;
193 }
195 int vmalloc_area_pages(unsigned long address, unsigned long size,
196 int gfp_mask, pgprot_t prot)
197 {
198 return __vmalloc_area_pages(address, size, gfp_mask, prot, NULL);
199 }
201 struct vm_struct * get_vm_area(unsigned long size, unsigned long flags)
202 {
203 unsigned long addr, next;
204 struct vm_struct **p, *tmp, *area;
206 area = (struct vm_struct *) kmalloc(sizeof(*area), GFP_KERNEL);
207 if (!area)
208 return NULL;
210 size += PAGE_SIZE;
211 if (!size) {
212 kfree (area);
213 return NULL;
214 }
216 addr = VMALLOC_START;
217 write_lock(&vmlist_lock);
218 for (p = &vmlist; (tmp = *p) ; p = &tmp->next) {
219 if ((size + addr) < addr)
220 goto out;
221 if (size + addr <= (unsigned long) tmp->addr)
222 break;
223 next = tmp->size + (unsigned long) tmp->addr;
224 if (next > addr)
225 addr = next;
226 if (addr > VMALLOC_END-size)
227 goto out;
228 }
229 area->flags = flags;
230 area->addr = (void *)addr;
231 area->size = size;
232 area->next = *p;
233 *p = area;
234 write_unlock(&vmlist_lock);
235 return area;
237 out:
238 write_unlock(&vmlist_lock);
239 kfree(area);
240 return NULL;
241 }
243 void __vfree(void * addr, int free_area_pages)
244 {
245 struct vm_struct **p, *tmp;
247 if (!addr)
248 return;
249 if ((PAGE_SIZE-1) & (unsigned long) addr) {
250 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
251 return;
252 }
253 write_lock(&vmlist_lock);
254 for (p = &vmlist ; (tmp = *p) ; p = &tmp->next) {
255 if (tmp->addr == addr) {
256 *p = tmp->next;
257 if (free_area_pages)
258 vmfree_area_pages(VMALLOC_VMADDR(tmp->addr), tmp->size);
259 write_unlock(&vmlist_lock);
260 kfree(tmp);
261 return;
262 }
263 }
264 write_unlock(&vmlist_lock);
265 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", addr);
266 }
268 void vfree(void * addr)
269 {
270 __vfree(addr,1);
271 }
273 void * __vmalloc (unsigned long size, int gfp_mask, pgprot_t prot)
274 {
275 void * addr;
276 struct vm_struct *area;
278 size = PAGE_ALIGN(size);
279 if (!size || (size >> PAGE_SHIFT) > num_physpages)
280 return NULL;
281 area = get_vm_area(size, VM_ALLOC);
282 if (!area)
283 return NULL;
284 addr = area->addr;
285 if (__vmalloc_area_pages(VMALLOC_VMADDR(addr), size, gfp_mask,
286 prot, NULL)) {
287 __vfree(addr, 0);
288 return NULL;
289 }
290 return addr;
291 }
293 void * vmap(struct page **pages, int count,
294 unsigned long flags, pgprot_t prot)
295 {
296 void * addr;
297 struct vm_struct *area;
298 unsigned long size = count << PAGE_SHIFT;
300 if (!size || size > (max_mapnr << PAGE_SHIFT))
301 return NULL;
302 area = get_vm_area(size, flags);
303 if (!area) {
304 return NULL;
305 }
306 addr = area->addr;
307 if (__vmalloc_area_pages(VMALLOC_VMADDR(addr), size, 0,
308 prot, &pages)) {
309 __vfree(addr, 0);
310 return NULL;
311 }
312 return addr;
313 }
315 long vread(char *buf, char *addr, unsigned long count)
316 {
317 struct vm_struct *tmp;
318 char *vaddr, *buf_start = buf;
319 unsigned long n;
321 /* Don't allow overflow */
322 if ((unsigned long) addr + count < count)
323 count = -(unsigned long) addr;
325 read_lock(&vmlist_lock);
326 for (tmp = vmlist; tmp; tmp = tmp->next) {
327 vaddr = (char *) tmp->addr;
328 if (addr >= vaddr + tmp->size - PAGE_SIZE)
329 continue;
330 while (addr < vaddr) {
331 if (count == 0)
332 goto finished;
333 *buf = '\0';
334 buf++;
335 addr++;
336 count--;
337 }
338 n = vaddr + tmp->size - PAGE_SIZE - addr;
339 do {
340 if (count == 0)
341 goto finished;
342 *buf = *addr;
343 buf++;
344 addr++;
345 count--;
346 } while (--n > 0);
347 }
348 finished:
349 read_unlock(&vmlist_lock);
350 return buf - buf_start;
351 }
353 long vwrite(char *buf, char *addr, unsigned long count)
354 {
355 struct vm_struct *tmp;
356 char *vaddr, *buf_start = buf;
357 unsigned long n;
359 /* Don't allow overflow */
360 if ((unsigned long) addr + count < count)
361 count = -(unsigned long) addr;
363 read_lock(&vmlist_lock);
364 for (tmp = vmlist; tmp; tmp = tmp->next) {
365 vaddr = (char *) tmp->addr;
366 if (addr >= vaddr + tmp->size - PAGE_SIZE)
367 continue;
368 while (addr < vaddr) {
369 if (count == 0)
370 goto finished;
371 buf++;
372 addr++;
373 count--;
374 }
375 n = vaddr + tmp->size - PAGE_SIZE - addr;
376 do {
377 if (count == 0)
378 goto finished;
379 *addr = *buf;
380 buf++;
381 addr++;
382 count--;
383 } while (--n > 0);
384 }
385 finished:
386 read_unlock(&vmlist_lock);
387 return buf - buf_start;
388 }