ia64/xen-unstable

view linux-2.4.26-xen-sparse/mm/mremap.c @ 1527:a815a43920c0

bitkeeper revision 1.994 (40d6ed9ePUmxTwjKFv1vprN2-xFpmQ)

Install mkernel odules with 'make install'
author iap10@labyrinth.cl.cam.ac.uk
date Mon Jun 21 14:15:58 2004 +0000 (2004-06-21)
parents f3123052268f
children
line source
1 /*
2 * linux/mm/remap.c
3 *
4 * (C) Copyright 1996 Linus Torvalds
5 */
7 #include <linux/slab.h>
8 #include <linux/smp_lock.h>
9 #include <linux/shm.h>
10 #include <linux/mman.h>
11 #include <linux/swap.h>
13 #include <asm/uaccess.h>
14 #include <asm/pgalloc.h>
16 extern int vm_enough_memory(long pages);
18 static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
19 {
20 pgd_t * pgd;
21 pmd_t * pmd;
22 pte_t * pte = NULL;
24 pgd = pgd_offset(mm, addr);
25 if (pgd_none(*pgd))
26 goto end;
27 if (pgd_bad(*pgd)) {
28 pgd_ERROR(*pgd);
29 pgd_clear(pgd);
30 goto end;
31 }
33 pmd = pmd_offset(pgd, addr);
34 if (pmd_none(*pmd))
35 goto end;
36 if (pmd_bad(*pmd)) {
37 pmd_ERROR(*pmd);
38 pmd_clear(pmd);
39 goto end;
40 }
42 pte = pte_offset(pmd, addr);
43 if (pte_none(*pte))
44 pte = NULL;
45 end:
46 return pte;
47 }
49 static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr)
50 {
51 pmd_t * pmd;
52 pte_t * pte = NULL;
54 pmd = pmd_alloc(mm, pgd_offset(mm, addr), addr);
55 if (pmd)
56 pte = pte_alloc(mm, pmd, addr);
57 return pte;
58 }
60 static inline int copy_one_pte(struct mm_struct *mm, pte_t * src, pte_t * dst)
61 {
62 int error = 0;
63 pte_t pte;
65 if (!pte_none(*src)) {
66 pte = ptep_get_and_clear(src);
67 if (!dst) {
68 /* No dest? We must put it back. */
69 dst = src;
70 error++;
71 }
72 set_pte(dst, pte);
73 }
74 return error;
75 }
77 static int move_one_page(struct mm_struct *mm, unsigned long old_addr, unsigned long new_addr)
78 {
79 int error = 0;
80 pte_t * src, * dst;
82 spin_lock(&mm->page_table_lock);
83 src = get_one_pte(mm, old_addr);
84 if (src) {
85 dst = alloc_one_pte(mm, new_addr);
86 src = get_one_pte(mm, old_addr);
87 if (src)
88 error = copy_one_pte(mm, src, dst);
89 }
90 spin_unlock(&mm->page_table_lock);
91 return error;
92 }
94 static int move_page_tables(struct mm_struct * mm,
95 unsigned long new_addr, unsigned long old_addr, unsigned long len)
96 {
97 unsigned long offset = len;
99 flush_cache_range(mm, old_addr, old_addr + len);
101 /*
102 * This is not the clever way to do this, but we're taking the
103 * easy way out on the assumption that most remappings will be
104 * only a few pages.. This also makes error recovery easier.
105 */
106 while (offset) {
107 offset -= PAGE_SIZE;
108 if (move_one_page(mm, old_addr + offset, new_addr + offset))
109 goto oops_we_failed;
110 }
111 flush_tlb_range(mm, old_addr, old_addr + len);
112 return 0;
114 /*
115 * Ok, the move failed because we didn't have enough pages for
116 * the new page table tree. This is unlikely, but we have to
117 * take the possibility into account. In that case we just move
118 * all the pages back (this will work, because we still have
119 * the old page tables)
120 */
121 oops_we_failed:
122 XEN_flush_page_update_queue();
123 flush_cache_range(mm, new_addr, new_addr + len);
124 while ((offset += PAGE_SIZE) < len)
125 move_one_page(mm, new_addr + offset, old_addr + offset);
126 XEN_flush_page_update_queue();
127 zap_page_range(mm, new_addr, len);
128 return -1;
129 }
131 static inline unsigned long move_vma(struct vm_area_struct * vma,
132 unsigned long addr, unsigned long old_len, unsigned long new_len,
133 unsigned long new_addr)
134 {
135 struct mm_struct * mm = vma->vm_mm;
136 struct vm_area_struct * new_vma, * next, * prev;
137 int allocated_vma;
139 new_vma = NULL;
140 next = find_vma_prev(mm, new_addr, &prev);
141 if (next) {
142 if (prev && prev->vm_end == new_addr &&
143 can_vma_merge(prev, vma->vm_flags) && !vma->vm_file && !(vma->vm_flags & VM_SHARED)) {
144 spin_lock(&mm->page_table_lock);
145 prev->vm_end = new_addr + new_len;
146 spin_unlock(&mm->page_table_lock);
147 new_vma = prev;
148 if (next != prev->vm_next)
149 BUG();
150 if (prev->vm_end == next->vm_start && can_vma_merge(next, prev->vm_flags)) {
151 spin_lock(&mm->page_table_lock);
152 prev->vm_end = next->vm_end;
153 __vma_unlink(mm, next, prev);
154 spin_unlock(&mm->page_table_lock);
156 mm->map_count--;
157 kmem_cache_free(vm_area_cachep, next);
158 }
159 } else if (next->vm_start == new_addr + new_len &&
160 can_vma_merge(next, vma->vm_flags) && !vma->vm_file && !(vma->vm_flags & VM_SHARED)) {
161 spin_lock(&mm->page_table_lock);
162 next->vm_start = new_addr;
163 spin_unlock(&mm->page_table_lock);
164 new_vma = next;
165 }
166 } else {
167 prev = find_vma(mm, new_addr-1);
168 if (prev && prev->vm_end == new_addr &&
169 can_vma_merge(prev, vma->vm_flags) && !vma->vm_file && !(vma->vm_flags & VM_SHARED)) {
170 spin_lock(&mm->page_table_lock);
171 prev->vm_end = new_addr + new_len;
172 spin_unlock(&mm->page_table_lock);
173 new_vma = prev;
174 }
175 }
177 allocated_vma = 0;
178 if (!new_vma) {
179 new_vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
180 if (!new_vma)
181 goto out;
182 allocated_vma = 1;
183 }
185 if (!move_page_tables(current->mm, new_addr, addr, old_len)) {
186 unsigned long vm_locked = vma->vm_flags & VM_LOCKED;
188 if (allocated_vma) {
189 *new_vma = *vma;
190 new_vma->vm_start = new_addr;
191 new_vma->vm_end = new_addr+new_len;
192 new_vma->vm_pgoff += (addr-vma->vm_start) >> PAGE_SHIFT;
193 new_vma->vm_raend = 0;
194 if (new_vma->vm_file)
195 get_file(new_vma->vm_file);
196 if (new_vma->vm_ops && new_vma->vm_ops->open)
197 new_vma->vm_ops->open(new_vma);
198 insert_vm_struct(current->mm, new_vma);
199 }
201 do_munmap(current->mm, addr, old_len);
203 current->mm->total_vm += new_len >> PAGE_SHIFT;
204 if (vm_locked) {
205 current->mm->locked_vm += new_len >> PAGE_SHIFT;
206 if (new_len > old_len)
207 make_pages_present(new_addr + old_len,
208 new_addr + new_len);
209 }
210 return new_addr;
211 }
212 if (allocated_vma)
213 kmem_cache_free(vm_area_cachep, new_vma);
214 out:
215 return -ENOMEM;
216 }
218 /*
219 * Expand (or shrink) an existing mapping, potentially moving it at the
220 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
221 *
222 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
223 * This option implies MREMAP_MAYMOVE.
224 */
225 unsigned long do_mremap(unsigned long addr,
226 unsigned long old_len, unsigned long new_len,
227 unsigned long flags, unsigned long new_addr)
228 {
229 struct vm_area_struct *vma;
230 unsigned long ret = -EINVAL;
232 if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
233 goto out;
235 if (addr & ~PAGE_MASK)
236 goto out;
238 old_len = PAGE_ALIGN(old_len);
239 new_len = PAGE_ALIGN(new_len);
241 /* new_addr is only valid if MREMAP_FIXED is specified */
242 if (flags & MREMAP_FIXED) {
243 if (new_addr & ~PAGE_MASK)
244 goto out;
245 if (!(flags & MREMAP_MAYMOVE))
246 goto out;
248 if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
249 goto out;
250 /*
251 * Allow new_len == 0 only if new_addr == addr
252 * to preserve truncation in place (that was working
253 * safe and some app may depend on it).
254 */
255 if (unlikely(!new_len && new_addr != addr))
256 goto out;
258 /* Check if the location we're moving into overlaps the
259 * old location at all, and fail if it does.
260 */
261 if ((new_addr <= addr) && (new_addr+new_len) > addr)
262 goto out;
264 if ((addr <= new_addr) && (addr+old_len) > new_addr)
265 goto out;
267 ret = do_munmap(current->mm, new_addr, new_len);
268 if (ret && new_len)
269 goto out;
270 }
272 /*
273 * Always allow a shrinking remap: that just unmaps
274 * the unnecessary pages..
275 */
276 if (old_len >= new_len) {
277 ret = do_munmap(current->mm, addr+new_len, old_len - new_len);
278 if (ret && old_len != new_len)
279 goto out;
280 ret = addr;
281 if (!(flags & MREMAP_FIXED) || (new_addr == addr))
282 goto out;
283 }
285 /*
286 * Ok, we need to grow.. or relocate.
287 */
288 ret = -EFAULT;
289 vma = find_vma(current->mm, addr);
290 if (!vma || vma->vm_start > addr)
291 goto out;
292 /* We can't remap across vm area boundaries */
293 if (old_len > vma->vm_end - addr)
294 goto out;
295 if (vma->vm_flags & VM_DONTEXPAND) {
296 if (new_len > old_len)
297 goto out;
298 }
299 if (vma->vm_flags & VM_LOCKED) {
300 unsigned long locked = current->mm->locked_vm << PAGE_SHIFT;
301 locked += new_len - old_len;
302 ret = -EAGAIN;
303 if (locked > current->rlim[RLIMIT_MEMLOCK].rlim_cur)
304 goto out;
305 }
306 ret = -ENOMEM;
307 if ((current->mm->total_vm << PAGE_SHIFT) + (new_len - old_len)
308 > current->rlim[RLIMIT_AS].rlim_cur)
309 goto out;
310 /* Private writable mapping? Check memory availability.. */
311 if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) == VM_WRITE &&
312 !(flags & MAP_NORESERVE) &&
313 !vm_enough_memory((new_len - old_len) >> PAGE_SHIFT))
314 goto out;
316 #if defined(CONFIG_XEN_PRIVILEGED_GUEST)
317 /* mremap() unsupported for I/O mappings in Xenolinux. */
318 ret = -EINVAL;
319 if (vma->vm_flags & VM_IO)
320 goto out;
321 #endif
323 /* old_len exactly to the end of the area..
324 * And we're not relocating the area.
325 */
326 if (old_len == vma->vm_end - addr &&
327 !((flags & MREMAP_FIXED) && (addr != new_addr)) &&
328 (old_len != new_len || !(flags & MREMAP_MAYMOVE))) {
329 unsigned long max_addr = TASK_SIZE;
330 if (vma->vm_next)
331 max_addr = vma->vm_next->vm_start;
332 /* can we just expand the current mapping? */
333 if (max_addr - addr >= new_len) {
334 int pages = (new_len - old_len) >> PAGE_SHIFT;
335 spin_lock(&vma->vm_mm->page_table_lock);
336 vma->vm_end = addr + new_len;
337 spin_unlock(&vma->vm_mm->page_table_lock);
338 current->mm->total_vm += pages;
339 if (vma->vm_flags & VM_LOCKED) {
340 current->mm->locked_vm += pages;
341 make_pages_present(addr + old_len,
342 addr + new_len);
343 }
344 ret = addr;
345 goto out;
346 }
347 }
349 /*
350 * We weren't able to just expand or shrink the area,
351 * we need to create a new one and move it..
352 */
353 ret = -ENOMEM;
354 if (flags & MREMAP_MAYMOVE) {
355 if (!(flags & MREMAP_FIXED)) {
356 unsigned long map_flags = 0;
357 if (vma->vm_flags & VM_SHARED)
358 map_flags |= MAP_SHARED;
360 new_addr = get_unmapped_area(vma->vm_file, 0, new_len, vma->vm_pgoff, map_flags);
361 ret = new_addr;
362 if (new_addr & ~PAGE_MASK)
363 goto out;
364 }
365 ret = move_vma(vma, addr, old_len, new_len, new_addr);
366 }
367 out:
368 return ret;
369 }
371 asmlinkage unsigned long sys_mremap(unsigned long addr,
372 unsigned long old_len, unsigned long new_len,
373 unsigned long flags, unsigned long new_addr)
374 {
375 unsigned long ret;
377 down_write(&current->mm->mmap_sem);
378 ret = do_mremap(addr, old_len, new_len, flags, new_addr);
379 up_write(&current->mm->mmap_sem);
380 return ret;
381 }