ia64/linux-2.6.18-xen.hg

view arch/i386/mm/hypervisor.c @ 708:e410857fd83c

Remove contiguous_bitmap[] as it's no longer needed.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Oct 22 14:55:29 2008 +0100 (2008-10-22)
parents 9dcb40286ad4
children 39a8680e7a70
line source
1 /******************************************************************************
2 * mm/hypervisor.c
3 *
4 * Update page tables via the hypervisor.
5 *
6 * Copyright (c) 2002-2004, K A Fraser
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License version 2
10 * as published by the Free Software Foundation; or, when distributed
11 * separately from the Linux kernel or incorporated into other
12 * software packages, subject to the following license:
13 *
14 * Permission is hereby granted, free of charge, to any person obtaining a copy
15 * of this source file (the "Software"), to deal in the Software without
16 * restriction, including without limitation the rights to use, copy, modify,
17 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
18 * and to permit persons to whom the Software is furnished to do so, subject to
19 * the following conditions:
20 *
21 * The above copyright notice and this permission notice shall be included in
22 * all copies or substantial portions of the Software.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
30 * IN THE SOFTWARE.
31 */
33 #include <linux/sched.h>
34 #include <linux/mm.h>
35 #include <linux/vmalloc.h>
36 #include <asm/page.h>
37 #include <asm/pgtable.h>
38 #include <asm/hypervisor.h>
39 #include <xen/balloon.h>
40 #include <xen/features.h>
41 #include <xen/interface/memory.h>
42 #include <linux/module.h>
43 #include <linux/percpu.h>
44 #include <asm/tlbflush.h>
45 #include <linux/highmem.h>
47 void xen_l1_entry_update(pte_t *ptr, pte_t val)
48 {
49 mmu_update_t u;
50 #ifdef CONFIG_HIGHPTE
51 u.ptr = ((unsigned long)ptr >= (unsigned long)high_memory) ?
52 arbitrary_virt_to_machine(ptr) : virt_to_machine(ptr);
53 #else
54 u.ptr = virt_to_machine(ptr);
55 #endif
56 u.val = __pte_val(val);
57 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
58 }
59 EXPORT_SYMBOL_GPL(xen_l1_entry_update);
61 void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
62 {
63 mmu_update_t u;
64 u.ptr = virt_to_machine(ptr);
65 u.val = __pmd_val(val);
66 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
67 }
69 #if defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
70 void xen_l3_entry_update(pud_t *ptr, pud_t val)
71 {
72 mmu_update_t u;
73 u.ptr = virt_to_machine(ptr);
74 u.val = __pud_val(val);
75 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
76 }
77 #endif
79 #ifdef CONFIG_X86_64
80 void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
81 {
82 mmu_update_t u;
83 u.ptr = virt_to_machine(ptr);
84 u.val = __pgd_val(val);
85 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
86 }
87 #endif /* CONFIG_X86_64 */
89 void xen_pt_switch(unsigned long ptr)
90 {
91 struct mmuext_op op;
92 op.cmd = MMUEXT_NEW_BASEPTR;
93 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
94 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
95 }
97 void xen_new_user_pt(unsigned long ptr)
98 {
99 struct mmuext_op op;
100 op.cmd = MMUEXT_NEW_USER_BASEPTR;
101 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
102 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
103 }
105 void xen_tlb_flush(void)
106 {
107 struct mmuext_op op;
108 op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
109 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
110 }
111 EXPORT_SYMBOL(xen_tlb_flush);
113 void xen_invlpg(unsigned long ptr)
114 {
115 struct mmuext_op op;
116 op.cmd = MMUEXT_INVLPG_LOCAL;
117 op.arg1.linear_addr = ptr & PAGE_MASK;
118 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
119 }
120 EXPORT_SYMBOL(xen_invlpg);
122 #ifdef CONFIG_SMP
124 void xen_tlb_flush_all(void)
125 {
126 struct mmuext_op op;
127 op.cmd = MMUEXT_TLB_FLUSH_ALL;
128 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
129 }
131 void xen_tlb_flush_mask(cpumask_t *mask)
132 {
133 struct mmuext_op op;
134 if ( cpus_empty(*mask) )
135 return;
136 op.cmd = MMUEXT_TLB_FLUSH_MULTI;
137 set_xen_guest_handle(op.arg2.vcpumask, mask->bits);
138 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
139 }
141 void xen_invlpg_all(unsigned long ptr)
142 {
143 struct mmuext_op op;
144 op.cmd = MMUEXT_INVLPG_ALL;
145 op.arg1.linear_addr = ptr & PAGE_MASK;
146 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
147 }
149 void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
150 {
151 struct mmuext_op op;
152 if ( cpus_empty(*mask) )
153 return;
154 op.cmd = MMUEXT_INVLPG_MULTI;
155 op.arg1.linear_addr = ptr & PAGE_MASK;
156 set_xen_guest_handle(op.arg2.vcpumask, mask->bits);
157 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
158 }
160 #endif /* CONFIG_SMP */
162 void xen_pgd_pin(unsigned long ptr)
163 {
164 struct mmuext_op op;
165 #ifdef CONFIG_X86_64
166 op.cmd = MMUEXT_PIN_L4_TABLE;
167 #elif defined(CONFIG_X86_PAE)
168 op.cmd = MMUEXT_PIN_L3_TABLE;
169 #else
170 op.cmd = MMUEXT_PIN_L2_TABLE;
171 #endif
172 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
173 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
174 }
176 void xen_pgd_unpin(unsigned long ptr)
177 {
178 struct mmuext_op op;
179 op.cmd = MMUEXT_UNPIN_TABLE;
180 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
181 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
182 }
184 void xen_set_ldt(const void *ptr, unsigned int ents)
185 {
186 struct mmuext_op op;
187 op.cmd = MMUEXT_SET_LDT;
188 op.arg1.linear_addr = (unsigned long)ptr;
189 op.arg2.nr_ents = ents;
190 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
191 }
193 /* Protected by balloon_lock. */
194 #define MAX_CONTIG_ORDER 9 /* 2MB */
195 static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
196 static unsigned long limited_frames[1<<MAX_CONTIG_ORDER];
197 static multicall_entry_t cr_mcl[1<<MAX_CONTIG_ORDER];
199 /* Ensure multi-page extents are contiguous in machine memory. */
200 int xen_create_contiguous_region(
201 unsigned long vstart, unsigned int order, unsigned int address_bits)
202 {
203 unsigned long *in_frames = discontig_frames, out_frame;
204 unsigned long frame, flags;
205 unsigned int i;
206 int rc, success;
207 struct xen_memory_exchange exchange = {
208 .in = {
209 .nr_extents = 1UL << order,
210 .extent_order = 0,
211 .domid = DOMID_SELF
212 },
213 .out = {
214 .nr_extents = 1,
215 .extent_order = order,
216 .address_bits = address_bits,
217 .domid = DOMID_SELF
218 }
219 };
221 /*
222 * Currently an auto-translated guest will not perform I/O, nor will
223 * it require PAE page directories below 4GB. Therefore any calls to
224 * this function are redundant and can be ignored.
225 */
226 if (xen_feature(XENFEAT_auto_translated_physmap))
227 return 0;
229 if (unlikely(order > MAX_CONTIG_ORDER))
230 return -ENOMEM;
232 set_xen_guest_handle(exchange.in.extent_start, in_frames);
233 set_xen_guest_handle(exchange.out.extent_start, &out_frame);
235 scrub_pages((void *)vstart, 1 << order);
237 balloon_lock(flags);
239 /* 1. Zap current PTEs, remembering MFNs. */
240 for (i = 0; i < (1U<<order); i++) {
241 in_frames[i] = pfn_to_mfn((__pa(vstart) >> PAGE_SHIFT) + i);
242 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
243 __pte_ma(0), 0);
244 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i,
245 INVALID_P2M_ENTRY);
246 }
247 if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
248 BUG();
250 /* 2. Get a new contiguous memory extent. */
251 out_frame = __pa(vstart) >> PAGE_SHIFT;
252 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
253 success = (exchange.nr_exchanged == (1UL << order));
254 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
255 BUG_ON(success && (rc != 0));
256 #if CONFIG_XEN_COMPAT <= 0x030002
257 if (unlikely(rc == -ENOSYS)) {
258 /* Compatibility when XENMEM_exchange is unsupported. */
259 if (HYPERVISOR_memory_op(XENMEM_decrease_reservation,
260 &exchange.in) != (1UL << order))
261 BUG();
262 success = (HYPERVISOR_memory_op(XENMEM_populate_physmap,
263 &exchange.out) == 1);
264 if (!success) {
265 /* Couldn't get special memory: fall back to normal. */
266 for (i = 0; i < (1U<<order); i++)
267 in_frames[i] = (__pa(vstart)>>PAGE_SHIFT) + i;
268 if (HYPERVISOR_memory_op(XENMEM_populate_physmap,
269 &exchange.in) != (1UL<<order))
270 BUG();
271 }
272 }
273 #endif
275 /* 3. Map the new extent in place of old pages. */
276 for (i = 0; i < (1U<<order); i++) {
277 frame = success ? (out_frame + i) : in_frames[i];
278 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
279 pfn_pte_ma(frame, PAGE_KERNEL), 0);
280 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i, frame);
281 }
283 cr_mcl[i - 1].args[MULTI_UVMFLAGS_INDEX] = order
284 ? UVMF_TLB_FLUSH|UVMF_ALL
285 : UVMF_INVLPG|UVMF_ALL;
286 if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
287 BUG();
289 balloon_unlock(flags);
291 return success ? 0 : -ENOMEM;
292 }
293 EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
295 void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
296 {
297 unsigned long *out_frames = discontig_frames, in_frame;
298 unsigned long frame, flags;
299 unsigned int i;
300 int rc, success;
301 struct xen_memory_exchange exchange = {
302 .in = {
303 .nr_extents = 1,
304 .extent_order = order,
305 .domid = DOMID_SELF
306 },
307 .out = {
308 .nr_extents = 1UL << order,
309 .extent_order = 0,
310 .domid = DOMID_SELF
311 }
312 };
314 if (xen_feature(XENFEAT_auto_translated_physmap))
315 return;
317 if (unlikely(order > MAX_CONTIG_ORDER))
318 return;
320 set_xen_guest_handle(exchange.in.extent_start, &in_frame);
321 set_xen_guest_handle(exchange.out.extent_start, out_frames);
323 scrub_pages((void *)vstart, 1 << order);
325 balloon_lock(flags);
327 /* 1. Find start MFN of contiguous extent. */
328 in_frame = pfn_to_mfn(__pa(vstart) >> PAGE_SHIFT);
330 /* 2. Zap current PTEs. */
331 for (i = 0; i < (1U<<order); i++) {
332 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
333 __pte_ma(0), 0);
334 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i,
335 INVALID_P2M_ENTRY);
336 out_frames[i] = (__pa(vstart) >> PAGE_SHIFT) + i;
337 }
338 if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
339 BUG();
341 /* 3. Do the exchange for non-contiguous MFNs. */
342 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
343 success = (exchange.nr_exchanged == 1);
344 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
345 BUG_ON(success && (rc != 0));
346 #if CONFIG_XEN_COMPAT <= 0x030002
347 if (unlikely(rc == -ENOSYS)) {
348 /* Compatibility when XENMEM_exchange is unsupported. */
349 if (HYPERVISOR_memory_op(XENMEM_decrease_reservation,
350 &exchange.in) != 1)
351 BUG();
352 if (HYPERVISOR_memory_op(XENMEM_populate_physmap,
353 &exchange.out) != (1UL << order))
354 BUG();
355 success = 1;
356 }
357 #endif
359 /* 4. Map new pages in place of old pages. */
360 for (i = 0; i < (1U<<order); i++) {
361 frame = success ? out_frames[i] : (in_frame + i);
362 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
363 pfn_pte_ma(frame, PAGE_KERNEL), 0);
364 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i, frame);
365 }
367 cr_mcl[i - 1].args[MULTI_UVMFLAGS_INDEX] = order
368 ? UVMF_TLB_FLUSH|UVMF_ALL
369 : UVMF_INVLPG|UVMF_ALL;
370 if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
371 BUG();
373 balloon_unlock(flags);
374 }
375 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
377 int xen_limit_pages_to_max_mfn(
378 struct page *pages, unsigned int order, unsigned int address_bits)
379 {
380 unsigned long flags, frame;
381 unsigned long *in_frames = discontig_frames, *out_frames = limited_frames;
382 struct page *page;
383 unsigned int i, n, nr_mcl;
384 int rc, success;
385 DECLARE_BITMAP(limit_map, 1 << MAX_CONTIG_ORDER);
387 struct xen_memory_exchange exchange = {
388 .in = {
389 .extent_order = 0,
390 .domid = DOMID_SELF
391 },
392 .out = {
393 .extent_order = 0,
394 .address_bits = address_bits,
395 .domid = DOMID_SELF
396 }
397 };
399 if (xen_feature(XENFEAT_auto_translated_physmap))
400 return 0;
402 if (unlikely(order > MAX_CONTIG_ORDER))
403 return -ENOMEM;
405 bitmap_zero(limit_map, 1U << order);
406 set_xen_guest_handle(exchange.in.extent_start, in_frames);
407 set_xen_guest_handle(exchange.out.extent_start, out_frames);
409 /* 0. Scrub the pages. */
410 for (i = 0, n = 0; i < 1U<<order ; i++) {
411 page = &pages[i];
412 if (!(pfn_to_mfn(page_to_pfn(page)) >> (address_bits - PAGE_SHIFT)))
413 continue;
414 __set_bit(i, limit_map);
416 if (!PageHighMem(page))
417 scrub_pages(page_address(page), 1);
418 #ifdef CONFIG_XEN_SCRUB_PAGES
419 else {
420 scrub_pages(kmap(page), 1);
421 kunmap(page);
422 ++n;
423 }
424 #endif
425 }
426 if (bitmap_empty(limit_map, 1U << order))
427 return 0;
429 if (n)
430 kmap_flush_unused();
432 balloon_lock(flags);
434 /* 1. Zap current PTEs (if any), remembering MFNs. */
435 for (i = 0, n = 0, nr_mcl = 0; i < (1U<<order); i++) {
436 if(!test_bit(i, limit_map))
437 continue;
438 page = &pages[i];
440 out_frames[n] = page_to_pfn(page);
441 in_frames[n] = pfn_to_mfn(out_frames[n]);
443 if (!PageHighMem(page))
444 MULTI_update_va_mapping(cr_mcl + nr_mcl++,
445 (unsigned long)page_address(page),
446 __pte_ma(0), 0);
448 set_phys_to_machine(out_frames[n], INVALID_P2M_ENTRY);
449 ++n;
450 }
451 if (nr_mcl && HYPERVISOR_multicall_check(cr_mcl, nr_mcl, NULL))
452 BUG();
454 /* 2. Get new memory below the required limit. */
455 exchange.in.nr_extents = n;
456 exchange.out.nr_extents = n;
457 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
458 success = (exchange.nr_exchanged == n);
459 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
460 BUG_ON(success && (rc != 0));
461 #if CONFIG_XEN_COMPAT <= 0x030002
462 if (unlikely(rc == -ENOSYS)) {
463 /* Compatibility when XENMEM_exchange is unsupported. */
464 if (HYPERVISOR_memory_op(XENMEM_decrease_reservation,
465 &exchange.in) != n)
466 BUG();
467 if (HYPERVISOR_memory_op(XENMEM_populate_physmap,
468 &exchange.out) != n)
469 BUG();
470 success = 1;
471 }
472 #endif
474 /* 3. Map the new pages in place of old pages. */
475 for (i = 0, n = 0, nr_mcl = 0; i < (1U<<order); i++) {
476 if(!test_bit(i, limit_map))
477 continue;
478 page = &pages[i];
480 frame = success ? out_frames[n] : in_frames[n];
482 if (!PageHighMem(page))
483 MULTI_update_va_mapping(cr_mcl + nr_mcl++,
484 (unsigned long)page_address(page),
485 pfn_pte_ma(frame, PAGE_KERNEL), 0);
487 set_phys_to_machine(page_to_pfn(page), frame);
488 ++n;
489 }
490 if (nr_mcl) {
491 cr_mcl[nr_mcl - 1].args[MULTI_UVMFLAGS_INDEX] = order
492 ? UVMF_TLB_FLUSH|UVMF_ALL
493 : UVMF_INVLPG|UVMF_ALL;
494 if (HYPERVISOR_multicall_check(cr_mcl, nr_mcl, NULL))
495 BUG();
496 }
498 balloon_unlock(flags);
500 return success ? 0 : -ENOMEM;
501 }
502 EXPORT_SYMBOL_GPL(xen_limit_pages_to_max_mfn);
504 #ifdef __i386__
505 int write_ldt_entry(void *ldt, int entry, __u32 entry_a, __u32 entry_b)
506 {
507 __u32 *lp = (__u32 *)((char *)ldt + entry * 8);
508 maddr_t mach_lp = arbitrary_virt_to_machine(lp);
509 return HYPERVISOR_update_descriptor(
510 mach_lp, (u64)entry_a | ((u64)entry_b<<32));
511 }
512 #endif
514 #define MAX_BATCHED_FULL_PTES 32
516 int xen_change_pte_range(struct mm_struct *mm, pmd_t *pmd,
517 unsigned long addr, unsigned long end, pgprot_t newprot)
518 {
519 int rc = 0, i = 0;
520 mmu_update_t u[MAX_BATCHED_FULL_PTES];
521 pte_t *pte;
522 spinlock_t *ptl;
524 if (!xen_feature(XENFEAT_mmu_pt_update_preserve_ad))
525 return 0;
527 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
528 do {
529 if (pte_present(*pte)) {
530 u[i].ptr = (__pmd_val(*pmd) & PHYSICAL_PAGE_MASK)
531 | ((unsigned long)pte & ~PAGE_MASK)
532 | MMU_PT_UPDATE_PRESERVE_AD;
533 u[i].val = __pte_val(pte_modify(*pte, newprot));
534 if (++i == MAX_BATCHED_FULL_PTES) {
535 if ((rc = HYPERVISOR_mmu_update(
536 &u[0], i, NULL, DOMID_SELF)) != 0)
537 break;
538 i = 0;
539 }
540 }
541 } while (pte++, addr += PAGE_SIZE, addr != end);
542 if (i)
543 rc = HYPERVISOR_mmu_update( &u[0], i, NULL, DOMID_SELF);
544 pte_unmap_unlock(pte - 1, ptl);
545 BUG_ON(rc && rc != -ENOSYS);
546 return !rc;
547 }