ia64/xen-unstable

view linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c @ 6689:7d0fb56b4a91

merge?
author cl349@firebug.cl.cam.ac.uk
date Wed Sep 07 19:01:31 2005 +0000 (2005-09-07)
parents dd668f7527cb cbf43be4cc17
children b2f4823b6ff0 b35215021b32 1936ccaccf5e
line source
1 /******************************************************************************
2 * mm/hypervisor.c
3 *
4 * Update page tables via the hypervisor.
5 *
6 * Copyright (c) 2002-2004, K A Fraser
7 *
8 * This file may be distributed separately from the Linux kernel, or
9 * incorporated into other software packages, subject to the following license:
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a copy
12 * of this source file (the "Software"), to deal in the Software without
13 * restriction, including without limitation the rights to use, copy, modify,
14 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
15 * and to permit persons to whom the Software is furnished to do so, subject to
16 * the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included in
19 * all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 * IN THE SOFTWARE.
28 */
30 #include <linux/config.h>
31 #include <linux/sched.h>
32 #include <linux/mm.h>
33 #include <linux/vmalloc.h>
34 #include <asm/page.h>
35 #include <asm/pgtable.h>
36 #include <asm-xen/hypervisor.h>
37 #include <asm-xen/balloon.h>
38 #include <asm-xen/xen-public/memory.h>
39 #include <linux/module.h>
40 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
41 #include <linux/percpu.h>
42 #include <asm/tlbflush.h>
43 #endif
45 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
46 #define pte_offset_kernel pte_offset
47 #define pud_t pgd_t
48 #define pud_offset(d, va) d
49 #elif defined(CONFIG_X86_64)
50 #define pmd_val_ma(v) (v).pmd
51 #else
52 #ifdef CONFIG_X86_PAE
53 # define pmd_val_ma(v) ((v).pmd)
54 # define pud_val_ma(v) ((v).pgd.pgd)
55 #else
56 # define pmd_val_ma(v) ((v).pud.pgd.pgd)
57 #endif
58 #endif
60 #ifndef CONFIG_XEN_SHADOW_MODE
61 void xen_l1_entry_update(pte_t *ptr, pte_t val)
62 {
63 mmu_update_t u;
64 u.ptr = virt_to_machine(ptr);
65 u.val = pte_val_ma(val);
66 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
67 }
69 void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
70 {
71 mmu_update_t u;
72 u.ptr = virt_to_machine(ptr);
73 u.val = pmd_val_ma(val);
74 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
75 }
77 #ifdef CONFIG_X86_PAE
78 void xen_l3_entry_update(pud_t *ptr, pud_t val)
79 {
80 mmu_update_t u;
81 u.ptr = virt_to_machine(ptr);
82 u.val = pud_val_ma(val);
83 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
84 }
85 #endif
87 #ifdef CONFIG_X86_64
88 void xen_l3_entry_update(pud_t *ptr, pud_t val)
89 {
90 mmu_update_t u;
91 u.ptr = virt_to_machine(ptr);
92 u.val = val.pud;
93 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
94 }
96 void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
97 {
98 mmu_update_t u;
99 u.ptr = virt_to_machine(ptr);
100 u.val = val.pgd;
101 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
102 }
103 #endif /* CONFIG_X86_64 */
104 #endif /* CONFIG_XEN_SHADOW_MODE */
106 void xen_machphys_update(unsigned long mfn, unsigned long pfn)
107 {
108 mmu_update_t u;
109 u.ptr = ((unsigned long long)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
110 u.val = pfn;
111 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
112 }
114 void xen_pt_switch(unsigned long ptr)
115 {
116 struct mmuext_op op;
117 op.cmd = MMUEXT_NEW_BASEPTR;
118 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
119 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
120 }
122 void xen_new_user_pt(unsigned long ptr)
123 {
124 struct mmuext_op op;
125 op.cmd = MMUEXT_NEW_USER_BASEPTR;
126 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
127 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
128 }
130 void xen_tlb_flush(void)
131 {
132 struct mmuext_op op;
133 op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
134 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
135 }
137 void xen_invlpg(unsigned long ptr)
138 {
139 struct mmuext_op op;
140 op.cmd = MMUEXT_INVLPG_LOCAL;
141 op.linear_addr = ptr & PAGE_MASK;
142 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
143 }
145 #ifdef CONFIG_SMP
147 void xen_tlb_flush_all(void)
148 {
149 struct mmuext_op op;
150 op.cmd = MMUEXT_TLB_FLUSH_ALL;
151 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
152 }
154 void xen_tlb_flush_mask(cpumask_t *mask)
155 {
156 struct mmuext_op op;
157 if ( cpus_empty(*mask) )
158 return;
159 op.cmd = MMUEXT_TLB_FLUSH_MULTI;
160 op.vcpumask = mask->bits;
161 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
162 }
164 void xen_invlpg_all(unsigned long ptr)
165 {
166 struct mmuext_op op;
167 op.cmd = MMUEXT_INVLPG_ALL;
168 op.linear_addr = ptr & PAGE_MASK;
169 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
170 }
172 void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
173 {
174 struct mmuext_op op;
175 if ( cpus_empty(*mask) )
176 return;
177 op.cmd = MMUEXT_INVLPG_MULTI;
178 op.vcpumask = mask->bits;
179 op.linear_addr = ptr & PAGE_MASK;
180 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
181 }
183 #endif /* CONFIG_SMP */
185 #ifndef CONFIG_XEN_SHADOW_MODE
186 void xen_pgd_pin(unsigned long ptr)
187 {
188 struct mmuext_op op;
189 #ifdef CONFIG_X86_64
190 op.cmd = MMUEXT_PIN_L4_TABLE;
191 #elif defined(CONFIG_X86_PAE)
192 op.cmd = MMUEXT_PIN_L3_TABLE;
193 #else
194 op.cmd = MMUEXT_PIN_L2_TABLE;
195 #endif
196 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
197 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
198 }
200 void xen_pgd_unpin(unsigned long ptr)
201 {
202 struct mmuext_op op;
203 op.cmd = MMUEXT_UNPIN_TABLE;
204 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
205 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
206 }
208 void xen_pte_pin(unsigned long ptr)
209 {
210 struct mmuext_op op;
211 op.cmd = MMUEXT_PIN_L1_TABLE;
212 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
213 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
214 }
216 void xen_pte_unpin(unsigned long ptr)
217 {
218 struct mmuext_op op;
219 op.cmd = MMUEXT_UNPIN_TABLE;
220 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
221 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
222 }
224 #ifdef CONFIG_X86_64
225 void xen_pud_pin(unsigned long ptr)
226 {
227 struct mmuext_op op;
228 op.cmd = MMUEXT_PIN_L3_TABLE;
229 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
230 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
231 }
233 void xen_pud_unpin(unsigned long ptr)
234 {
235 struct mmuext_op op;
236 op.cmd = MMUEXT_UNPIN_TABLE;
237 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
238 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
239 }
241 void xen_pmd_pin(unsigned long ptr)
242 {
243 struct mmuext_op op;
244 op.cmd = MMUEXT_PIN_L2_TABLE;
245 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
246 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
247 }
249 void xen_pmd_unpin(unsigned long ptr)
250 {
251 struct mmuext_op op;
252 op.cmd = MMUEXT_UNPIN_TABLE;
253 op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
254 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
255 }
256 #endif /* CONFIG_X86_64 */
257 #endif /* CONFIG_XEN_SHADOW_MODE */
259 void xen_set_ldt(unsigned long ptr, unsigned long len)
260 {
261 struct mmuext_op op;
262 op.cmd = MMUEXT_SET_LDT;
263 op.linear_addr = ptr;
264 op.nr_ents = len;
265 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
266 }
268 /*
269 * Bitmap is indexed by page number. If bit is set, the page is part of a
270 * xen_create_contiguous_region() area of memory.
271 */
272 unsigned long *contiguous_bitmap;
274 static void contiguous_bitmap_set(
275 unsigned long first_page, unsigned long nr_pages)
276 {
277 unsigned long start_off, end_off, curr_idx, end_idx;
279 curr_idx = first_page / BITS_PER_LONG;
280 start_off = first_page & (BITS_PER_LONG-1);
281 end_idx = (first_page + nr_pages) / BITS_PER_LONG;
282 end_off = (first_page + nr_pages) & (BITS_PER_LONG-1);
284 if (curr_idx == end_idx) {
285 contiguous_bitmap[curr_idx] |=
286 ((1UL<<end_off)-1) & -(1UL<<start_off);
287 } else {
288 contiguous_bitmap[curr_idx] |= -(1UL<<start_off);
289 while ( ++curr_idx < end_idx )
290 contiguous_bitmap[curr_idx] = ~0UL;
291 contiguous_bitmap[curr_idx] |= (1UL<<end_off)-1;
292 }
293 }
295 static void contiguous_bitmap_clear(
296 unsigned long first_page, unsigned long nr_pages)
297 {
298 unsigned long start_off, end_off, curr_idx, end_idx;
300 curr_idx = first_page / BITS_PER_LONG;
301 start_off = first_page & (BITS_PER_LONG-1);
302 end_idx = (first_page + nr_pages) / BITS_PER_LONG;
303 end_off = (first_page + nr_pages) & (BITS_PER_LONG-1);
305 if (curr_idx == end_idx) {
306 contiguous_bitmap[curr_idx] &=
307 -(1UL<<end_off) | ((1UL<<start_off)-1);
308 } else {
309 contiguous_bitmap[curr_idx] &= (1UL<<start_off)-1;
310 while ( ++curr_idx != end_idx )
311 contiguous_bitmap[curr_idx] = 0;
312 contiguous_bitmap[curr_idx] &= -(1UL<<end_off);
313 }
314 }
316 /* Ensure multi-page extents are contiguous in machine memory. */
317 void xen_create_contiguous_region(unsigned long vstart, unsigned int order)
318 {
319 pgd_t *pgd;
320 pud_t *pud;
321 pmd_t *pmd;
322 pte_t *pte;
323 unsigned long mfn, i, flags;
324 struct xen_memory_reservation reservation = {
325 .extent_start = &mfn,
326 .nr_extents = 1,
327 .extent_order = 0,
328 .domid = DOMID_SELF
329 };
331 scrub_pages(vstart, 1 << order);
333 balloon_lock(flags);
335 /* 1. Zap current PTEs, giving away the underlying pages. */
336 for (i = 0; i < (1<<order); i++) {
337 pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
338 pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
339 pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
340 pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
341 mfn = pte_mfn(*pte);
342 BUG_ON(HYPERVISOR_update_va_mapping(
343 vstart + (i*PAGE_SIZE), __pte_ma(0), 0));
344 phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
345 INVALID_P2M_ENTRY;
346 BUG_ON(HYPERVISOR_memory_op(
347 XENMEM_decrease_reservation, &reservation) != 1);
348 }
350 /* 2. Get a new contiguous memory extent. */
351 reservation.extent_order = order;
352 reservation.address_bits = 31; /* aacraid limitation */
353 BUG_ON(HYPERVISOR_memory_op(
354 XENMEM_increase_reservation, &reservation) != 1);
356 /* 3. Map the new extent in place of old pages. */
357 for (i = 0; i < (1<<order); i++) {
358 BUG_ON(HYPERVISOR_update_va_mapping(
359 vstart + (i*PAGE_SIZE),
360 pfn_pte_ma(mfn+i, PAGE_KERNEL), 0));
361 xen_machphys_update(mfn+i, (__pa(vstart)>>PAGE_SHIFT)+i);
362 phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = mfn+i;
363 }
365 flush_tlb_all();
367 contiguous_bitmap_set(__pa(vstart) >> PAGE_SHIFT, 1UL << order);
369 balloon_unlock(flags);
370 }
372 void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
373 {
374 pgd_t *pgd;
375 pud_t *pud;
376 pmd_t *pmd;
377 pte_t *pte;
378 unsigned long mfn, i, flags;
379 struct xen_memory_reservation reservation = {
380 .extent_start = &mfn,
381 .nr_extents = 1,
382 .extent_order = 0,
383 .domid = DOMID_SELF
384 };
386 scrub_pages(vstart, 1 << order);
388 balloon_lock(flags);
390 contiguous_bitmap_clear(__pa(vstart) >> PAGE_SHIFT, 1UL << order);
392 /* 1. Zap current PTEs, giving away the underlying pages. */
393 for (i = 0; i < (1<<order); i++) {
394 pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
395 pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
396 pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
397 pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
398 mfn = pte_mfn(*pte);
399 BUG_ON(HYPERVISOR_update_va_mapping(
400 vstart + (i*PAGE_SIZE), __pte_ma(0), 0));
401 phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
402 INVALID_P2M_ENTRY;
403 BUG_ON(HYPERVISOR_memory_op(
404 XENMEM_decrease_reservation, &reservation) != 1);
405 }
407 /* 2. Map new pages in place of old pages. */
408 for (i = 0; i < (1<<order); i++) {
409 BUG_ON(HYPERVISOR_memory_op(
410 XENMEM_increase_reservation, &reservation) != 1);
411 BUG_ON(HYPERVISOR_update_va_mapping(
412 vstart + (i*PAGE_SIZE),
413 pfn_pte_ma(mfn, PAGE_KERNEL), 0));
414 xen_machphys_update(mfn, (__pa(vstart)>>PAGE_SHIFT)+i);
415 phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = mfn;
416 }
418 flush_tlb_all();
420 balloon_unlock(flags);
421 }
423 /*
424 * Local variables:
425 * c-file-style: "linux"
426 * indent-tabs-mode: t
427 * c-indent-level: 8
428 * c-basic-offset: 8
429 * tab-width: 8
430 * End:
431 */