ia64/xen-unstable

view linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c @ 8401:8ebcbcda3d37

Add and use write_ldt_entry, as seen in later linux kernel versions.
Signed-off-by: Christian Limpach <Christian.Limpach@cl.cam.ac.uk>
author chris@ubuntu.eng.hq.xensource.com
date Thu Dec 15 16:17:13 2005 -0700 (2005-12-15)
parents 419b32f72179
children 17dc21008351
line source
1 /******************************************************************************
2 * mm/hypervisor.c
3 *
4 * Update page tables via the hypervisor.
5 *
6 * Copyright (c) 2002-2004, K A Fraser
7 *
8 * This file may be distributed separately from the Linux kernel, or
9 * incorporated into other software packages, subject to the following license:
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a copy
12 * of this source file (the "Software"), to deal in the Software without
13 * restriction, including without limitation the rights to use, copy, modify,
14 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
15 * and to permit persons to whom the Software is furnished to do so, subject to
16 * the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included in
19 * all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 * IN THE SOFTWARE.
28 */
30 #include <linux/config.h>
31 #include <linux/sched.h>
32 #include <linux/mm.h>
33 #include <linux/vmalloc.h>
34 #include <asm/page.h>
35 #include <asm/pgtable.h>
36 #include <asm/hypervisor.h>
37 #include <asm-xen/balloon.h>
38 #include <asm-xen/xen-public/memory.h>
39 #include <linux/module.h>
40 #include <linux/percpu.h>
41 #include <asm/tlbflush.h>
43 #ifdef CONFIG_X86_64
44 #define pmd_val_ma(v) (v).pmd
45 #else
46 #ifdef CONFIG_X86_PAE
47 # define pmd_val_ma(v) ((v).pmd)
48 # define pud_val_ma(v) ((v).pgd.pgd)
49 #else
50 # define pmd_val_ma(v) ((v).pud.pgd.pgd)
51 #endif
52 #endif
54 #ifndef CONFIG_XEN_SHADOW_MODE
55 void xen_l1_entry_update(pte_t *ptr, pte_t val)
56 {
57 mmu_update_t u;
58 u.ptr = virt_to_machine(ptr);
59 u.val = pte_val_ma(val);
60 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
61 }
63 void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
64 {
65 mmu_update_t u;
66 u.ptr = virt_to_machine(ptr);
67 u.val = pmd_val_ma(val);
68 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
69 }
71 #ifdef CONFIG_X86_PAE
72 void xen_l3_entry_update(pud_t *ptr, pud_t val)
73 {
74 mmu_update_t u;
75 u.ptr = virt_to_machine(ptr);
76 u.val = pud_val_ma(val);
77 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
78 }
79 #endif
81 #ifdef CONFIG_X86_64
82 void xen_l3_entry_update(pud_t *ptr, pud_t val)
83 {
84 mmu_update_t u;
85 u.ptr = virt_to_machine(ptr);
86 u.val = val.pud;
87 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
88 }
90 void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
91 {
92 mmu_update_t u;
93 u.ptr = virt_to_machine(ptr);
94 u.val = val.pgd;
95 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
96 }
97 #endif /* CONFIG_X86_64 */
98 #endif /* CONFIG_XEN_SHADOW_MODE */
100 void xen_machphys_update(unsigned long mfn, unsigned long pfn)
101 {
102 mmu_update_t u;
103 u.ptr = ((unsigned long long)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
104 u.val = pfn;
105 BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
106 }
108 void xen_pt_switch(unsigned long ptr)
109 {
110 struct mmuext_op op;
111 op.cmd = MMUEXT_NEW_BASEPTR;
112 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
113 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
114 }
116 void xen_new_user_pt(unsigned long ptr)
117 {
118 struct mmuext_op op;
119 op.cmd = MMUEXT_NEW_USER_BASEPTR;
120 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
121 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
122 }
124 void xen_tlb_flush(void)
125 {
126 struct mmuext_op op;
127 op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
128 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
129 }
131 void xen_invlpg(unsigned long ptr)
132 {
133 struct mmuext_op op;
134 op.cmd = MMUEXT_INVLPG_LOCAL;
135 op.arg1.linear_addr = ptr & PAGE_MASK;
136 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
137 }
139 #ifdef CONFIG_SMP
141 void xen_tlb_flush_all(void)
142 {
143 struct mmuext_op op;
144 op.cmd = MMUEXT_TLB_FLUSH_ALL;
145 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
146 }
148 void xen_tlb_flush_mask(cpumask_t *mask)
149 {
150 struct mmuext_op op;
151 if ( cpus_empty(*mask) )
152 return;
153 op.cmd = MMUEXT_TLB_FLUSH_MULTI;
154 op.arg2.vcpumask = mask->bits;
155 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
156 }
158 void xen_invlpg_all(unsigned long ptr)
159 {
160 struct mmuext_op op;
161 op.cmd = MMUEXT_INVLPG_ALL;
162 op.arg1.linear_addr = ptr & PAGE_MASK;
163 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
164 }
166 void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
167 {
168 struct mmuext_op op;
169 if ( cpus_empty(*mask) )
170 return;
171 op.cmd = MMUEXT_INVLPG_MULTI;
172 op.arg1.linear_addr = ptr & PAGE_MASK;
173 op.arg2.vcpumask = mask->bits;
174 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
175 }
177 #endif /* CONFIG_SMP */
179 #ifndef CONFIG_XEN_SHADOW_MODE
180 void xen_pgd_pin(unsigned long ptr)
181 {
182 struct mmuext_op op;
183 #ifdef CONFIG_X86_64
184 op.cmd = MMUEXT_PIN_L4_TABLE;
185 #elif defined(CONFIG_X86_PAE)
186 op.cmd = MMUEXT_PIN_L3_TABLE;
187 #else
188 op.cmd = MMUEXT_PIN_L2_TABLE;
189 #endif
190 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
191 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
192 }
194 void xen_pgd_unpin(unsigned long ptr)
195 {
196 struct mmuext_op op;
197 op.cmd = MMUEXT_UNPIN_TABLE;
198 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
199 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
200 }
202 void xen_pte_pin(unsigned long ptr)
203 {
204 struct mmuext_op op;
205 op.cmd = MMUEXT_PIN_L1_TABLE;
206 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
207 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
208 }
210 void xen_pte_unpin(unsigned long ptr)
211 {
212 struct mmuext_op op;
213 op.cmd = MMUEXT_UNPIN_TABLE;
214 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
215 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
216 }
218 #ifdef CONFIG_X86_64
219 void xen_pud_pin(unsigned long ptr)
220 {
221 struct mmuext_op op;
222 op.cmd = MMUEXT_PIN_L3_TABLE;
223 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
224 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
225 }
227 void xen_pud_unpin(unsigned long ptr)
228 {
229 struct mmuext_op op;
230 op.cmd = MMUEXT_UNPIN_TABLE;
231 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
232 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
233 }
235 void xen_pmd_pin(unsigned long ptr)
236 {
237 struct mmuext_op op;
238 op.cmd = MMUEXT_PIN_L2_TABLE;
239 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
240 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
241 }
243 void xen_pmd_unpin(unsigned long ptr)
244 {
245 struct mmuext_op op;
246 op.cmd = MMUEXT_UNPIN_TABLE;
247 op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
248 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
249 }
250 #endif /* CONFIG_X86_64 */
251 #endif /* CONFIG_XEN_SHADOW_MODE */
253 void xen_set_ldt(unsigned long ptr, unsigned long len)
254 {
255 struct mmuext_op op;
256 op.cmd = MMUEXT_SET_LDT;
257 op.arg1.linear_addr = ptr;
258 op.arg2.nr_ents = len;
259 BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
260 }
262 /*
263 * Bitmap is indexed by page number. If bit is set, the page is part of a
264 * xen_create_contiguous_region() area of memory.
265 */
266 unsigned long *contiguous_bitmap;
268 static void contiguous_bitmap_set(
269 unsigned long first_page, unsigned long nr_pages)
270 {
271 unsigned long start_off, end_off, curr_idx, end_idx;
273 curr_idx = first_page / BITS_PER_LONG;
274 start_off = first_page & (BITS_PER_LONG-1);
275 end_idx = (first_page + nr_pages) / BITS_PER_LONG;
276 end_off = (first_page + nr_pages) & (BITS_PER_LONG-1);
278 if (curr_idx == end_idx) {
279 contiguous_bitmap[curr_idx] |=
280 ((1UL<<end_off)-1) & -(1UL<<start_off);
281 } else {
282 contiguous_bitmap[curr_idx] |= -(1UL<<start_off);
283 while ( ++curr_idx < end_idx )
284 contiguous_bitmap[curr_idx] = ~0UL;
285 contiguous_bitmap[curr_idx] |= (1UL<<end_off)-1;
286 }
287 }
289 static void contiguous_bitmap_clear(
290 unsigned long first_page, unsigned long nr_pages)
291 {
292 unsigned long start_off, end_off, curr_idx, end_idx;
294 curr_idx = first_page / BITS_PER_LONG;
295 start_off = first_page & (BITS_PER_LONG-1);
296 end_idx = (first_page + nr_pages) / BITS_PER_LONG;
297 end_off = (first_page + nr_pages) & (BITS_PER_LONG-1);
299 if (curr_idx == end_idx) {
300 contiguous_bitmap[curr_idx] &=
301 -(1UL<<end_off) | ((1UL<<start_off)-1);
302 } else {
303 contiguous_bitmap[curr_idx] &= (1UL<<start_off)-1;
304 while ( ++curr_idx != end_idx )
305 contiguous_bitmap[curr_idx] = 0;
306 contiguous_bitmap[curr_idx] &= -(1UL<<end_off);
307 }
308 }
310 /* Ensure multi-page extents are contiguous in machine memory. */
311 int xen_create_contiguous_region(
312 unsigned long vstart, unsigned int order, unsigned int address_bits)
313 {
314 pgd_t *pgd;
315 pud_t *pud;
316 pmd_t *pmd;
317 pte_t *pte;
318 unsigned long mfn, i, flags;
319 struct xen_memory_reservation reservation = {
320 .extent_start = &mfn,
321 .nr_extents = 1,
322 .extent_order = 0,
323 .domid = DOMID_SELF
324 };
326 scrub_pages(vstart, 1 << order);
328 balloon_lock(flags);
330 /* 1. Zap current PTEs, giving away the underlying pages. */
331 for (i = 0; i < (1<<order); i++) {
332 pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
333 pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
334 pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
335 pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
336 mfn = pte_mfn(*pte);
337 BUG_ON(HYPERVISOR_update_va_mapping(
338 vstart + (i*PAGE_SIZE), __pte_ma(0), 0));
339 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i,
340 INVALID_P2M_ENTRY);
341 BUG_ON(HYPERVISOR_memory_op(
342 XENMEM_decrease_reservation, &reservation) != 1);
343 }
345 /* 2. Get a new contiguous memory extent. */
346 reservation.extent_order = order;
347 reservation.address_bits = address_bits;
348 if (HYPERVISOR_memory_op(XENMEM_increase_reservation,
349 &reservation) != 1)
350 goto fail;
352 /* 3. Map the new extent in place of old pages. */
353 for (i = 0; i < (1<<order); i++) {
354 BUG_ON(HYPERVISOR_update_va_mapping(
355 vstart + (i*PAGE_SIZE),
356 pfn_pte_ma(mfn+i, PAGE_KERNEL), 0));
357 xen_machphys_update(mfn+i, (__pa(vstart)>>PAGE_SHIFT)+i);
358 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i, mfn+i);
359 }
361 flush_tlb_all();
363 contiguous_bitmap_set(__pa(vstart) >> PAGE_SHIFT, 1UL << order);
365 balloon_unlock(flags);
367 return 0;
369 fail:
370 reservation.extent_order = 0;
371 reservation.address_bits = 0;
373 for (i = 0; i < (1<<order); i++) {
374 BUG_ON(HYPERVISOR_memory_op(
375 XENMEM_increase_reservation, &reservation) != 1);
376 BUG_ON(HYPERVISOR_update_va_mapping(
377 vstart + (i*PAGE_SIZE),
378 pfn_pte_ma(mfn, PAGE_KERNEL), 0));
379 xen_machphys_update(mfn, (__pa(vstart)>>PAGE_SHIFT)+i);
380 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i, mfn);
381 }
383 flush_tlb_all();
385 balloon_unlock(flags);
387 return -ENOMEM;
388 }
390 void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
391 {
392 pgd_t *pgd;
393 pud_t *pud;
394 pmd_t *pmd;
395 pte_t *pte;
396 unsigned long mfn, i, flags;
397 struct xen_memory_reservation reservation = {
398 .extent_start = &mfn,
399 .nr_extents = 1,
400 .extent_order = 0,
401 .domid = DOMID_SELF
402 };
404 scrub_pages(vstart, 1 << order);
406 balloon_lock(flags);
408 contiguous_bitmap_clear(__pa(vstart) >> PAGE_SHIFT, 1UL << order);
410 /* 1. Zap current PTEs, giving away the underlying pages. */
411 for (i = 0; i < (1<<order); i++) {
412 pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
413 pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
414 pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
415 pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
416 mfn = pte_mfn(*pte);
417 BUG_ON(HYPERVISOR_update_va_mapping(
418 vstart + (i*PAGE_SIZE), __pte_ma(0), 0));
419 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i,
420 INVALID_P2M_ENTRY);
421 BUG_ON(HYPERVISOR_memory_op(
422 XENMEM_decrease_reservation, &reservation) != 1);
423 }
425 /* 2. Map new pages in place of old pages. */
426 for (i = 0; i < (1<<order); i++) {
427 BUG_ON(HYPERVISOR_memory_op(
428 XENMEM_increase_reservation, &reservation) != 1);
429 BUG_ON(HYPERVISOR_update_va_mapping(
430 vstart + (i*PAGE_SIZE),
431 pfn_pte_ma(mfn, PAGE_KERNEL), 0));
432 xen_machphys_update(mfn, (__pa(vstart)>>PAGE_SHIFT)+i);
433 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i, mfn);
434 }
436 flush_tlb_all();
438 balloon_unlock(flags);
439 }
441 #ifdef __i386__
442 int write_ldt_entry(void *ldt, int entry, __u32 entry_a, __u32 entry_b)
443 {
444 __u32 *lp = (__u32 *)((char *)ldt + entry * 8);
445 maddr_t mach_lp = arbitrary_virt_to_machine(lp);
446 return HYPERVISOR_update_descriptor(
447 mach_lp, (u64)entry_a | ((u64)entry_b<<32));
448 }
449 #endif
451 /*
452 * Local variables:
453 * c-file-style: "linux"
454 * indent-tabs-mode: t
455 * c-indent-level: 8
456 * c-basic-offset: 8
457 * tab-width: 8
458 * End:
459 */