ia64/xen-unstable

view xen/arch/x86/shadow.c @ 3813:4b4a77f35103

bitkeeper revision 1.1159.260.2 (420e0856crqXXEkQoCUddas8u5ksXA)

Rename check_pagetable() function to _check_pagetable to make it easier
to selectively enable it during debugging. There's still the same
check_pagetable macro, but now it's used both when SHADOW_DEBUG is
enabled and disabled.
author mafetter@fleming.research
date Sat Feb 12 13:44:54 2005 +0000 (2005-02-12)
parents acf0fa185f4a
children 58be428f51a8
line source
1 /* -*- Mode:C++; c-file-style:BSD; c-basic-offset:4; tab-width:4 -*- */
3 #include <xen/config.h>
4 #include <xen/types.h>
5 #include <xen/mm.h>
6 #include <asm/shadow.h>
7 #include <asm/domain_page.h>
8 #include <asm/page.h>
9 #include <xen/event.h>
10 #include <xen/trace.h>
12 /********
14 To use these shadow page tables, guests must not rely on the ACCESSED
15 and DIRTY bits on L2 pte's being accurate -- they will typically all be set.
17 I doubt this will break anything. (If guests want to use the va_update
18 mechanism they've signed up for this anyhow...)
20 There's a per-domain shadow table spin lock which works fine for SMP
21 hosts. We don't have to worry about interrupts as no shadow operations
22 happen in an interrupt context. It's probably not quite ready for SMP
23 guest operation as we have to worry about synchonisation between gpte
24 and spte updates. Its possible that this might only happen in a
25 hypercall context, in which case we'll probably at have a per-domain
26 hypercall lock anyhow (at least initially).
28 ********/
30 static inline void free_shadow_page(
31 struct mm_struct *m, struct pfn_info *page)
32 {
33 m->shadow_page_count--;
35 switch ( page->u.inuse.type_info & PGT_type_mask )
36 {
37 case PGT_l1_page_table:
38 perfc_decr(shadow_l1_pages);
39 break;
41 case PGT_l2_page_table:
42 perfc_decr(shadow_l2_pages);
43 break;
45 default:
46 printk("Free shadow weird page type pfn=%08x type=%08x\n",
47 frame_table-page, page->u.inuse.type_info);
48 break;
49 }
51 free_domheap_page(page);
52 }
54 static void free_shadow_state(struct mm_struct *m)
55 {
56 int i, free = 0;
57 struct shadow_status *x, *n;
59 /*
60 * WARNING! The shadow page table must not currently be in use!
61 * e.g., You are expected to have paused the domain and synchronized CR3.
62 */
64 shadow_audit(m, 1);
66 /* Free each hash chain in turn. */
67 for ( i = 0; i < shadow_ht_buckets; i++ )
68 {
69 /* Skip empty buckets. */
70 x = &m->shadow_ht[i];
71 if ( x->pfn == 0 )
72 continue;
74 /* Free the head page. */
75 free_shadow_page(
76 m, &frame_table[x->spfn_and_flags & PSH_pfn_mask]);
78 /* Reinitialise the head node. */
79 x->pfn = 0;
80 x->spfn_and_flags = 0;
81 n = x->next;
82 x->next = NULL;
84 free++;
86 /* Iterate over non-head nodes. */
87 for ( x = n; x != NULL; x = n )
88 {
89 /* Free the shadow page. */
90 free_shadow_page(
91 m, &frame_table[x->spfn_and_flags & PSH_pfn_mask]);
93 /* Re-initialise the chain node. */
94 x->pfn = 0;
95 x->spfn_and_flags = 0;
97 /* Add to the free list. */
98 n = x->next;
99 x->next = m->shadow_ht_free;
100 m->shadow_ht_free = x;
102 free++;
103 }
105 shadow_audit(m, 0);
106 }
108 SH_LOG("Free shadow table. Freed=%d.", free);
109 }
111 static inline int clear_shadow_page(
112 struct mm_struct *m, struct shadow_status *x)
113 {
114 unsigned long *p;
115 int restart = 0;
116 struct pfn_info *spage = &frame_table[x->spfn_and_flags & PSH_pfn_mask];
118 switch ( spage->u.inuse.type_info & PGT_type_mask )
119 {
120 /* We clear L2 pages by zeroing the guest entries. */
121 case PGT_l2_page_table:
122 p = map_domain_mem((spage - frame_table) << PAGE_SHIFT);
123 memset(p, 0, DOMAIN_ENTRIES_PER_L2_PAGETABLE * sizeof(*p));
124 unmap_domain_mem(p);
125 break;
127 /* We clear L1 pages by freeing them: no benefit from zeroing them. */
128 case PGT_l1_page_table:
129 delete_shadow_status(m, x->pfn);
130 free_shadow_page(m, spage);
131 restart = 1; /* We need to go to start of list again. */
132 break;
133 }
135 return restart;
136 }
138 static void clear_shadow_state(struct mm_struct *m)
139 {
140 int i;
141 struct shadow_status *x;
143 shadow_audit(m, 1);
145 for ( i = 0; i < shadow_ht_buckets; i++ )
146 {
147 retry:
148 /* Skip empty buckets. */
149 x = &m->shadow_ht[i];
150 if ( x->pfn == 0 )
151 continue;
153 if ( clear_shadow_page(m, x) )
154 goto retry;
156 for ( x = x->next; x != NULL; x = x->next )
157 if ( clear_shadow_page(m, x) )
158 goto retry;
160 shadow_audit(m, 0);
161 }
163 SH_VLOG("Scan shadow table. l1=%d l2=%d",
164 perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages));
165 }
168 void shadow_mode_init(void)
169 {
170 }
172 int shadow_mode_enable(struct domain *p, unsigned int mode)
173 {
174 struct mm_struct *m = &p->mm;
176 m->shadow_ht = xmalloc(
177 shadow_ht_buckets * sizeof(struct shadow_status));
178 if ( m->shadow_ht == NULL )
179 goto nomem;
180 memset(m->shadow_ht, 0, shadow_ht_buckets * sizeof(struct shadow_status));
182 if ( mode == SHM_logdirty )
183 {
184 m->shadow_dirty_bitmap_size = (p->max_pages + 63) & ~63;
185 m->shadow_dirty_bitmap =
186 xmalloc(m->shadow_dirty_bitmap_size/8);
187 if ( m->shadow_dirty_bitmap == NULL )
188 {
189 m->shadow_dirty_bitmap_size = 0;
190 goto nomem;
191 }
192 memset(m->shadow_dirty_bitmap, 0, m->shadow_dirty_bitmap_size/8);
193 }
195 m->shadow_mode = mode;
197 __shadow_mk_pagetable(m);
198 return 0;
200 nomem:
201 if ( m->shadow_ht != NULL )
202 xfree( m->shadow_ht );
203 m->shadow_ht = NULL;
204 return -ENOMEM;
205 }
207 void __shadow_mode_disable(struct domain *d)
208 {
209 struct mm_struct *m = &d->mm;
210 struct shadow_status *x, *n;
212 free_shadow_state(m);
213 m->shadow_mode = 0;
215 SH_VLOG("freed tables count=%d l1=%d l2=%d",
216 m->shadow_page_count, perfc_value(shadow_l1_pages),
217 perfc_value(shadow_l2_pages));
219 n = m->shadow_ht_extras;
220 while ( (x = n) != NULL )
221 {
222 m->shadow_extras_count--;
223 n = *((struct shadow_status **)(&x[shadow_ht_extra_size]));
224 xfree(x);
225 }
227 m->shadow_ht_extras = NULL;
228 ASSERT(m->shadow_extras_count == 0);
229 SH_LOG("freed extras, now %d", m->shadow_extras_count);
231 if ( m->shadow_dirty_bitmap != NULL )
232 {
233 xfree(m->shadow_dirty_bitmap);
234 m->shadow_dirty_bitmap = 0;
235 m->shadow_dirty_bitmap_size = 0;
236 }
238 xfree(m->shadow_ht);
239 m->shadow_ht = NULL;
240 }
242 static int shadow_mode_table_op(
243 struct domain *d, dom0_shadow_control_t *sc)
244 {
245 unsigned int op = sc->op;
246 struct mm_struct *m = &d->mm;
247 int i, rc = 0;
249 ASSERT(spin_is_locked(&m->shadow_lock));
251 SH_VLOG("shadow mode table op %08lx %08lx count %d",
252 pagetable_val(m->pagetable), pagetable_val(m->shadow_table),
253 m->shadow_page_count);
255 shadow_audit(m, 1);
257 switch ( op )
258 {
259 case DOM0_SHADOW_CONTROL_OP_FLUSH:
260 free_shadow_state(m);
262 m->shadow_fault_count = 0;
263 m->shadow_dirty_count = 0;
264 m->shadow_dirty_net_count = 0;
265 m->shadow_dirty_block_count = 0;
267 break;
269 case DOM0_SHADOW_CONTROL_OP_CLEAN:
270 clear_shadow_state(m);
272 sc->stats.fault_count = m->shadow_fault_count;
273 sc->stats.dirty_count = m->shadow_dirty_count;
274 sc->stats.dirty_net_count = m->shadow_dirty_net_count;
275 sc->stats.dirty_block_count = m->shadow_dirty_block_count;
277 m->shadow_fault_count = 0;
278 m->shadow_dirty_count = 0;
279 m->shadow_dirty_net_count = 0;
280 m->shadow_dirty_block_count = 0;
282 if ( (d->max_pages > sc->pages) ||
283 (sc->dirty_bitmap == NULL) ||
284 (m->shadow_dirty_bitmap == NULL) )
285 {
286 rc = -EINVAL;
287 break;
288 }
290 sc->pages = d->max_pages;
292 #define chunk (8*1024) /* Transfer and clear in 1kB chunks for L1 cache. */
293 for ( i = 0; i < d->max_pages; i += chunk )
294 {
295 int bytes = ((((d->max_pages - i) > chunk) ?
296 chunk : (d->max_pages - i)) + 7) / 8;
298 if (copy_to_user(
299 sc->dirty_bitmap + (i/(8*sizeof(unsigned long))),
300 m->shadow_dirty_bitmap +(i/(8*sizeof(unsigned long))),
301 bytes))
302 {
303 // copy_to_user can fail when copying to guest app memory.
304 // app should zero buffer after mallocing, and pin it
305 rc = -EINVAL;
306 memset(
307 m->shadow_dirty_bitmap + (i/(8*sizeof(unsigned long))),
308 0, (d->max_pages/8) - (i/(8*sizeof(unsigned long))));
309 break;
310 }
312 memset(
313 m->shadow_dirty_bitmap + (i/(8*sizeof(unsigned long))),
314 0, bytes);
315 }
317 break;
319 case DOM0_SHADOW_CONTROL_OP_PEEK:
320 sc->stats.fault_count = m->shadow_fault_count;
321 sc->stats.dirty_count = m->shadow_dirty_count;
322 sc->stats.dirty_net_count = m->shadow_dirty_net_count;
323 sc->stats.dirty_block_count = m->shadow_dirty_block_count;
325 if ( (d->max_pages > sc->pages) ||
326 (sc->dirty_bitmap == NULL) ||
327 (m->shadow_dirty_bitmap == NULL) )
328 {
329 rc = -EINVAL;
330 break;
331 }
333 sc->pages = d->max_pages;
334 if (copy_to_user(
335 sc->dirty_bitmap, m->shadow_dirty_bitmap, (d->max_pages+7)/8))
336 {
337 rc = -EINVAL;
338 break;
339 }
341 break;
343 default:
344 rc = -EINVAL;
345 break;
346 }
348 SH_VLOG("shadow mode table op : page count %d", m->shadow_page_count);
349 shadow_audit(m, 1);
350 __shadow_mk_pagetable(m);
351 return rc;
352 }
354 int shadow_mode_control(struct domain *d, dom0_shadow_control_t *sc)
355 {
356 unsigned int op = sc->op;
357 int rc = 0;
359 if ( unlikely(d == current) )
360 {
361 DPRINTK("Don't try to do a shadow op on yourself!\n");
362 return -EINVAL;
363 }
365 domain_pause(d);
366 synchronise_pagetables(~0UL);
368 shadow_lock(&d->mm);
370 switch ( op )
371 {
372 case DOM0_SHADOW_CONTROL_OP_OFF:
373 shadow_mode_disable(d);
374 break;
376 case DOM0_SHADOW_CONTROL_OP_ENABLE_TEST:
377 shadow_mode_disable(d);
378 rc = shadow_mode_enable(d, SHM_test);
379 break;
381 case DOM0_SHADOW_CONTROL_OP_ENABLE_LOGDIRTY:
382 shadow_mode_disable(d);
383 rc = shadow_mode_enable(d, SHM_logdirty);
384 break;
386 default:
387 rc = shadow_mode(d) ? shadow_mode_table_op(d, sc) : -EINVAL;
388 break;
389 }
391 shadow_unlock(&d->mm);
393 domain_unpause(d);
395 return rc;
396 }
398 static inline struct pfn_info *alloc_shadow_page(struct mm_struct *m)
399 {
400 struct pfn_info *page = alloc_domheap_page(NULL);
402 m->shadow_page_count++;
404 if ( unlikely(page == NULL) )
405 {
406 printk("Couldn't alloc shadow page! count=%d\n",
407 m->shadow_page_count);
408 SH_VLOG("Shadow tables l1=%d l2=%d",
409 perfc_value(shadow_l1_pages),
410 perfc_value(shadow_l2_pages));
411 BUG(); /* XXX FIXME: try a shadow flush to free up some memory. */
412 }
414 return page;
415 }
417 void unshadow_table(unsigned long gpfn, unsigned int type)
418 {
419 unsigned long spfn;
420 struct domain *d = frame_table[gpfn].u.inuse.domain;
422 SH_VLOG("unshadow_table type=%08x gpfn=%08lx", type, gpfn);
424 perfc_incrc(unshadow_table_count);
426 /*
427 * This function is the same for all p.t. pages. Even for multi-processor
428 * guests there won't be a race here as this CPU was the one that
429 * cmpxchg'ed the page to invalid.
430 */
431 spfn = __shadow_status(&d->mm, gpfn) & PSH_pfn_mask;
432 delete_shadow_status(&d->mm, gpfn);
433 free_shadow_page(&d->mm, &frame_table[spfn]);
434 }
436 unsigned long shadow_l2_table(
437 struct mm_struct *m, unsigned long gpfn)
438 {
439 struct pfn_info *spfn_info;
440 unsigned long spfn;
441 l2_pgentry_t *spl2e;
443 SH_VVLOG("shadow_l2_table( %08lx )", gpfn);
445 perfc_incrc(shadow_l2_table_count);
447 if ( (spfn_info = alloc_shadow_page(m)) == NULL )
448 BUG(); /* XXX Deal gracefully with failure. */
450 spfn_info->u.inuse.type_info = PGT_l2_page_table;
451 perfc_incr(shadow_l2_pages);
453 spfn = spfn_info - frame_table;
455 /* Mark pfn as being shadowed; update field to point at shadow. */
456 set_shadow_status(m, gpfn, spfn | PSH_shadowed);
458 spl2e = (l2_pgentry_t *)map_domain_mem(spfn << PAGE_SHIFT);
460 /*
461 * We could proactively fill in PDEs for pages that are already shadowed.
462 * However, we tried it and it didn't help performance. This is simpler.
463 */
464 memset(spl2e, 0, DOMAIN_ENTRIES_PER_L2_PAGETABLE * sizeof(l2_pgentry_t));
466 #ifdef __i386__
467 /* Install hypervisor and 2x linear p.t. mapings. */
468 memcpy(&spl2e[DOMAIN_ENTRIES_PER_L2_PAGETABLE],
469 &idle_pg_table[DOMAIN_ENTRIES_PER_L2_PAGETABLE],
470 HYPERVISOR_ENTRIES_PER_L2_PAGETABLE * sizeof(l2_pgentry_t));
471 spl2e[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] =
472 mk_l2_pgentry((gpfn << PAGE_SHIFT) | __PAGE_HYPERVISOR);
473 spl2e[SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] =
474 mk_l2_pgentry((spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR);
475 spl2e[PERDOMAIN_VIRT_START >> L2_PAGETABLE_SHIFT] =
476 mk_l2_pgentry(__pa(frame_table[gpfn].u.inuse.domain->mm.perdomain_pt) |
477 __PAGE_HYPERVISOR);
478 #endif
480 unmap_domain_mem(spl2e);
482 SH_VLOG("shadow_l2_table( %08lx -> %08lx)", gpfn, spfn);
483 return spfn;
484 }
486 static void shadow_map_l1_into_current_l2(unsigned long va)
487 {
488 struct mm_struct *m = &current->mm;
489 unsigned long *gpl1e, *spl1e, gpde, spde, gl1pfn, sl1pfn, sl1ss;
490 struct pfn_info *sl1pfn_info;
491 int i;
493 gpde = l2_pgentry_val(linear_l2_table[va >> L2_PAGETABLE_SHIFT]);
495 gl1pfn = gpde >> PAGE_SHIFT;
497 sl1ss = __shadow_status(m, gl1pfn);
498 if ( !(sl1ss & PSH_shadowed) )
499 {
500 /* This L1 is NOT already shadowed so we need to shadow it. */
501 SH_VVLOG("4a: l1 not shadowed ( %08lx )", sl1pfn);
503 sl1pfn_info = alloc_shadow_page(m);
504 sl1pfn_info->u.inuse.type_info = PGT_l1_page_table;
506 sl1pfn = sl1pfn_info - frame_table;
508 perfc_incrc(shadow_l1_table_count);
509 perfc_incr(shadow_l1_pages);
511 set_shadow_status(m, gl1pfn, PSH_shadowed | sl1pfn);
513 l2pde_general(m, &gpde, &spde, sl1pfn);
515 linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(gpde);
516 shadow_linear_l2_table[va>>L2_PAGETABLE_SHIFT] =
517 mk_l2_pgentry(spde);
519 gpl1e = (unsigned long *) &(linear_pg_table[
520 (va>>L1_PAGETABLE_SHIFT) & ~(ENTRIES_PER_L1_PAGETABLE-1)]);
522 spl1e = (unsigned long *) &(shadow_linear_pg_table[
523 (va>>L1_PAGETABLE_SHIFT) & ~(ENTRIES_PER_L1_PAGETABLE-1)]);
525 for ( i = 0; i < ENTRIES_PER_L1_PAGETABLE; i++ )
526 l1pte_propagate_from_guest(m, &gpl1e[i], &spl1e[i]);
527 }
528 else
529 {
530 /* This L1 is shadowed already, but the L2 entry is missing. */
531 SH_VVLOG("4b: was shadowed, l2 missing ( %08lx )", sl1pfn);
533 sl1pfn = sl1ss & PSH_pfn_mask;
534 l2pde_general(m, &gpde, &spde, sl1pfn);
536 linear_l2_table[va >> L2_PAGETABLE_SHIFT] = mk_l2_pgentry(gpde);
537 shadow_linear_l2_table[va >> L2_PAGETABLE_SHIFT] = mk_l2_pgentry(spde);
538 }
539 }
541 int shadow_fault(unsigned long va, long error_code)
542 {
543 unsigned long gpte, spte;
544 struct mm_struct *m = &current->mm;
546 SH_VVLOG("shadow_fault( va=%08lx, code=%ld )", va, error_code );
548 check_pagetable(m, current->mm.pagetable, "pre-sf");
550 /*
551 * STEP 1. A fast-reject set of checks with no locking.
552 */
554 if ( unlikely(__get_user(gpte, (unsigned long *)
555 &linear_pg_table[va >> PAGE_SHIFT])) )
556 {
557 SH_VVLOG("shadow_fault - EXIT: read gpte faulted" );
558 return 0;
559 }
561 if ( !(gpte & _PAGE_PRESENT) )
562 {
563 SH_VVLOG("shadow_fault - EXIT: gpte not present (%lx)",gpte );
564 return 0;
565 }
567 if ( (error_code & 2) && !(gpte & _PAGE_RW) )
568 {
569 /* Write fault on a read-only mapping. */
570 return 0;
571 }
573 /*
574 * STEP 2. Take the shadow lock and re-check the guest PTE.
575 */
577 shadow_lock(m);
579 if ( unlikely(__get_user(gpte, (unsigned long *)
580 &linear_pg_table[va >> PAGE_SHIFT])) )
581 {
582 SH_VVLOG("shadow_fault - EXIT: read gpte faulted" );
583 shadow_unlock(m);
584 return 0;
585 }
587 if ( unlikely(!(gpte & _PAGE_PRESENT)) )
588 {
589 SH_VVLOG("shadow_fault - EXIT: gpte not present (%lx)",gpte );
590 shadow_unlock(m);
591 return 0;
592 }
594 /* Write fault? */
595 if ( error_code & 2 )
596 {
597 if ( unlikely(!(gpte & _PAGE_RW)) )
598 {
599 /* Write fault on a read-only mapping. */
600 SH_VVLOG("shadow_fault - EXIT: wr fault on RO page (%lx)", gpte);
601 shadow_unlock(m);
602 return 0;
603 }
605 l1pte_write_fault(m, &gpte, &spte);
606 }
607 else
608 {
609 l1pte_read_fault(m, &gpte, &spte);
610 }
612 /*
613 * STEP 3. Write the modified shadow PTE and guest PTE back to the tables.
614 */
616 /* XXX Watch out for read-only L2 entries! (not used in Linux). */
617 if ( unlikely(__put_user(gpte, (unsigned long *)
618 &linear_pg_table[va >> PAGE_SHIFT])) )
619 domain_crash();
621 /*
622 * Update of shadow PTE can fail because the L1 p.t. is not shadowed,
623 * or because the shadow isn't linked into this shadow L2 p.t.
624 */
625 if ( unlikely(__put_user(spte, (unsigned long *)
626 &shadow_linear_pg_table[va >> PAGE_SHIFT])) )
627 {
628 SH_VVLOG("3: not shadowed/mapped gpte=%08lx spte=%08lx", gpte, spte);
629 shadow_map_l1_into_current_l2(va);
630 shadow_linear_pg_table[va >> PAGE_SHIFT] = mk_l1_pgentry(spte);
631 }
633 perfc_incrc(shadow_fixup_count);
634 m->shadow_fault_count++;
636 shadow_unlock(m);
638 check_pagetable(m, current->mm.pagetable, "post-sf");
639 return EXCRET_fault_fixed;
640 }
643 void shadow_l1_normal_pt_update(
644 unsigned long pa, unsigned long gpte,
645 unsigned long *prev_spfn_ptr,
646 l1_pgentry_t **prev_spl1e_ptr)
647 {
648 unsigned long spfn, spte, prev_spfn = *prev_spfn_ptr;
649 l1_pgentry_t *spl1e, *prev_spl1e = *prev_spl1e_ptr;
651 /* N.B. To get here, we know the l1 page *must* be shadowed. */
652 SH_VVLOG("shadow_l1_normal_pt_update pa=%08lx, gpte=%08lx, "
653 "prev_spfn=%08lx, prev_spl1e=%p\n",
654 pa, gpte, prev_spfn, prev_spl1e);
656 spfn = __shadow_status(&current->mm, pa >> PAGE_SHIFT) & PSH_pfn_mask;
658 if ( spfn == prev_spfn )
659 {
660 spl1e = prev_spl1e;
661 }
662 else
663 {
664 if ( prev_spl1e != NULL )
665 unmap_domain_mem( prev_spl1e );
666 spl1e = (l1_pgentry_t *)map_domain_mem(spfn << PAGE_SHIFT);
667 *prev_spfn_ptr = spfn;
668 *prev_spl1e_ptr = spl1e;
669 }
671 l1pte_propagate_from_guest(&current->mm, &gpte, &spte);
672 spl1e[(pa & ~PAGE_MASK) / sizeof(l1_pgentry_t)] = mk_l1_pgentry(spte);
673 }
675 void shadow_l2_normal_pt_update(unsigned long pa, unsigned long gpte)
676 {
677 unsigned long spfn, spte;
678 l2_pgentry_t *spl2e;
679 unsigned long s_sh;
681 /* N.B. To get here, we know the l2 page *must* be shadowed. */
682 SH_VVLOG("shadow_l2_normal_pt_update pa=%08lx, gpte=%08lx",pa,gpte);
684 spfn = __shadow_status(&current->mm, pa >> PAGE_SHIFT) & PSH_pfn_mask;
686 s_sh = (gpte & _PAGE_PRESENT) ?
687 __shadow_status(&current->mm, gpte >> PAGE_SHIFT) : 0;
689 /* XXXX Should mark guest pte as DIRTY and ACCESSED too! */
690 l2pde_general(&current->mm, &gpte, &spte, s_sh);
691 spl2e = (l2_pgentry_t *)map_domain_mem(spfn << PAGE_SHIFT);
692 spl2e[(pa & ~PAGE_MASK) / sizeof(l2_pgentry_t)] = mk_l2_pgentry(spte);
693 unmap_domain_mem(spl2e);
694 }
699 /************************************************************************/
700 /************************************************************************/
701 /************************************************************************/
703 #if SHADOW_DEBUG
705 static int sh_l2_present;
706 static int sh_l1_present;
707 char * sh_check_name;
709 #define FAIL(_f, _a...) \
710 do { \
711 printk("XXX %s-FAIL (%d,%d)" _f " g=%08lx s=%08lx\n", \
712 sh_check_name, level, i, ## _a , gpte, spte); \
713 BUG(); \
714 } while ( 0 )
716 static int check_pte(
717 struct mm_struct *m, unsigned long gpte, unsigned long spte,
718 int level, int i)
719 {
720 unsigned long mask, gpfn, spfn;
722 if ( (spte == 0) || (spte == 0xdeadface) || (spte == 0x00000E00) )
723 return 1; /* always safe */
725 if ( !(spte & _PAGE_PRESENT) )
726 FAIL("Non zero not present spte");
728 if ( level == 2 ) sh_l2_present++;
729 if ( level == 1 ) sh_l1_present++;
731 if ( !(gpte & _PAGE_PRESENT) )
732 FAIL("Guest not present yet shadow is");
734 mask = ~(_PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_RW|0xFFFFF000);
736 if ( (spte & mask) != (gpte & mask) )
737 FAIL("Corrupt?");
739 if ( (spte & _PAGE_DIRTY ) && !(gpte & _PAGE_DIRTY) )
740 FAIL("Dirty coherence");
742 if ( (spte & _PAGE_ACCESSED ) && !(gpte & _PAGE_ACCESSED) )
743 FAIL("Accessed coherence");
745 if ( (spte & _PAGE_RW ) && !(gpte & _PAGE_RW) )
746 FAIL("RW coherence");
748 if ( (spte & _PAGE_RW ) && !((gpte & _PAGE_RW) && (gpte & _PAGE_DIRTY)) )
749 FAIL("RW2 coherence");
751 spfn = spte >> PAGE_SHIFT;
752 gpfn = gpte >> PAGE_SHIFT;
754 if ( gpfn == spfn )
755 {
756 if ( level > 1 )
757 FAIL("Linear map ???"); /* XXX this will fail on BSD */
758 }
759 else
760 {
761 if ( level < 2 )
762 FAIL("Shadow in L1 entry?");
764 if ( __shadow_status(m, gpfn) != (PSH_shadowed | spfn) )
765 FAIL("spfn problem g.sf=%08lx", __shadow_status(m, gpfn));
766 }
768 return 1;
769 }
772 static int check_l1_table(
773 struct mm_struct *m, unsigned long va,
774 unsigned long g2, unsigned long s2)
775 {
776 int i;
777 unsigned long *gpl1e, *spl1e;
779 gpl1e = map_domain_mem(g2 << PAGE_SHIFT);
780 spl1e = map_domain_mem(s2 << PAGE_SHIFT);
782 for ( i = 0; i < ENTRIES_PER_L1_PAGETABLE; i++ )
783 check_pte(m, gpl1e[i], spl1e[i], 1, i);
785 unmap_domain_mem(spl1e);
786 unmap_domain_mem(gpl1e);
788 return 1;
789 }
791 #define FAILPT(_f, _a...) \
792 do { \
793 printk("XXX FAIL %s-PT" _f "\n", s, ## _a ); \
794 BUG(); \
795 } while ( 0 )
797 int _check_pagetable(struct mm_struct *m, pagetable_t pt, char *s)
798 {
799 unsigned long gptbase = pagetable_val(pt);
800 unsigned long gpfn, spfn;
801 int i;
802 l2_pgentry_t *gpl2e, *spl2e;
804 sh_check_name = s;
806 SH_VVLOG("%s-PT Audit", s);
808 sh_l2_present = sh_l1_present = 0;
810 gpfn = gptbase >> PAGE_SHIFT;
812 if ( !(__shadow_status(m, gpfn) & PSH_shadowed) )
813 {
814 printk("%s-PT %08lx not shadowed\n", s, gptbase);
815 if ( __shadow_status(m, gpfn) != 0 )
816 BUG();
817 return 0;
818 }
820 spfn = __shadow_status(m, gpfn) & PSH_pfn_mask;
822 if ( __shadow_status(m, gpfn) != (PSH_shadowed | spfn) )
823 FAILPT("ptbase shadow inconsistent1");
825 gpl2e = (l2_pgentry_t *) map_domain_mem( gpfn << PAGE_SHIFT );
826 spl2e = (l2_pgentry_t *) map_domain_mem( spfn << PAGE_SHIFT );
828 if ( memcmp(&spl2e[DOMAIN_ENTRIES_PER_L2_PAGETABLE],
829 &gpl2e[DOMAIN_ENTRIES_PER_L2_PAGETABLE],
830 ((SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT) -
831 DOMAIN_ENTRIES_PER_L2_PAGETABLE) * sizeof(l2_pgentry_t)) )
832 {
833 printk("gpfn=%08lx spfn=%08lx\n", gpfn, spfn);
834 for ( i = DOMAIN_ENTRIES_PER_L2_PAGETABLE;
835 i < (SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT);
836 i++ )
837 printk("+++ (%d) %08lx %08lx\n",i,
838 l2_pgentry_val(gpl2e[i]), l2_pgentry_val(spl2e[i]));
839 FAILPT("hypervisor entries inconsistent");
840 }
842 if ( (l2_pgentry_val(spl2e[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT]) !=
843 l2_pgentry_val(gpl2e[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT])) )
844 FAILPT("hypervisor linear map inconsistent");
846 if ( (l2_pgentry_val(spl2e[SH_LINEAR_PT_VIRT_START >>
847 L2_PAGETABLE_SHIFT]) !=
848 ((spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR)) )
849 FAILPT("hypervisor shadow linear map inconsistent %08lx %08lx",
850 l2_pgentry_val(spl2e[SH_LINEAR_PT_VIRT_START >>
851 L2_PAGETABLE_SHIFT]),
852 (spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR);
854 if ( (l2_pgentry_val(spl2e[PERDOMAIN_VIRT_START >> L2_PAGETABLE_SHIFT]) !=
855 ((__pa(frame_table[gpfn].u.inuse.domain->mm.perdomain_pt) |
856 __PAGE_HYPERVISOR))) )
857 FAILPT("hypervisor per-domain map inconsistent");
860 /* Check the whole L2. */
861 for ( i = 0; i < DOMAIN_ENTRIES_PER_L2_PAGETABLE; i++ )
862 check_pte(m, l2_pgentry_val(gpl2e[i]), l2_pgentry_val(spl2e[i]), 2, i);
864 /* Go back and recurse. */
865 for ( i = 0; i < DOMAIN_ENTRIES_PER_L2_PAGETABLE; i++ )
866 {
867 if ( l2_pgentry_val(spl2e[i]) != 0 )
868 check_l1_table(
869 m, i << L2_PAGETABLE_SHIFT,
870 l2_pgentry_val(gpl2e[i]) >> PAGE_SHIFT,
871 l2_pgentry_val(spl2e[i]) >> PAGE_SHIFT);
872 }
874 unmap_domain_mem(spl2e);
875 unmap_domain_mem(gpl2e);
877 SH_VVLOG("PT verified : l2_present = %d, l1_present = %d\n",
878 sh_l2_present, sh_l1_present);
880 return 1;
881 }
883 #endif