ia64/xen-unstable

view xen/include/asm-x86/shadow.h @ 4629:303fb016437c

bitkeeper revision 1.1363 (42680d0cgIEixfhEIj_iwmyZ73FnfA)

Fix assertion in delete_shadow_status to allow for removal of PGT_writable_pred's...

Signed-off-by: michael.fetterman@cl.cam.ac.uk
author maf46@burn.cl.cam.ac.uk
date Thu Apr 21 20:29:00 2005 +0000 (2005-04-21)
parents 744349042cd0
children 18a8f5216548
line source
1 /******************************************************************************
2 * include/asm-x86/shadow.h
3 *
4 * Copyright (c) 2005 Michael A Fetterman
5 * Based on an earlier implementation by Ian Pratt et al
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 */
22 #ifndef _XEN_SHADOW_H
23 #define _XEN_SHADOW_H
25 #include <xen/config.h>
26 #include <xen/types.h>
27 #include <xen/perfc.h>
28 #include <xen/sched.h>
29 #include <asm/processor.h>
30 #include <asm/domain_page.h>
31 #include <public/dom0_ops.h>
33 /* Shadow PT operation mode : shadow-mode variable in arch_domain. */
35 #define SHM_enable (1<<0) /* we're in one of the shadow modes */
36 #define SHM_log_dirty (1<<1) /* enable log dirty mode */
37 #define SHM_translate (1<<2) /* do p2m tranaltion on guest tables */
38 #define SHM_external (1<<3) /* external page table, not used by Xen */
40 #define shadow_mode_enabled(_d) ((_d)->arch.shadow_mode)
41 #define shadow_mode_log_dirty(_d) ((_d)->arch.shadow_mode & SHM_log_dirty)
42 #define shadow_mode_translate(_d) ((_d)->arch.shadow_mode & SHM_translate)
43 #define shadow_mode_external(_d) ((_d)->arch.shadow_mode & SHM_external)
45 #define shadow_linear_pg_table ((l1_pgentry_t *)SH_LINEAR_PT_VIRT_START)
46 #define __shadow_linear_l2_table ((l2_pgentry_t *)(SH_LINEAR_PT_VIRT_START + \
47 (SH_LINEAR_PT_VIRT_START >> (L2_PAGETABLE_SHIFT - L1_PAGETABLE_SHIFT))))
48 #define shadow_linear_l2_table(_ed) ((_ed)->arch.shadow_vtable)
50 // easy access to the hl2 table (for translated but not external modes only)
51 #define __linear_hl2_table ((l1_pgentry_t *)(LINEAR_PT_VIRT_START + \
52 (PERDOMAIN_VIRT_START >> (L2_PAGETABLE_SHIFT - L1_PAGETABLE_SHIFT))))
54 #define shadow_lock_init(_d) spin_lock_init(&(_d)->arch.shadow_lock)
55 #define shadow_lock(_d) do { ASSERT(!spin_is_locked(&(_d)->arch.shadow_lock)); spin_lock(&(_d)->arch.shadow_lock); } while (0)
56 #define shadow_unlock(_d) spin_unlock(&(_d)->arch.shadow_lock)
58 #define SHADOW_ENCODE_MIN_MAX(_min, _max) ((((L1_PAGETABLE_ENTRIES - 1) - (_max)) << 16) | (_min))
59 #define SHADOW_MIN(_encoded) ((_encoded) & ((1u<<16) - 1))
60 #define SHADOW_MAX(_encoded) ((L1_PAGETABLE_ENTRIES - 1) - ((_encoded) >> 16))
62 extern void shadow_mode_init(void);
63 extern int shadow_mode_control(struct domain *p, dom0_shadow_control_t *sc);
64 extern int shadow_fault(unsigned long va, struct xen_regs *regs);
65 extern int shadow_mode_enable(struct domain *p, unsigned int mode);
66 extern void shadow_invlpg(struct exec_domain *, unsigned long);
67 extern struct out_of_sync_entry *shadow_mark_mfn_out_of_sync(
68 struct exec_domain *ed, unsigned long gpfn, unsigned long mfn);
69 extern void free_monitor_pagetable(struct exec_domain *ed);
70 extern void __shadow_sync_all(struct domain *d);
71 extern int __shadow_out_of_sync(struct exec_domain *ed, unsigned long va);
72 extern int set_p2m_entry(
73 struct domain *d, unsigned long pfn, unsigned long mfn);
75 static inline unsigned long __shadow_status(
76 struct domain *d, unsigned long gpfn, unsigned long stype);
77 static inline void update_hl2e(struct exec_domain *ed, unsigned long va);
79 extern void vmx_shadow_clear_state(struct domain *);
81 static inline int page_is_page_table(struct pfn_info *page)
82 {
83 return page->count_info & PGC_page_table;
84 }
86 static inline int mfn_is_page_table(unsigned long mfn)
87 {
88 if ( !pfn_valid(mfn) )
89 return 0;
91 return frame_table[mfn].count_info & PGC_page_table;
92 }
94 static inline int page_out_of_sync(struct pfn_info *page)
95 {
96 return page->count_info & PGC_out_of_sync;
97 }
99 static inline int mfn_out_of_sync(unsigned long mfn)
100 {
101 if ( !pfn_valid(mfn) )
102 return 0;
104 return frame_table[mfn].count_info & PGC_out_of_sync;
105 }
108 /************************************************************************/
110 static void inline
111 __shadow_sync_mfn(struct domain *d, unsigned long mfn)
112 {
113 if ( d->arch.out_of_sync )
114 {
115 // XXX - could be smarter
116 //
117 __shadow_sync_all(d);
118 }
119 }
121 static void inline
122 __shadow_sync_va(struct exec_domain *ed, unsigned long va)
123 {
124 struct domain *d = ed->domain;
126 if ( d->arch.out_of_sync && __shadow_out_of_sync(ed, va) )
127 {
128 perfc_incrc(shadow_sync_va);
130 // XXX - could be smarter
131 //
132 __shadow_sync_all(ed->domain);
133 }
135 // Also make sure the HL2 is up-to-date for this address.
136 //
137 if ( unlikely(shadow_mode_translate(ed->domain)) )
138 update_hl2e(ed, va);
139 }
141 static void inline
142 shadow_sync_all(struct domain *d)
143 {
144 if ( unlikely(shadow_mode_enabled(d)) )
145 {
146 shadow_lock(d);
148 if ( d->arch.out_of_sync )
149 __shadow_sync_all(d);
151 ASSERT(d->arch.out_of_sync == NULL);
153 shadow_unlock(d);
154 }
155 }
157 // SMP BUG: This routine can't ever be used properly in an SMP context.
158 // It should be something like get_shadow_and_sync_va().
159 // This probably shouldn't exist.
160 //
161 static void inline
162 shadow_sync_va(struct exec_domain *ed, unsigned long gva)
163 {
164 struct domain *d = ed->domain;
165 if ( unlikely(shadow_mode_enabled(d)) )
166 {
167 shadow_lock(d);
168 __shadow_sync_va(ed, gva);
169 shadow_unlock(d);
170 }
171 }
173 extern void __shadow_mode_disable(struct domain *d);
174 static inline void shadow_mode_disable(struct domain *d)
175 {
176 if ( shadow_mode_enabled(d) )
177 __shadow_mode_disable(d);
178 }
180 extern void shadow_mode_destroy(struct domain *d);
182 /************************************************************************/
184 #define __mfn_to_gpfn(_d, mfn) \
185 ( (shadow_mode_translate(_d)) \
186 ? machine_to_phys_mapping[(mfn)] \
187 : (mfn) )
189 #define __gpfn_to_mfn(_d, gpfn) \
190 ( (shadow_mode_translate(_d)) \
191 ? ({ ASSERT(current->domain == (_d)); \
192 phys_to_machine_mapping(gpfn); }) \
193 : (gpfn) )
195 #define __gpfn_to_mfn_foreign(_d, gpfn) \
196 ( (shadow_mode_translate(_d)) \
197 ? gpfn_to_mfn_foreign(_d, gpfn) \
198 : (gpfn) )
200 extern unsigned long gpfn_to_mfn_foreign(
201 struct domain *d, unsigned long gpfn);
203 /************************************************************************/
205 struct shadow_status {
206 struct shadow_status *next; /* Pull-to-front list per hash bucket. */
207 unsigned long gpfn_and_flags; /* Guest pfn plus flags. */
208 unsigned long smfn; /* Shadow mfn. */
209 };
211 #define shadow_ht_extra_size 128
212 #define shadow_ht_buckets 256
214 struct out_of_sync_entry {
215 struct out_of_sync_entry *next;
216 unsigned long gpfn; /* why is this here? */
217 unsigned long gmfn;
218 unsigned long snapshot_mfn;
219 unsigned long writable_pl1e; /* NB: this is a machine address */
220 };
222 #define out_of_sync_extra_size 127
224 #define SHADOW_SNAPSHOT_ELSEWHERE (-1L)
226 /************************************************************************/
227 #define SHADOW_DEBUG 0
228 #define SHADOW_VERBOSE_DEBUG 0
229 #define SHADOW_VVERBOSE_DEBUG 0
230 #define SHADOW_VVVERBOSE_DEBUG 0
231 #define SHADOW_HASH_DEBUG 0
232 #define FULLSHADOW_DEBUG 0
234 #if SHADOW_DEBUG
235 extern int shadow_status_noswap;
236 #endif
238 #ifdef VERBOSE
239 #define SH_LOG(_f, _a...) \
240 printk("DOM%uP%u: SH_LOG(%d): " _f "\n", \
241 current->domain->id , current->processor, __LINE__ , ## _a )
242 #else
243 #define SH_LOG(_f, _a...) ((void)0)
244 #endif
246 #if SHADOW_VERBOSE_DEBUG
247 #define SH_VLOG(_f, _a...) \
248 printk("DOM%uP%u: SH_VLOG(%d): " _f "\n", \
249 current->domain->id, current->processor, __LINE__ , ## _a )
250 #else
251 #define SH_VLOG(_f, _a...) ((void)0)
252 #endif
254 #if SHADOW_VVERBOSE_DEBUG
255 #define SH_VVLOG(_f, _a...) \
256 printk("DOM%uP%u: SH_VVLOG(%d): " _f "\n", \
257 current->domain->id, current->processor, __LINE__ , ## _a )
258 #else
259 #define SH_VVLOG(_f, _a...) ((void)0)
260 #endif
262 #if SHADOW_VVVERBOSE_DEBUG
263 #define SH_VVVLOG(_f, _a...) \
264 printk("DOM%uP%u: SH_VVVLOG(%d): " _f "\n", \
265 current->domain->id, current->processor, __LINE__ , ## _a )
266 #else
267 #define SH_VVVLOG(_f, _a...) ((void)0)
268 #endif
270 #if FULLSHADOW_DEBUG
271 #define FSH_LOG(_f, _a...) \
272 printk("DOM%uP%u: FSH_LOG(%d): " _f "\n", \
273 current->domain->id, current->processor, __LINE__ , ## _a )
274 #else
275 #define FSH_LOG(_f, _a...) ((void)0)
276 #endif
279 /************************************************************************/
281 static inline int
282 shadow_get_page_from_l1e(l1_pgentry_t l1e, struct domain *d)
283 {
284 l1_pgentry_t nl1e;
285 int res;
286 unsigned long mfn;
287 struct domain *owner;
289 ASSERT(l1e_get_flags(l1e) & _PAGE_PRESENT);
291 nl1e = l1e;
292 l1e_remove_flags(&nl1e, _PAGE_GLOBAL);
293 res = get_page_from_l1e(nl1e, d);
295 if ( unlikely(!res) && IS_PRIV(d) && !shadow_mode_translate(d) &&
296 !(l1e_get_flags(l1e) & L1_DISALLOW_MASK) &&
297 (mfn = l1e_get_pfn(l1e)) &&
298 pfn_valid(mfn) &&
299 (owner = page_get_owner(pfn_to_page(l1e_get_pfn(l1e)))) &&
300 (d != owner) )
301 {
302 res = get_page_from_l1e(nl1e, owner);
303 printk("tried to map mfn %p from domain %d into shadow page tables "
304 "of domain %d; %s\n",
305 mfn, owner->id, d->id, res ? "success" : "failed");
306 }
308 if ( unlikely(!res) )
309 {
310 perfc_incrc(shadow_get_page_fail);
311 FSH_LOG("%s failed to get ref l1e=%p\n", __func__, l1e_get_value(l1e));
312 }
314 return res;
315 }
317 /************************************************************************/
319 static inline void
320 __shadow_get_l2e(
321 struct exec_domain *ed, unsigned long va, l2_pgentry_t *psl2e)
322 {
323 ASSERT(shadow_mode_enabled(ed->domain));
325 *psl2e = ed->arch.shadow_vtable[l2_table_offset(va)];
326 }
328 static inline void
329 __shadow_set_l2e(
330 struct exec_domain *ed, unsigned long va, l2_pgentry_t value)
331 {
332 ASSERT(shadow_mode_enabled(ed->domain));
334 ed->arch.shadow_vtable[l2_table_offset(va)] = value;
335 }
337 static inline void
338 __guest_get_l2e(
339 struct exec_domain *ed, unsigned long va, l2_pgentry_t *pl2e)
340 {
341 *pl2e = ed->arch.guest_vtable[l2_table_offset(va)];
342 }
344 static inline void
345 __guest_set_l2e(
346 struct exec_domain *ed, unsigned long va, l2_pgentry_t value)
347 {
348 ed->arch.guest_vtable[l2_table_offset(va)] = value;
350 if ( unlikely(shadow_mode_translate(ed->domain)) )
351 update_hl2e(ed, va);
352 }
354 static inline void
355 update_hl2e(struct exec_domain *ed, unsigned long va)
356 {
357 int index = l2_table_offset(va);
358 unsigned long mfn;
359 l2_pgentry_t gl2e = ed->arch.guest_vtable[index];
360 l1_pgentry_t old_hl2e, new_hl2e;
361 int need_flush = 0;
363 ASSERT(shadow_mode_translate(ed->domain));
365 old_hl2e = ed->arch.hl2_vtable[index];
367 if ( (l2e_get_flags(gl2e) & _PAGE_PRESENT) &&
368 VALID_MFN(mfn = phys_to_machine_mapping(l2e_get_pfn(gl2e)) ))
369 new_hl2e = l1e_create_pfn(mfn, __PAGE_HYPERVISOR);
370 else
371 new_hl2e = l1e_empty();
373 // only do the ref counting if something important changed.
374 //
375 if ( (l1e_has_changed(&old_hl2e, &new_hl2e, _PAGE_PRESENT)) )
376 {
377 if ( (l1e_get_flags(new_hl2e) & _PAGE_PRESENT) &&
378 !get_page(pfn_to_page(l1e_get_pfn(new_hl2e)), ed->domain) )
379 new_hl2e = l1e_empty();
380 if ( l1e_get_flags(old_hl2e) & _PAGE_PRESENT )
381 {
382 put_page(pfn_to_page(l1e_get_pfn(old_hl2e)));
383 need_flush = 1;
384 }
385 }
387 ed->arch.hl2_vtable[l2_table_offset(va)] = new_hl2e;
389 if ( need_flush )
390 {
391 perfc_incrc(update_hl2e_invlpg);
392 local_flush_tlb_one(&linear_pg_table[l1_linear_offset(va)]);
393 }
394 }
396 static inline void shadow_drop_references(
397 struct domain *d, struct pfn_info *page)
398 {
399 if ( likely(!shadow_mode_enabled(d)) ||
400 ((page->u.inuse.type_info & PGT_count_mask) == 0) )
401 return;
403 /* XXX This needs more thought... */
404 printk("%s: needing to call shadow_remove_all_access for mfn=%p\n",
405 __func__, page_to_pfn(page));
406 printk("Before: mfn=%p c=%p t=%p\n", page_to_pfn(page),
407 page->count_info, page->u.inuse.type_info);
409 shadow_lock(d);
410 shadow_remove_all_access(d, page_to_pfn(page));
411 shadow_unlock(d);
413 printk("After: mfn=%p c=%p t=%p\n", page_to_pfn(page),
414 page->count_info, page->u.inuse.type_info);
415 }
417 /* XXX Needs more thought. Neither pretty nor fast: a place holder. */
418 static inline void shadow_sync_and_drop_references(
419 struct domain *d, struct pfn_info *page)
420 {
421 if ( likely(!shadow_mode_enabled(d)) )
422 return;
424 shadow_lock(d);
426 if ( page_out_of_sync(page) )
427 __shadow_sync_mfn(d, page_to_pfn(page));
429 shadow_remove_all_access(d, page_to_pfn(page));
431 shadow_unlock(d);
432 }
434 /************************************************************************/
436 /*
437 * Add another shadow reference to smfn.
438 */
439 static inline int
440 get_shadow_ref(unsigned long smfn)
441 {
442 u32 x, nx;
444 ASSERT(pfn_valid(smfn));
446 x = frame_table[smfn].count_info;
447 nx = x + 1;
449 if ( unlikely(nx == 0) )
450 {
451 printk("get_shadow_ref overflow, gmfn=%p smfn=%p\n",
452 frame_table[smfn].u.inuse.type_info & PGT_mfn_mask, smfn);
453 BUG();
454 }
456 // Guarded by the shadow lock...
457 //
458 frame_table[smfn].count_info = nx;
460 return 1;
461 }
463 extern void free_shadow_page(unsigned long smfn);
465 /*
466 * Drop a shadow reference to smfn.
467 */
468 static inline void
469 put_shadow_ref(unsigned long smfn)
470 {
471 u32 x, nx;
473 ASSERT(pfn_valid(smfn));
475 x = frame_table[smfn].count_info;
476 nx = x - 1;
478 if ( unlikely(x == 0) )
479 {
480 printk("put_shadow_ref underflow, smfn=%p oc=%p t=%p\n",
481 smfn,
482 frame_table[smfn].count_info,
483 frame_table[smfn].u.inuse.type_info);
484 BUG();
485 }
487 // Guarded by the shadow lock...
488 //
489 frame_table[smfn].count_info = nx;
491 if ( unlikely(nx == 0) )
492 {
493 free_shadow_page(smfn);
494 }
495 }
497 static inline void
498 shadow_pin(unsigned long smfn)
499 {
500 ASSERT( !(frame_table[smfn].u.inuse.type_info & PGT_pinned) );
502 frame_table[smfn].u.inuse.type_info |= PGT_pinned;
503 if ( unlikely(!get_shadow_ref(smfn)) )
504 BUG();
505 }
507 static inline void
508 shadow_unpin(unsigned long smfn)
509 {
510 ASSERT( (frame_table[smfn].u.inuse.type_info & PGT_pinned) );
512 frame_table[smfn].u.inuse.type_info &= ~PGT_pinned;
513 put_shadow_ref(smfn);
514 }
517 /************************************************************************/
519 static inline int __mark_dirty(struct domain *d, unsigned int mfn)
520 {
521 unsigned long pfn;
522 int rc = 0;
524 ASSERT(spin_is_locked(&d->arch.shadow_lock));
525 ASSERT(d->arch.shadow_dirty_bitmap != NULL);
527 if ( !VALID_MFN(mfn) )
528 return rc;
530 pfn = __mfn_to_gpfn(d, mfn);
532 /*
533 * Values with the MSB set denote MFNs that aren't really part of the
534 * domain's pseudo-physical memory map (e.g., the shared info frame).
535 * Nothing to do here...
536 */
537 if ( unlikely(IS_INVALID_M2P_ENTRY(pfn)) )
538 return rc;
540 if ( likely(pfn < d->arch.shadow_dirty_bitmap_size) )
541 {
542 /* N.B. Can use non-atomic TAS because protected by shadow_lock. */
543 if ( !__test_and_set_bit(pfn, d->arch.shadow_dirty_bitmap) )
544 {
545 d->arch.shadow_dirty_count++;
546 rc = 1;
547 }
548 }
549 #ifndef NDEBUG
550 else if ( mfn < max_page )
551 {
552 SH_LOG("mark_dirty OOR! mfn=%x pfn=%lx max=%x (dom %p)",
553 mfn, pfn, d->arch.shadow_dirty_bitmap_size, d);
554 SH_LOG("dom=%p caf=%08x taf=%08x\n",
555 page_get_owner(&frame_table[mfn]),
556 frame_table[mfn].count_info,
557 frame_table[mfn].u.inuse.type_info );
558 }
559 #endif
561 return rc;
562 }
565 static inline int mark_dirty(struct domain *d, unsigned int mfn)
566 {
567 int rc;
568 shadow_lock(d);
569 rc = __mark_dirty(d, mfn);
570 shadow_unlock(d);
571 return rc;
572 }
575 /************************************************************************/
577 extern void shadow_mark_va_out_of_sync(
578 struct exec_domain *ed, unsigned long gpfn, unsigned long mfn,
579 unsigned long va);
581 static inline int l1pte_write_fault(
582 struct exec_domain *ed, l1_pgentry_t *gpte_p, l1_pgentry_t *spte_p,
583 unsigned long va)
584 {
585 struct domain *d = ed->domain;
586 l1_pgentry_t gpte = *gpte_p;
587 l1_pgentry_t spte;
588 unsigned long gpfn = l1e_get_pfn(gpte);
589 unsigned long gmfn = __gpfn_to_mfn(d, gpfn);
591 //printk("l1pte_write_fault gmfn=%p\n", gmfn);
593 if ( unlikely(!VALID_MFN(gmfn)) )
594 {
595 SH_LOG("l1pte_write_fault: invalid gpfn=%p", gpfn);
596 *spte_p = l1e_empty();
597 return 0;
598 }
600 ASSERT(l1e_get_flags(gpte) & _PAGE_RW);
601 l1e_add_flags(&gpte, _PAGE_DIRTY | _PAGE_ACCESSED);
602 spte = l1e_create_pfn(gmfn, l1e_get_flags(gpte) & ~_PAGE_GLOBAL);
604 SH_VVLOG("l1pte_write_fault: updating spte=0x%p gpte=0x%p",
605 l1e_get_value(spte), l1e_get_value(gpte));
607 if ( shadow_mode_log_dirty(d) )
608 __mark_dirty(d, gmfn);
610 if ( mfn_is_page_table(gmfn) )
611 shadow_mark_va_out_of_sync(ed, gpfn, gmfn, va);
613 *gpte_p = gpte;
614 *spte_p = spte;
616 return 1;
617 }
619 static inline int l1pte_read_fault(
620 struct domain *d, l1_pgentry_t *gpte_p, l1_pgentry_t *spte_p)
621 {
622 l1_pgentry_t gpte = *gpte_p;
623 l1_pgentry_t spte = *spte_p;
624 unsigned long pfn = l1e_get_pfn(gpte);
625 unsigned long mfn = __gpfn_to_mfn(d, pfn);
627 if ( unlikely(!VALID_MFN(mfn)) )
628 {
629 SH_LOG("l1pte_read_fault: invalid gpfn=%p", pfn);
630 *spte_p = l1e_empty();
631 return 0;
632 }
634 l1e_add_flags(&gpte, _PAGE_ACCESSED);
635 spte = l1e_create_pfn(mfn, l1e_get_flags(gpte) & ~_PAGE_GLOBAL);
637 if ( shadow_mode_log_dirty(d) || !(l1e_get_flags(gpte) & _PAGE_DIRTY) ||
638 mfn_is_page_table(mfn) )
639 {
640 l1e_remove_flags(&spte, _PAGE_RW);
641 }
643 SH_VVLOG("l1pte_read_fault: updating spte=0x%p gpte=0x%p",
644 l1e_get_value(spte), l1e_get_value(gpte));
645 *gpte_p = gpte;
646 *spte_p = spte;
648 return 1;
649 }
651 static inline void l1pte_propagate_from_guest(
652 struct domain *d, l1_pgentry_t gpte, l1_pgentry_t *spte_p)
653 {
654 unsigned long mfn;
655 l1_pgentry_t spte;
657 spte = l1e_empty();
659 if ( ((l1e_get_flags(gpte) & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
660 (_PAGE_PRESENT|_PAGE_ACCESSED)) &&
661 VALID_MFN(mfn = __gpfn_to_mfn(d, l1e_get_pfn(gpte))) )
662 {
663 spte = l1e_create_pfn(mfn, l1e_get_flags(gpte) & ~_PAGE_GLOBAL);
665 if ( shadow_mode_log_dirty(d) ||
666 !(l1e_get_flags(gpte) & _PAGE_DIRTY) ||
667 mfn_is_page_table(mfn) )
668 {
669 l1e_remove_flags(&spte, _PAGE_RW);
670 }
671 }
673 if ( l1e_get_value(spte) || l1e_get_value(gpte) )
674 SH_VVVLOG("%s: gpte=%p, new spte=%p",
675 __func__, l1e_get_value(gpte), l1e_get_value(spte));
677 *spte_p = spte;
678 }
680 static inline void hl2e_propagate_from_guest(
681 struct domain *d, l2_pgentry_t gpde, l1_pgentry_t *hl2e_p)
682 {
683 unsigned long pfn = l2e_get_pfn(gpde);
684 unsigned long mfn;
685 l1_pgentry_t hl2e;
687 hl2e = l1e_empty();
689 if ( l2e_get_flags(gpde) & _PAGE_PRESENT )
690 {
691 if ( unlikely((current->domain != d) && !shadow_mode_external(d)) )
692 {
693 // Can't use __gpfn_to_mfn() if we don't have one of this domain's
694 // page tables currently installed.
695 // This isn't common -- it only happens during shadow mode setup
696 // and mode changes.
697 //
698 mfn = gpfn_to_mfn_foreign(d, pfn);
699 }
700 else
701 mfn = __gpfn_to_mfn(d, pfn);
703 if ( VALID_MFN(mfn) && (mfn < max_page) )
704 hl2e = l1e_create_pfn(mfn, __PAGE_HYPERVISOR);
705 }
707 if ( l1e_get_value(hl2e) || l2e_get_value(gpde) )
708 SH_VVLOG("%s: gpde=%p hl2e=%p", __func__,
709 l2e_get_value(gpde), l1e_get_value(hl2e));
711 *hl2e_p = hl2e;
712 }
714 static inline void l2pde_general(
715 struct domain *d,
716 l2_pgentry_t *gpde_p,
717 l2_pgentry_t *spde_p,
718 unsigned long sl1mfn)
719 {
720 l2_pgentry_t gpde = *gpde_p;
721 l2_pgentry_t spde;
723 spde = l2e_empty();
724 if ( (l2e_get_flags(gpde) & _PAGE_PRESENT) && (sl1mfn != 0) )
725 {
726 spde = l2e_create_pfn(sl1mfn,
727 l2e_get_flags(gpde) | _PAGE_RW | _PAGE_ACCESSED);
728 l2e_add_flags(&gpde, _PAGE_ACCESSED); /* N.B. PDEs do not have a dirty bit. */
730 // XXX mafetter: Hmm...
731 // Shouldn't the dirty log be checked/updated here?
732 // Actually, it needs to be done in this function's callers.
733 //
734 *gpde_p = gpde;
735 }
737 if ( l2e_get_value(spde) || l2e_get_value(gpde) )
738 SH_VVLOG("%s: gpde=%p, new spde=%p", __func__,
739 l2e_get_value(gpde), l2e_get_value(spde));
741 *spde_p = spde;
742 }
744 static inline void l2pde_propagate_from_guest(
745 struct domain *d, l2_pgentry_t *gpde_p, l2_pgentry_t *spde_p)
746 {
747 l2_pgentry_t gpde = *gpde_p;
748 unsigned long sl1mfn = 0;
750 if ( l2e_get_flags(gpde) & _PAGE_PRESENT )
751 sl1mfn = __shadow_status(d, l2e_get_pfn(gpde), PGT_l1_shadow);
752 l2pde_general(d, gpde_p, spde_p, sl1mfn);
753 }
755 /************************************************************************/
757 // returns true if a tlb flush is needed
758 //
759 static int inline
760 validate_pte_change(
761 struct domain *d,
762 l1_pgentry_t new_pte,
763 l1_pgentry_t *shadow_pte_p)
764 {
765 l1_pgentry_t old_spte, new_spte;
767 perfc_incrc(validate_pte_calls);
769 #if 0
770 FSH_LOG("validate_pte(old=%p new=%p)", old_pte, new_pte);
771 #endif
773 old_spte = *shadow_pte_p;
774 l1pte_propagate_from_guest(d, new_pte, &new_spte);
776 // only do the ref counting if something important changed.
777 //
778 if ( ((l1e_get_value(old_spte) | l1e_get_value(new_spte)) & _PAGE_PRESENT ) &&
779 l1e_has_changed(&old_spte, &new_spte, _PAGE_RW | _PAGE_PRESENT) )
780 {
781 perfc_incrc(validate_pte_changes);
783 if ( (l1e_get_flags(new_spte) & _PAGE_PRESENT) &&
784 !shadow_get_page_from_l1e(new_spte, d) )
785 new_spte = l1e_empty();
786 if ( l1e_get_flags(old_spte) & _PAGE_PRESENT )
787 put_page_from_l1e(old_spte, d);
788 }
790 *shadow_pte_p = new_spte;
792 // paranoia rules!
793 return 1;
794 }
796 // returns true if a tlb flush is needed
797 //
798 static int inline
799 validate_hl2e_change(
800 struct domain *d,
801 l2_pgentry_t new_gpde,
802 l1_pgentry_t *shadow_hl2e_p)
803 {
804 l1_pgentry_t old_hl2e, new_hl2e;
806 perfc_incrc(validate_hl2e_calls);
808 old_hl2e = *shadow_hl2e_p;
809 hl2e_propagate_from_guest(d, new_gpde, &new_hl2e);
811 // Only do the ref counting if something important changed.
812 //
813 if ( ((l1e_get_flags(old_hl2e) | l1e_get_flags(new_hl2e)) & _PAGE_PRESENT) &&
814 l1e_has_changed(&old_hl2e, &new_hl2e, _PAGE_PRESENT) )
815 {
816 perfc_incrc(validate_hl2e_changes);
818 if ( (l1e_get_flags(new_hl2e) & _PAGE_PRESENT) &&
819 !get_page(pfn_to_page(l1e_get_pfn(new_hl2e)), d) )
820 new_hl2e = l1e_empty();
821 if ( l1e_get_flags(old_hl2e) & _PAGE_PRESENT )
822 put_page(pfn_to_page(l1e_get_pfn(old_hl2e)));
823 }
825 *shadow_hl2e_p = new_hl2e;
827 // paranoia rules!
828 return 1;
830 }
832 // returns true if a tlb flush is needed
833 //
834 static int inline
835 validate_pde_change(
836 struct domain *d,
837 l2_pgentry_t new_gpde,
838 l2_pgentry_t *shadow_pde_p)
839 {
840 l2_pgentry_t old_spde, new_spde;
842 perfc_incrc(validate_pde_calls);
844 old_spde = *shadow_pde_p;
845 l2pde_propagate_from_guest(d, &new_gpde, &new_spde);
847 // XXX Shouldn't we propagate the new_gpde to the guest?
848 // And then mark the guest's L2 page as dirty?
850 // Only do the ref counting if something important changed.
851 //
852 if ( ((l2e_get_value(old_spde) | l2e_get_value(new_spde)) & _PAGE_PRESENT) &&
853 l2e_has_changed(&old_spde, &new_spde, _PAGE_PRESENT) )
854 {
855 perfc_incrc(validate_pde_changes);
857 if ( (l2e_get_flags(new_spde) & _PAGE_PRESENT) &&
858 !get_shadow_ref(l2e_get_pfn(new_spde)) )
859 BUG();
860 if ( l2e_get_flags(old_spde) & _PAGE_PRESENT )
861 put_shadow_ref(l2e_get_pfn(old_spde));
862 }
864 *shadow_pde_p = new_spde;
866 // paranoia rules!
867 return 1;
868 }
870 /*********************************************************************/
872 #if SHADOW_HASH_DEBUG
874 static void shadow_audit(struct domain *d, int print)
875 {
876 int live = 0, free = 0, j = 0, abs;
877 struct shadow_status *a;
879 for ( j = 0; j < shadow_ht_buckets; j++ )
880 {
881 a = &d->arch.shadow_ht[j];
882 if ( a->gpfn_and_flags )
883 {
884 live++;
885 ASSERT(a->smfn);
886 }
887 else
888 ASSERT(!a->next);
890 a = a->next;
891 while ( a && (live < 9999) )
892 {
893 live++;
894 if ( (a->gpfn_and_flags == 0) || (a->smfn == 0) )
895 {
896 printk("XXX live=%d gpfn+flags=%p sp=%p next=%p\n",
897 live, a->gpfn_and_flags, a->smfn, a->next);
898 BUG();
899 }
900 ASSERT(a->smfn);
901 a = a->next;
902 }
903 ASSERT(live < 9999);
904 }
906 for ( a = d->arch.shadow_ht_free; a != NULL; a = a->next )
907 free++;
909 if ( print )
910 printk("Xlive=%d free=%d\n", live, free);
912 // BUG: this only works if there's only a single domain which is
913 // using shadow tables.
914 //
915 abs = (
916 perfc_value(shadow_l1_pages) +
917 perfc_value(shadow_l2_pages) +
918 perfc_value(hl2_table_pages) +
919 perfc_value(snapshot_pages) +
920 perfc_value(writable_pte_predictions)
921 ) - live;
922 #ifdef PERF_COUNTERS
923 if ( (abs < -1) || (abs > 1) )
924 {
925 printk("live=%d free=%d l1=%d l2=%d hl2=%d snapshot=%d writable_ptes=%d\n",
926 live, free,
927 perfc_value(shadow_l1_pages),
928 perfc_value(shadow_l2_pages),
929 perfc_value(hl2_table_pages),
930 perfc_value(snapshot_pages),
931 perfc_value(writable_pte_predictions));
932 BUG();
933 }
934 #endif
936 // XXX ought to add some code to audit the out-of-sync entries, too.
937 //
938 }
939 #else
940 #define shadow_audit(p, print) ((void)0)
941 #endif
944 static inline struct shadow_status *hash_bucket(
945 struct domain *d, unsigned int gpfn)
946 {
947 return &d->arch.shadow_ht[gpfn % shadow_ht_buckets];
948 }
951 /*
952 * N.B. This takes a guest pfn (i.e. a pfn in the guest's namespace,
953 * which, depending on full shadow mode, may or may not equal
954 * its mfn).
955 * It returns the shadow's mfn, or zero if it doesn't exist.
956 */
958 static inline unsigned long ___shadow_status(
959 struct domain *d, unsigned long gpfn, unsigned long stype)
960 {
961 struct shadow_status *p, *x, *head;
962 unsigned long key = gpfn | stype;
964 perfc_incrc(shadow_status_calls);
966 x = head = hash_bucket(d, gpfn);
967 p = NULL;
969 //SH_VVLOG("lookup gpfn=%08x type=%08x bucket=%p", gpfn, stype, x);
970 shadow_audit(d, 0);
972 do
973 {
974 ASSERT(x->gpfn_and_flags || ((x == head) && (x->next == NULL)));
976 if ( x->gpfn_and_flags == key )
977 {
978 #if SHADOW_DEBUG
979 if ( unlikely(shadow_status_noswap) )
980 return x->smfn;
981 #endif
982 /* Pull-to-front if 'x' isn't already the head item. */
983 if ( unlikely(x != head) )
984 {
985 /* Delete 'x' from list and reinsert immediately after head. */
986 p->next = x->next;
987 x->next = head->next;
988 head->next = x;
990 /* Swap 'x' contents with head contents. */
991 SWAP(head->gpfn_and_flags, x->gpfn_and_flags);
992 SWAP(head->smfn, x->smfn);
993 }
994 else
995 {
996 perfc_incrc(shadow_status_hit_head);
997 }
999 //SH_VVLOG("lookup gpfn=%p => status=%p", key, head->smfn);
1000 return head->smfn;
1003 p = x;
1004 x = x->next;
1006 while ( x != NULL );
1008 //SH_VVLOG("lookup gpfn=%p => status=0", key);
1009 perfc_incrc(shadow_status_miss);
1010 return 0;
1013 static inline unsigned long __shadow_status(
1014 struct domain *d, unsigned long gpfn, unsigned long stype)
1016 unsigned long gmfn = ((current->domain == d)
1017 ? __gpfn_to_mfn(d, gpfn)
1018 : INVALID_MFN);
1020 ASSERT(spin_is_locked(&d->arch.shadow_lock));
1021 ASSERT(gpfn == (gpfn & PGT_mfn_mask));
1022 ASSERT(stype && !(stype & ~PGT_type_mask));
1024 if ( VALID_MFN(gmfn) && (gmfn < max_page) &&
1025 (stype != PGT_writable_pred) &&
1026 ((stype == PGT_snapshot)
1027 ? !mfn_out_of_sync(gmfn)
1028 : !mfn_is_page_table(gmfn)) )
1030 perfc_incrc(shadow_status_shortcut);
1031 #ifndef NDEBUG
1032 ASSERT(___shadow_status(d, gpfn, stype) == 0);
1034 // Undo the affects of the above ASSERT on ___shadow_status()'s perf
1035 // counters.
1036 //
1037 perfc_decrc(shadow_status_calls);
1038 perfc_decrc(shadow_status_miss);
1039 #endif
1040 return 0;
1043 return ___shadow_status(d, gpfn, stype);
1046 /*
1047 * Not clear if pull-to-front is worth while for this or not,
1048 * as it generally needs to scan the entire bucket anyway.
1049 * Much simpler without.
1051 * Either returns PGT_none, or PGT_l{1,2,3,4}_page_table.
1052 */
1053 static inline unsigned long
1054 shadow_max_pgtable_type(struct domain *d, unsigned long gpfn,
1055 unsigned long *smfn)
1057 struct shadow_status *x;
1058 unsigned long pttype = PGT_none, type;
1060 ASSERT(spin_is_locked(&d->arch.shadow_lock));
1061 ASSERT(gpfn == (gpfn & PGT_mfn_mask));
1063 perfc_incrc(shadow_max_type);
1065 x = hash_bucket(d, gpfn);
1067 while ( x && x->gpfn_and_flags )
1069 if ( (x->gpfn_and_flags & PGT_mfn_mask) == gpfn )
1071 type = x->gpfn_and_flags & PGT_type_mask;
1073 switch ( type )
1075 case PGT_hl2_shadow:
1076 // Treat an HL2 as if it's an L1
1077 //
1078 type = PGT_l1_shadow;
1079 break;
1080 case PGT_snapshot:
1081 case PGT_writable_pred:
1082 // Ignore snapshots -- they don't in and of themselves constitute
1083 // treating a page as a page table
1084 //
1085 goto next;
1086 case PGT_base_page_table:
1087 // Early exit if we found the max possible value
1088 //
1089 return type;
1090 default:
1091 break;
1094 if ( type > pttype )
1096 pttype = type;
1097 if ( smfn )
1098 *smfn = x->smfn;
1101 next:
1102 x = x->next;
1105 return pttype;
1108 /*
1109 * N.B. We can make this locking more fine grained (e.g., per shadow page) if
1110 * it ever becomes a problem, but since we need a spin lock on the hash table
1111 * anyway it's probably not worth being too clever.
1112 */
1113 static inline unsigned long get_shadow_status(
1114 struct domain *d, unsigned long gpfn, unsigned long stype)
1116 unsigned long res;
1118 ASSERT(shadow_mode_enabled(d));
1120 /*
1121 * If we get here we know that some sort of update has happened to the
1122 * underlying page table page: either a PTE has been updated, or the page
1123 * has changed type. If we're in log dirty mode, we should set the
1124 * appropriate bit in the dirty bitmap.
1125 * N.B. The VA update path doesn't use this and is handled independently.
1127 * XXX need to think this through for vmx guests, but probably OK
1128 */
1130 shadow_lock(d);
1132 if ( shadow_mode_log_dirty(d) )
1133 __mark_dirty(d, __gpfn_to_mfn(d, gpfn));
1135 if ( !(res = __shadow_status(d, gpfn, stype)) )
1136 shadow_unlock(d);
1138 return res;
1142 static inline void put_shadow_status(struct domain *d)
1144 shadow_unlock(d);
1148 static inline void delete_shadow_status(
1149 struct domain *d, unsigned long gpfn, unsigned long gmfn, unsigned int stype)
1151 struct shadow_status *p, *x, *n, *head;
1152 unsigned long key = gpfn | stype;
1154 ASSERT(spin_is_locked(&d->arch.shadow_lock));
1155 ASSERT(!(gpfn & ~PGT_mfn_mask));
1156 ASSERT(stype && !(stype & ~PGT_type_mask));
1158 head = hash_bucket(d, gpfn);
1160 SH_VLOG("delete gpfn=%p t=%p bucket=%p", gpfn, stype, head);
1161 shadow_audit(d, 0);
1163 /* Match on head item? */
1164 if ( head->gpfn_and_flags == key )
1166 if ( (n = head->next) != NULL )
1168 /* Overwrite head with contents of following node. */
1169 head->gpfn_and_flags = n->gpfn_and_flags;
1170 head->smfn = n->smfn;
1172 /* Delete following node. */
1173 head->next = n->next;
1175 /* Add deleted node to the free list. */
1176 n->gpfn_and_flags = 0;
1177 n->smfn = 0;
1178 n->next = d->arch.shadow_ht_free;
1179 d->arch.shadow_ht_free = n;
1181 else
1183 /* This bucket is now empty. Initialise the head node. */
1184 head->gpfn_and_flags = 0;
1185 head->smfn = 0;
1188 goto found;
1191 p = head;
1192 x = head->next;
1194 do
1196 if ( x->gpfn_and_flags == key )
1198 /* Delete matching node. */
1199 p->next = x->next;
1201 /* Add deleted node to the free list. */
1202 x->gpfn_and_flags = 0;
1203 x->smfn = 0;
1204 x->next = d->arch.shadow_ht_free;
1205 d->arch.shadow_ht_free = x;
1207 goto found;
1210 p = x;
1211 x = x->next;
1213 while ( x != NULL );
1215 /* If we got here, it wasn't in the list! */
1216 BUG();
1218 found:
1219 // release ref to page
1220 if ( stype != PGT_writable_pred )
1221 put_page(pfn_to_page(gmfn));
1223 shadow_audit(d, 0);
1226 static inline void set_shadow_status(
1227 struct domain *d, unsigned long gpfn, unsigned long gmfn,
1228 unsigned long smfn, unsigned long stype)
1230 struct shadow_status *x, *head, *extra;
1231 int i;
1232 unsigned long key = gpfn | stype;
1234 SH_VVLOG("set gpfn=%p gmfn=%p smfn=%p t=%p", gpfn, gmfn, smfn, stype);
1236 ASSERT(spin_is_locked(&d->arch.shadow_lock));
1238 ASSERT(shadow_mode_translate(d) || gpfn);
1239 ASSERT(!(gpfn & ~PGT_mfn_mask));
1241 // XXX - need to be more graceful.
1242 ASSERT(VALID_MFN(gmfn));
1244 ASSERT(stype && !(stype & ~PGT_type_mask));
1246 x = head = hash_bucket(d, gpfn);
1248 SH_VLOG("set gpfn=%p smfn=%p t=%p bucket=%p(%p)",
1249 gpfn, smfn, stype, x, x->next);
1250 shadow_audit(d, 0);
1252 // grab a reference to the guest page to represent the entry in the shadow
1253 // hash table
1254 //
1255 // XXX - Should PGT_writable_pred grab a page ref?
1256 // - Who/how are these hash table entry refs flushed if/when a page
1257 // is given away by the domain?
1258 //
1259 if ( stype != PGT_writable_pred )
1260 get_page(pfn_to_page(gmfn), d);
1262 /*
1263 * STEP 1. If page is already in the table, update it in place.
1264 */
1265 do
1267 if ( unlikely(x->gpfn_and_flags == key) )
1269 if ( stype != PGT_writable_pred )
1270 BUG(); // we should never replace entries into the hash table
1271 x->smfn = smfn;
1272 put_page(pfn_to_page(gmfn)); // already had a ref...
1273 goto done;
1276 x = x->next;
1278 while ( x != NULL );
1280 /*
1281 * STEP 2. The page must be inserted into the table.
1282 */
1284 /* If the bucket is empty then insert the new page as the head item. */
1285 if ( head->gpfn_and_flags == 0 )
1287 head->gpfn_and_flags = key;
1288 head->smfn = smfn;
1289 ASSERT(head->next == NULL);
1290 goto done;
1293 /* We need to allocate a new node. Ensure the quicklist is non-empty. */
1294 if ( unlikely(d->arch.shadow_ht_free == NULL) )
1296 SH_LOG("Allocate more shadow hashtable blocks.");
1298 extra = xmalloc_bytes(
1299 sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
1301 /* XXX Should be more graceful here. */
1302 if ( extra == NULL )
1303 BUG();
1305 memset(extra, 0, sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
1307 /* Record the allocation block so it can be correctly freed later. */
1308 d->arch.shadow_extras_count++;
1309 *((struct shadow_status **)&extra[shadow_ht_extra_size]) =
1310 d->arch.shadow_ht_extras;
1311 d->arch.shadow_ht_extras = &extra[0];
1313 /* Thread a free chain through the newly-allocated nodes. */
1314 for ( i = 0; i < (shadow_ht_extra_size - 1); i++ )
1315 extra[i].next = &extra[i+1];
1316 extra[i].next = NULL;
1318 /* Add the new nodes to the free list. */
1319 d->arch.shadow_ht_free = &extra[0];
1322 /* Allocate a new node from the quicklist. */
1323 x = d->arch.shadow_ht_free;
1324 d->arch.shadow_ht_free = x->next;
1326 /* Initialise the new node and insert directly after the head item. */
1327 x->gpfn_and_flags = key;
1328 x->smfn = smfn;
1329 x->next = head->next;
1330 head->next = x;
1332 done:
1333 shadow_audit(d, 0);
1335 if ( stype <= PGT_l4_shadow )
1337 // add to front of list of pages to check when removing write
1338 // permissions for a page...
1339 //
1343 /************************************************************************/
1345 void static inline
1346 shadow_update_min_max(unsigned long smfn, int index)
1348 struct pfn_info *sl1page = pfn_to_page(smfn);
1349 u32 min_max = sl1page->tlbflush_timestamp;
1350 int min = SHADOW_MIN(min_max);
1351 int max = SHADOW_MAX(min_max);
1352 int update = 0;
1354 if ( index < min )
1356 min = index;
1357 update = 1;
1359 if ( index > max )
1361 max = index;
1362 update = 1;
1364 if ( update )
1365 sl1page->tlbflush_timestamp = SHADOW_ENCODE_MIN_MAX(min, max);
1368 extern void shadow_map_l1_into_current_l2(unsigned long va);
1370 void static inline
1371 shadow_set_l1e(unsigned long va, l1_pgentry_t new_spte, int create_l1_shadow)
1373 struct exec_domain *ed = current;
1374 struct domain *d = ed->domain;
1375 l2_pgentry_t sl2e;
1376 l1_pgentry_t old_spte;
1378 #if 0
1379 printk("shadow_set_l1e(va=%p, new_spte=%p, create=%d)\n",
1380 va, l1e_get_value(new_spte), create_l1_shadow);
1381 #endif
1383 __shadow_get_l2e(ed, va, &sl2e);
1384 if ( !(l2e_get_flags(sl2e) & _PAGE_PRESENT) )
1386 /*
1387 * Either the L1 is not shadowed, or the shadow isn't linked into
1388 * the current shadow L2.
1389 */
1390 if ( create_l1_shadow )
1392 perfc_incrc(shadow_set_l1e_force_map);
1393 shadow_map_l1_into_current_l2(va);
1395 else /* check to see if it exists; if so, link it in */
1397 l2_pgentry_t gpde = linear_l2_table(ed)[l2_table_offset(va)];
1398 unsigned long gl1pfn = l2e_get_pfn(gpde);
1399 unsigned long sl1mfn = __shadow_status(d, gl1pfn, PGT_l1_shadow);
1401 ASSERT( l2e_get_flags(gpde) & _PAGE_PRESENT );
1403 if ( sl1mfn )
1405 perfc_incrc(shadow_set_l1e_unlinked);
1406 if ( !get_shadow_ref(sl1mfn) )
1407 BUG();
1408 l2pde_general(d, &gpde, &sl2e, sl1mfn);
1409 __guest_set_l2e(ed, va, gpde);
1410 __shadow_set_l2e(ed, va, sl2e);
1412 else
1414 // no shadow exists, so there's nothing to do.
1415 perfc_incrc(shadow_set_l1e_fail);
1416 return;
1421 old_spte = shadow_linear_pg_table[l1_linear_offset(va)];
1423 // only do the ref counting if something important changed.
1424 //
1425 if ( l1e_has_changed(&old_spte, &new_spte, _PAGE_RW | _PAGE_PRESENT) )
1427 if ( (l1e_get_flags(new_spte) & _PAGE_PRESENT) &&
1428 !shadow_get_page_from_l1e(new_spte, d) )
1429 new_spte = l1e_empty();
1430 if ( l1e_get_flags(old_spte) & _PAGE_PRESENT )
1431 put_page_from_l1e(old_spte, d);
1434 shadow_linear_pg_table[l1_linear_offset(va)] = new_spte;
1436 shadow_update_min_max(l2e_get_pfn(sl2e), l1_table_offset(va));
1439 /************************************************************************/
1441 static inline l1_pgentry_t gva_to_gpte(unsigned long gva)
1443 l2_pgentry_t gpde;
1444 l1_pgentry_t gpte;
1445 struct exec_domain *ed = current;
1447 ASSERT( shadow_mode_translate(current->domain) );
1449 __guest_get_l2e(ed, gva, &gpde);
1450 if ( unlikely(!(l2e_get_flags(gpde) & _PAGE_PRESENT)) )
1451 return l1e_empty();;
1453 // This is actually overkill - we only need to make sure the hl2
1454 // is in-sync.
1455 //
1456 shadow_sync_va(ed, gva);
1458 if ( unlikely(__copy_from_user(&gpte,
1459 &linear_pg_table[gva >> PAGE_SHIFT],
1460 sizeof(gpte))) )
1462 FSH_LOG("gva_to_gpte got a fault on gva=%p", gva);
1463 return l1e_empty();
1466 return gpte;
1469 static inline unsigned long gva_to_gpa(unsigned long gva)
1471 l1_pgentry_t gpte;
1473 gpte = gva_to_gpte(gva);
1474 if ( !(l1e_get_flags(gpte) & _PAGE_PRESENT) )
1475 return 0;
1477 return l1e_get_phys(gpte) + (gva & ~PAGE_MASK);
1480 /************************************************************************/
1482 extern void __update_pagetables(struct exec_domain *ed);
1483 static inline void update_pagetables(struct exec_domain *ed)
1485 struct domain *d = ed->domain;
1486 int paging_enabled;
1488 #ifdef CONFIG_VMX
1489 if ( VMX_DOMAIN(ed) )
1490 paging_enabled =
1491 test_bit(VMX_CPU_STATE_PG_ENABLED, &ed->arch.arch_vmx.cpu_state);
1492 else
1493 #endif
1494 // HACK ALERT: there's currently no easy way to figure out if a domU
1495 // has set its arch.guest_table to zero, vs not yet initialized it.
1496 //
1497 paging_enabled = !!pagetable_val(ed->arch.guest_table);
1499 /*
1500 * We don't call __update_pagetables() when vmx guest paging is
1501 * disabled as we want the linear_pg_table to be inaccessible so that
1502 * we bail out early of shadow_fault() if the vmx guest tries illegal
1503 * accesses while it thinks paging is turned off.
1504 */
1505 if ( unlikely(shadow_mode_enabled(d)) && paging_enabled )
1507 shadow_lock(d);
1508 __update_pagetables(ed);
1509 shadow_unlock(d);
1512 if ( likely(!shadow_mode_external(d)) )
1514 #ifdef __x86_64__
1515 if ( !(ed->arch.flags & TF_kernel_mode) )
1516 ed->arch.monitor_table = ed->arch.guest_table_user;
1517 else
1518 #endif
1519 if ( shadow_mode_enabled(d) )
1520 ed->arch.monitor_table = ed->arch.shadow_table;
1521 else
1522 ed->arch.monitor_table = ed->arch.guest_table;
1526 #if SHADOW_DEBUG
1527 extern int _check_pagetable(struct exec_domain *ed, char *s);
1528 extern int _check_all_pagetables(struct exec_domain *ed, char *s);
1530 #define check_pagetable(_ed, _s) _check_pagetable(_ed, _s)
1531 //#define check_pagetable(_ed, _s) _check_all_pagetables(_ed, _s)
1533 #else
1534 #define check_pagetable(_ed, _s) ((void)0)
1535 #endif
1537 #endif /* XEN_SHADOW_H */
1539 /*
1540 * Local variables:
1541 * mode: C
1542 * c-set-style: "BSD"
1543 * c-basic-offset: 4
1544 * tab-width: 4
1545 * indent-tabs-mode: nil
1546 * End:
1547 */