ia64/xen-unstable

view xen/arch/x86/mm/shadow/private.h @ 18319:641e10533c89

x86 shadow: Relax BUILD_BUG_ON() regarding struct sizes.
Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Aug 13 12:12:08 2008 +0100 (2008-08-13)
parents 9a1d98a9d21b
children c6402709acc8
line source
1 /******************************************************************************
2 * arch/x86/mm/shadow/private.h
3 *
4 * Shadow code that is private, and does not need to be multiply compiled.
5 * Parts of this code are Copyright (c) 2006 by XenSource Inc.
6 * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
7 * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 */
24 #ifndef _XEN_SHADOW_PRIVATE_H
25 #define _XEN_SHADOW_PRIVATE_H
27 // In order to override the definition of mfn_to_page, we make sure page.h has
28 // been included...
29 #include <asm/page.h>
30 #include <xen/domain_page.h>
31 #include <asm/x86_emulate.h>
32 #include <asm/hvm/support.h>
35 /******************************************************************************
36 * Levels of self-test and paranoia
37 */
39 #define SHADOW_AUDIT_HASH 0x01 /* Check current hash bucket */
40 #define SHADOW_AUDIT_HASH_FULL 0x02 /* Check every hash bucket */
41 #define SHADOW_AUDIT_ENTRIES 0x04 /* Check this walk's shadows */
42 #define SHADOW_AUDIT_ENTRIES_FULL 0x08 /* Check every shadow */
43 #define SHADOW_AUDIT_ENTRIES_MFNS 0x10 /* Check gfn-mfn map in shadows */
45 #ifdef NDEBUG
46 #define SHADOW_AUDIT 0
47 #define SHADOW_AUDIT_ENABLE 0
48 #else
49 #define SHADOW_AUDIT 0x15 /* Basic audit of all */
50 #define SHADOW_AUDIT_ENABLE shadow_audit_enable
51 extern int shadow_audit_enable;
52 #endif
54 /******************************************************************************
55 * Levels of optimization
56 */
58 #define SHOPT_WRITABLE_HEURISTIC 0x01 /* Guess at RW PTEs via linear maps */
59 #define SHOPT_EARLY_UNSHADOW 0x02 /* Unshadow l1s on fork or exit */
60 #define SHOPT_FAST_FAULT_PATH 0x04 /* Fast-path MMIO and not-present */
61 #define SHOPT_PREFETCH 0x08 /* Shadow multiple entries per fault */
62 #define SHOPT_LINUX_L3_TOPLEVEL 0x10 /* Pin l3es on early 64bit linux */
63 #define SHOPT_SKIP_VERIFY 0x20 /* Skip PTE v'fy when safe to do so */
64 #define SHOPT_VIRTUAL_TLB 0x40 /* Cache guest v->p translations */
65 #define SHOPT_FAST_EMULATION 0x80 /* Fast write emulation */
66 #define SHOPT_OUT_OF_SYNC 0x100 /* Allow guest writes to L1 PTs */
68 #define SHADOW_OPTIMIZATIONS 0x1ff
71 /******************************************************************************
72 * Debug and error-message output
73 */
75 #define SHADOW_PRINTK(_f, _a...) \
76 debugtrace_printk("sh: %s(): " _f, __func__, ##_a)
77 #define SHADOW_ERROR(_f, _a...) \
78 printk("sh error: %s(): " _f, __func__, ##_a)
79 #define SHADOW_DEBUG(flag, _f, _a...) \
80 do { \
81 if (SHADOW_DEBUG_ ## flag) \
82 debugtrace_printk("shdebug: %s(): " _f, __func__, ##_a); \
83 } while (0)
85 // The flags for use with SHADOW_DEBUG:
86 #define SHADOW_DEBUG_PROPAGATE 1
87 #define SHADOW_DEBUG_MAKE_SHADOW 1
88 #define SHADOW_DEBUG_DESTROY_SHADOW 1
89 #define SHADOW_DEBUG_A_AND_D 1
90 #define SHADOW_DEBUG_EMULATE 1
91 #define SHADOW_DEBUG_P2M 1
92 #define SHADOW_DEBUG_LOGDIRTY 0
94 /******************************************************************************
95 * The shadow lock.
96 *
97 * This lock is per-domain. It is intended to allow us to make atomic
98 * updates to the software TLB that the shadow tables provide.
99 *
100 * Specifically, it protects:
101 * - all changes to shadow page table pages
102 * - the shadow hash table
103 * - the shadow page allocator
104 * - all changes to guest page table pages
105 * - all changes to the page_info->tlbflush_timestamp
106 * - the page_info->count fields on shadow pages
107 * - the shadow dirty bit array and count
108 */
109 #ifndef CONFIG_SMP
110 #error shadow.h currently requires CONFIG_SMP
111 #endif
113 #define shadow_lock_init(_d) \
114 do { \
115 spin_lock_init(&(_d)->arch.paging.shadow.lock); \
116 (_d)->arch.paging.shadow.locker = -1; \
117 (_d)->arch.paging.shadow.locker_function = "nobody"; \
118 } while (0)
120 #define shadow_locked_by_me(_d) \
121 (current->processor == (_d)->arch.paging.shadow.locker)
123 #define shadow_lock(_d) \
124 do { \
125 if ( unlikely((_d)->arch.paging.shadow.locker == current->processor) )\
126 { \
127 printk("Error: shadow lock held by %s\n", \
128 (_d)->arch.paging.shadow.locker_function); \
129 BUG(); \
130 } \
131 spin_lock(&(_d)->arch.paging.shadow.lock); \
132 ASSERT((_d)->arch.paging.shadow.locker == -1); \
133 (_d)->arch.paging.shadow.locker = current->processor; \
134 (_d)->arch.paging.shadow.locker_function = __func__; \
135 } while (0)
137 #define shadow_unlock(_d) \
138 do { \
139 ASSERT((_d)->arch.paging.shadow.locker == current->processor); \
140 (_d)->arch.paging.shadow.locker = -1; \
141 (_d)->arch.paging.shadow.locker_function = "nobody"; \
142 spin_unlock(&(_d)->arch.paging.shadow.lock); \
143 } while (0)
147 /******************************************************************************
148 * Auditing routines
149 */
151 #if SHADOW_AUDIT & SHADOW_AUDIT_ENTRIES_FULL
152 extern void shadow_audit_tables(struct vcpu *v);
153 #else
154 #define shadow_audit_tables(_v) do {} while(0)
155 #endif
157 /******************************************************************************
158 * Macro for dealing with the naming of the internal names of the
159 * shadow code's external entry points.
160 */
161 #define SHADOW_INTERNAL_NAME_HIDDEN(name, guest_levels) \
162 name ## __guest_ ## guest_levels
163 #define SHADOW_INTERNAL_NAME(name, guest_levels) \
164 SHADOW_INTERNAL_NAME_HIDDEN(name, guest_levels)
166 #define GUEST_LEVELS 2
167 #include "multi.h"
168 #undef GUEST_LEVELS
170 #define GUEST_LEVELS 3
171 #include "multi.h"
172 #undef GUEST_LEVELS
174 #if CONFIG_PAGING_LEVELS == 4
175 #define GUEST_LEVELS 4
176 #include "multi.h"
177 #undef GUEST_LEVELS
178 #endif /* CONFIG_PAGING_LEVELS == 4 */
180 /******************************************************************************
181 * Page metadata for shadow pages.
182 */
184 struct shadow_page_info
185 {
186 union {
187 /* When in use, guest page we're a shadow of */
188 unsigned long backpointer;
189 /* When free, order of the freelist we're on */
190 unsigned int order;
191 };
192 union {
193 /* When in use, next shadow in this hash chain */
194 struct shadow_page_info *next_shadow;
195 /* When free, TLB flush time when freed */
196 u32 tlbflush_timestamp;
197 };
198 struct {
199 unsigned int type:5; /* What kind of shadow is this? */
200 unsigned int pinned:1; /* Is the shadow pinned? */
201 unsigned int count:26; /* Reference count */
202 u32 mbz; /* Must be zero: this is where the owner
203 * field lives in a non-shadow page */
204 } __attribute__((packed));
205 union {
206 /* For unused shadow pages, a list of pages of this order;
207 * for pinnable shadows, if pinned, a list of other pinned shadows
208 * (see sh_type_is_pinnable() below for the definition of
209 * "pinnable" shadow types). */
210 struct list_head list;
211 /* For non-pinnable shadows, a higher entry that points at us */
212 paddr_t up;
213 };
214 };
216 /* The structure above *must* be no larger than a struct page_info
217 * from mm.h, since we'll be using the same space in the frametable.
218 * Also, the mbz field must line up with the owner field of normal
219 * pages, so they look properly like anonymous/xen pages. */
220 static inline void shadow_check_page_struct_offsets(void) {
221 BUILD_BUG_ON(sizeof (struct shadow_page_info) < sizeof (struct page_info));
222 BUILD_BUG_ON(offsetof(struct shadow_page_info, mbz) !=
223 offsetof(struct page_info, u.inuse._domain));
224 };
226 /* Shadow type codes */
227 #define SH_type_none (0U) /* on the shadow free list */
228 #define SH_type_min_shadow (1U)
229 #define SH_type_l1_32_shadow (1U) /* shadowing a 32-bit L1 guest page */
230 #define SH_type_fl1_32_shadow (2U) /* L1 shadow for a 32b 4M superpage */
231 #define SH_type_l2_32_shadow (3U) /* shadowing a 32-bit L2 guest page */
232 #define SH_type_l1_pae_shadow (4U) /* shadowing a pae L1 page */
233 #define SH_type_fl1_pae_shadow (5U) /* L1 shadow for pae 2M superpg */
234 #define SH_type_l2_pae_shadow (6U) /* shadowing a pae L2-low page */
235 #define SH_type_l2h_pae_shadow (7U) /* shadowing a pae L2-high page */
236 #define SH_type_l1_64_shadow (8U) /* shadowing a 64-bit L1 page */
237 #define SH_type_fl1_64_shadow (9U) /* L1 shadow for 64-bit 2M superpg */
238 #define SH_type_l2_64_shadow (10U) /* shadowing a 64-bit L2 page */
239 #define SH_type_l2h_64_shadow (11U) /* shadowing a compat PAE L2 high page */
240 #define SH_type_l3_64_shadow (12U) /* shadowing a 64-bit L3 page */
241 #define SH_type_l4_64_shadow (13U) /* shadowing a 64-bit L4 page */
242 #define SH_type_max_shadow (13U)
243 #define SH_type_p2m_table (14U) /* in use as the p2m table */
244 #define SH_type_monitor_table (15U) /* in use as a monitor table */
245 #define SH_type_oos_snapshot (16U) /* in use as OOS snapshot */
246 #define SH_type_unused (17U)
248 /*
249 * What counts as a pinnable shadow?
250 */
252 static inline int sh_type_is_pinnable(struct vcpu *v, unsigned int t)
253 {
254 /* Top-level shadow types in each mode can be pinned, so that they
255 * persist even when not currently in use in a guest CR3 */
256 if ( t == SH_type_l2_32_shadow
257 || t == SH_type_l2_pae_shadow
258 || t == SH_type_l2h_pae_shadow
259 || t == SH_type_l4_64_shadow )
260 return 1;
262 #if (SHADOW_OPTIMIZATIONS & SHOPT_LINUX_L3_TOPLEVEL)
263 /* Early 64-bit linux used three levels of pagetables for the guest
264 * and context switched by changing one l4 entry in a per-cpu l4
265 * page. When we're shadowing those kernels, we have to pin l3
266 * shadows so they don't just evaporate on every context switch.
267 * For all other guests, we'd rather use the up-pointer field in l3s. */
268 if ( unlikely((v->domain->arch.paging.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL)
269 && CONFIG_PAGING_LEVELS >= 4
270 && t == SH_type_l3_64_shadow) )
271 return 1;
272 #endif
274 /* Everything else is not pinnable, and can use the "up" pointer */
275 return 0;
276 }
278 /*
279 * Definitions for the shadow_flags field in page_info.
280 * These flags are stored on *guest* pages...
281 * Bits 1-13 are encodings for the shadow types.
282 */
283 #define SHF_page_type_mask \
284 (((1u << (SH_type_max_shadow + 1u)) - 1u) - \
285 ((1u << SH_type_min_shadow) - 1u))
287 #define SHF_L1_32 (1u << SH_type_l1_32_shadow)
288 #define SHF_FL1_32 (1u << SH_type_fl1_32_shadow)
289 #define SHF_L2_32 (1u << SH_type_l2_32_shadow)
290 #define SHF_L1_PAE (1u << SH_type_l1_pae_shadow)
291 #define SHF_FL1_PAE (1u << SH_type_fl1_pae_shadow)
292 #define SHF_L2_PAE (1u << SH_type_l2_pae_shadow)
293 #define SHF_L2H_PAE (1u << SH_type_l2h_pae_shadow)
294 #define SHF_L1_64 (1u << SH_type_l1_64_shadow)
295 #define SHF_FL1_64 (1u << SH_type_fl1_64_shadow)
296 #define SHF_L2_64 (1u << SH_type_l2_64_shadow)
297 #define SHF_L2H_64 (1u << SH_type_l2h_64_shadow)
298 #define SHF_L3_64 (1u << SH_type_l3_64_shadow)
299 #define SHF_L4_64 (1u << SH_type_l4_64_shadow)
301 #define SHF_32 (SHF_L1_32|SHF_FL1_32|SHF_L2_32)
302 #define SHF_PAE (SHF_L1_PAE|SHF_FL1_PAE|SHF_L2_PAE|SHF_L2H_PAE)
303 #define SHF_64 (SHF_L1_64|SHF_FL1_64|SHF_L2_64|SHF_L2H_64|SHF_L3_64|SHF_L4_64)
305 #define SHF_L1_ANY (SHF_L1_32|SHF_L1_PAE|SHF_L1_64)
307 #if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
308 /* Marks a guest L1 page table which is shadowed but not write-protected.
309 * If set, then *only* L1 shadows (SHF_L1_*) are allowed.
310 *
311 * out_of_sync indicates that the shadow tables may not reflect the
312 * guest tables. If it is clear, then the shadow tables *must* reflect
313 * the guest tables.
314 *
315 * oos_may_write indicates that a page may have writable mappings.
316 *
317 * Most of the time the flags are synonymous. There is a short period of time
318 * during resync that oos_may_write is clear but out_of_sync is not. If a
319 * codepath is called during that time and is sensitive to oos issues, it may
320 * need to use the second flag.
321 */
322 #define SHF_out_of_sync (1u<<30)
323 #define SHF_oos_may_write (1u<<29)
325 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC) */
327 static inline int sh_page_has_multiple_shadows(struct page_info *pg)
328 {
329 u32 shadows;
330 if ( !(pg->count_info & PGC_page_table) )
331 return 0;
332 shadows = pg->shadow_flags & SHF_page_type_mask;
333 /* More than one type bit set in shadow-flags? */
334 return ( (shadows & ~(1UL << find_first_set_bit(shadows))) != 0 );
335 }
337 #if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
338 /* The caller must verify this is reasonable to call; i.e., valid mfn,
339 * domain is translated, &c */
340 static inline int page_is_out_of_sync(struct page_info *p)
341 {
342 return (p->count_info & PGC_page_table)
343 && (p->shadow_flags & SHF_out_of_sync);
344 }
346 static inline int mfn_is_out_of_sync(mfn_t gmfn)
347 {
348 return page_is_out_of_sync(mfn_to_page(mfn_x(gmfn)));
349 }
351 static inline int page_oos_may_write(struct page_info *p)
352 {
353 return (p->count_info & PGC_page_table)
354 && (p->shadow_flags & SHF_oos_may_write);
355 }
357 static inline int mfn_oos_may_write(mfn_t gmfn)
358 {
359 return page_oos_may_write(mfn_to_page(mfn_x(gmfn)));
360 }
361 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC) */
363 /******************************************************************************
364 * Various function declarations
365 */
367 /* Hash table functions */
368 mfn_t shadow_hash_lookup(struct vcpu *v, unsigned long n, unsigned int t);
369 void shadow_hash_insert(struct vcpu *v,
370 unsigned long n, unsigned int t, mfn_t smfn);
371 void shadow_hash_delete(struct vcpu *v,
372 unsigned long n, unsigned int t, mfn_t smfn);
374 /* shadow promotion */
375 void shadow_promote(struct vcpu *v, mfn_t gmfn, u32 type);
376 void shadow_demote(struct vcpu *v, mfn_t gmfn, u32 type);
378 /* Shadow page allocation functions */
379 void shadow_prealloc(struct domain *d, u32 shadow_type, unsigned int count);
380 mfn_t shadow_alloc(struct domain *d,
381 u32 shadow_type,
382 unsigned long backpointer);
383 void shadow_free(struct domain *d, mfn_t smfn);
385 /* Install the xen mappings in various flavours of shadow */
386 void sh_install_xen_entries_in_l4(struct vcpu *v, mfn_t gl4mfn, mfn_t sl4mfn);
388 /* Update the shadows in response to a pagetable write from Xen */
389 int sh_validate_guest_entry(struct vcpu *v, mfn_t gmfn, void *entry, u32 size);
391 /* Update the shadows in response to a pagetable write from a HVM guest */
392 void sh_validate_guest_pt_write(struct vcpu *v, mfn_t gmfn,
393 void *entry, u32 size);
395 /* Remove all writeable mappings of a guest frame from the shadows.
396 * Returns non-zero if we need to flush TLBs.
397 * level and fault_addr desribe how we found this to be a pagetable;
398 * level==0 means we have some other reason for revoking write access. */
399 extern int sh_remove_write_access(struct vcpu *v, mfn_t readonly_mfn,
400 unsigned int level,
401 unsigned long fault_addr);
403 /* Functions that atomically write PT/P2M entries and update state */
404 void shadow_write_p2m_entry(struct vcpu *v, unsigned long gfn,
405 l1_pgentry_t *p, mfn_t table_mfn,
406 l1_pgentry_t new, unsigned int level);
407 int shadow_write_guest_entry(struct vcpu *v, intpte_t *p,
408 intpte_t new, mfn_t gmfn);
409 int shadow_cmpxchg_guest_entry(struct vcpu *v, intpte_t *p,
410 intpte_t *old, intpte_t new, mfn_t gmfn);
412 #if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
413 /* Allow a shadowed page to go out of sync */
414 int sh_unsync(struct vcpu *v, mfn_t gmfn);
416 /* Pull an out-of-sync page back into sync. */
417 void sh_resync(struct vcpu *v, mfn_t gmfn);
419 void oos_fixup_add(struct vcpu *v, mfn_t gmfn, mfn_t smfn, unsigned long off);
421 int sh_remove_write_access_from_sl1p(struct vcpu *v, mfn_t gmfn,
422 mfn_t smfn, unsigned long offset);
424 /* Pull all out-of-sync shadows back into sync. If skip != 0, we try
425 * to avoid resyncing where we think we can get away with it. */
427 void sh_resync_all(struct vcpu *v, int skip, int this, int others, int do_locking);
429 static inline void
430 shadow_resync_all(struct vcpu *v, int do_locking)
431 {
432 sh_resync_all(v,
433 0 /* skip */,
434 1 /* this */,
435 1 /* others */,
436 do_locking);
437 }
439 static inline void
440 shadow_resync_current_vcpu(struct vcpu *v, int do_locking)
441 {
442 sh_resync_all(v,
443 0 /* skip */,
444 1 /* this */,
445 0 /* others */,
446 do_locking);
447 }
449 static inline void
450 shadow_sync_other_vcpus(struct vcpu *v, int do_locking)
451 {
452 sh_resync_all(v,
453 1 /* skip */,
454 0 /* this */,
455 1 /* others */,
456 do_locking);
457 }
459 void oos_audit_hash_is_present(struct domain *d, mfn_t gmfn);
460 mfn_t oos_snapshot_lookup(struct vcpu *v, mfn_t gmfn);
462 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC) */
464 /******************************************************************************
465 * Flags used in the return value of the shadow_set_lXe() functions...
466 */
468 /* We actually wrote something new to the shadow */
469 #define SHADOW_SET_CHANGED 0x1
470 /* Caller should flush TLBs to clear the old entry */
471 #define SHADOW_SET_FLUSH 0x2
472 /* Something went wrong: the shadow entry was invalid or refcount failed */
473 #define SHADOW_SET_ERROR 0x4
476 /******************************************************************************
477 * MFN/page-info handling
478 */
480 // Override mfn_to_page from asm/page.h, which was #include'd above,
481 // in order to make it work with our mfn type.
482 #undef mfn_to_page
483 #define mfn_to_page(_m) (frame_table + mfn_x(_m))
484 #define mfn_to_shadow_page(_m) ((struct shadow_page_info *)mfn_to_page(_m))
486 // Override page_to_mfn from asm/page.h, which was #include'd above,
487 // in order to make it work with our mfn type.
488 #undef page_to_mfn
489 #define page_to_mfn(_pg) (_mfn((_pg) - frame_table))
490 #define shadow_page_to_mfn(_spg) (page_to_mfn((struct page_info *)_spg))
492 // Override mfn_valid from asm/page.h, which was #include'd above,
493 // in order to make it work with our mfn type.
494 #undef mfn_valid
495 #define mfn_valid(_mfn) (mfn_x(_mfn) < max_page)
497 /* Override pagetable_t <-> struct page_info conversions to work with mfn_t */
498 #undef pagetable_get_page
499 #define pagetable_get_page(x) mfn_to_page(pagetable_get_mfn(x))
500 #undef pagetable_from_page
501 #define pagetable_from_page(pg) pagetable_from_mfn(page_to_mfn(pg))
503 static inline int
504 sh_mfn_is_a_page_table(mfn_t gmfn)
505 {
506 struct page_info *page = mfn_to_page(gmfn);
507 struct domain *owner;
508 unsigned long type_info;
510 if ( !mfn_valid(gmfn) )
511 return 0;
513 owner = page_get_owner(page);
514 if ( owner && shadow_mode_refcounts(owner)
515 && (page->count_info & PGC_page_table) )
516 return 1;
518 type_info = page->u.inuse.type_info & PGT_type_mask;
519 return type_info && (type_info <= PGT_l4_page_table);
520 }
522 // Provide mfn_t-aware versions of common xen functions
523 static inline void *
524 sh_map_domain_page(mfn_t mfn)
525 {
526 return map_domain_page(mfn_x(mfn));
527 }
529 static inline void
530 sh_unmap_domain_page(void *p)
531 {
532 unmap_domain_page(p);
533 }
535 static inline void *
536 sh_map_domain_page_global(mfn_t mfn)
537 {
538 return map_domain_page_global(mfn_x(mfn));
539 }
541 static inline void
542 sh_unmap_domain_page_global(void *p)
543 {
544 unmap_domain_page_global(p);
545 }
547 /******************************************************************************
548 * Log-dirty mode bitmap handling
549 */
551 extern void sh_mark_dirty(struct domain *d, mfn_t gmfn);
553 static inline int
554 sh_mfn_is_dirty(struct domain *d, mfn_t gmfn)
555 /* Is this guest page dirty? Call only in log-dirty mode. */
556 {
557 unsigned long pfn;
558 mfn_t mfn, *l4, *l3, *l2;
559 unsigned long *l1;
560 int rv;
562 ASSERT(shadow_mode_log_dirty(d));
563 ASSERT(mfn_valid(d->arch.paging.log_dirty.top));
565 /* We /really/ mean PFN here, even for non-translated guests. */
566 pfn = get_gpfn_from_mfn(mfn_x(gmfn));
567 if ( unlikely(!VALID_M2P(pfn)) )
568 return 0;
570 if ( d->arch.paging.log_dirty.failed_allocs > 0 )
571 /* If we have any failed allocations our dirty log is bogus.
572 * Since we can't signal an error here, be conservative and
573 * report "dirty" in this case. (The only current caller,
574 * _sh_propagate, leaves known-dirty pages writable, preventing
575 * subsequent dirty-logging faults from them.)
576 */
577 return 1;
579 l4 = map_domain_page(mfn_x(d->arch.paging.log_dirty.top));
580 mfn = l4[L4_LOGDIRTY_IDX(pfn)];
581 unmap_domain_page(l4);
582 if ( !mfn_valid(mfn) )
583 return 0;
585 l3 = map_domain_page(mfn_x(mfn));
586 mfn = l3[L3_LOGDIRTY_IDX(pfn)];
587 unmap_domain_page(l3);
588 if ( !mfn_valid(mfn) )
589 return 0;
591 l2 = map_domain_page(mfn_x(mfn));
592 mfn = l2[L2_LOGDIRTY_IDX(pfn)];
593 unmap_domain_page(l2);
594 if ( !mfn_valid(mfn) )
595 return 0;
597 l1 = map_domain_page(mfn_x(mfn));
598 rv = test_bit(L1_LOGDIRTY_IDX(pfn), l1);
599 unmap_domain_page(l1);
601 return rv;
602 }
604 /**************************************************************************/
605 /* VRAM dirty tracking support */
607 struct sh_dirty_vram {
608 unsigned long begin_pfn;
609 unsigned long end_pfn;
610 paddr_t *sl1ma;
611 uint8_t *dirty_bitmap;
612 s_time_t last_dirty;
613 };
615 /**************************************************************************/
616 /* Shadow-page refcounting. */
618 void sh_destroy_shadow(struct vcpu *v, mfn_t smfn);
620 /* Increase the refcount of a shadow page. Arguments are the mfn to refcount,
621 * and the physical address of the shadow entry that holds the ref (or zero
622 * if the ref is held by something else).
623 * Returns 0 for failure, 1 for success. */
624 static inline int sh_get_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
625 {
626 u32 x, nx;
627 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
629 ASSERT(mfn_valid(smfn));
631 x = sp->count;
632 nx = x + 1;
634 if ( unlikely(nx >= 1U<<26) )
635 {
636 SHADOW_PRINTK("shadow ref overflow, gmfn=%" PRtype_info " smfn=%lx\n",
637 sp->backpointer, mfn_x(smfn));
638 return 0;
639 }
641 /* Guarded by the shadow lock, so no need for atomic update */
642 sp->count = nx;
644 /* We remember the first shadow entry that points to each shadow. */
645 if ( entry_pa != 0
646 && !sh_type_is_pinnable(v, sp->type)
647 && sp->up == 0 )
648 sp->up = entry_pa;
650 return 1;
651 }
654 /* Decrease the refcount of a shadow page. As for get_ref, takes the
655 * physical address of the shadow entry that held this reference. */
656 static inline void sh_put_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
657 {
658 u32 x, nx;
659 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
661 ASSERT(mfn_valid(smfn));
662 ASSERT(sp->mbz == 0);
664 /* If this is the entry in the up-pointer, remove it */
665 if ( entry_pa != 0
666 && !sh_type_is_pinnable(v, sp->type)
667 && sp->up == entry_pa )
668 sp->up = 0;
670 x = sp->count;
671 nx = x - 1;
673 if ( unlikely(x == 0) )
674 {
675 SHADOW_ERROR("shadow ref underflow, smfn=%lx oc=%08x t=%#x\n",
676 mfn_x(smfn), sp->count, sp->type);
677 BUG();
678 }
680 /* Guarded by the shadow lock, so no need for atomic update */
681 sp->count = nx;
683 if ( unlikely(nx == 0) )
684 sh_destroy_shadow(v, smfn);
685 }
688 /* Pin a shadow page: take an extra refcount, set the pin bit,
689 * and put the shadow at the head of the list of pinned shadows.
690 * Returns 0 for failure, 1 for success. */
691 static inline int sh_pin(struct vcpu *v, mfn_t smfn)
692 {
693 struct shadow_page_info *sp;
695 ASSERT(mfn_valid(smfn));
696 sp = mfn_to_shadow_page(smfn);
697 ASSERT(sh_type_is_pinnable(v, sp->type));
698 if ( sp->pinned )
699 {
700 /* Already pinned: take it out of the pinned-list so it can go
701 * at the front */
702 list_del(&sp->list);
703 }
704 else
705 {
706 /* Not pinned: pin it! */
707 if ( !sh_get_ref(v, smfn, 0) )
708 return 0;
709 sp->pinned = 1;
710 }
711 /* Put it at the head of the list of pinned shadows */
712 list_add(&sp->list, &v->domain->arch.paging.shadow.pinned_shadows);
713 return 1;
714 }
716 /* Unpin a shadow page: unset the pin bit, take the shadow off the list
717 * of pinned shadows, and release the extra ref. */
718 static inline void sh_unpin(struct vcpu *v, mfn_t smfn)
719 {
720 struct shadow_page_info *sp;
722 ASSERT(mfn_valid(smfn));
723 sp = mfn_to_shadow_page(smfn);
724 ASSERT(sh_type_is_pinnable(v, sp->type));
725 if ( sp->pinned )
726 {
727 sp->pinned = 0;
728 list_del(&sp->list);
729 sp->up = 0; /* in case this stops being a pinnable type in future */
730 sh_put_ref(v, smfn, 0);
731 }
732 }
735 /**************************************************************************/
736 /* PTE-write emulation. */
738 struct sh_emulate_ctxt {
739 struct x86_emulate_ctxt ctxt;
741 /* Cache of up to 31 bytes of instruction. */
742 uint8_t insn_buf[31];
743 uint8_t insn_buf_bytes;
744 unsigned long insn_buf_eip;
746 /* Cache of segment registers already gathered for this emulation. */
747 unsigned int valid_seg_regs;
748 struct segment_register seg_reg[6];
750 /* MFNs being written to in write/cmpxchg callbacks */
751 mfn_t mfn1, mfn2;
753 #if (SHADOW_OPTIMIZATIONS & SHOPT_SKIP_VERIFY)
754 /* Special case for avoiding having to verify writes: remember
755 * whether the old value had its low bit (_PAGE_PRESENT) clear. */
756 int low_bit_was_clear:1;
757 #endif
758 };
760 struct x86_emulate_ops *shadow_init_emulation(
761 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
762 void shadow_continue_emulation(
763 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
764 struct segment_register *hvm_get_seg_reg(
765 enum x86_segment seg, struct sh_emulate_ctxt *sh_ctxt);
767 #if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
768 /**************************************************************************/
769 /* Virtual TLB entries
770 *
771 * We keep a cache of virtual-to-physical translations that we have seen
772 * since the last TLB flush. This is safe to use for frame translations,
773 * but callers need to re-check the actual guest tables if the lookup fails.
774 *
775 * Lookups and updates are protected by a per-vTLB (and hence per-vcpu)
776 * lock. This lock is held *only* while reading or writing the table,
777 * so it is safe to take in any non-interrupt context. Most lookups
778 * happen with v==current, so we expect contention to be low.
779 */
781 #define VTLB_ENTRIES 13
783 struct shadow_vtlb {
784 unsigned long page_number; /* Guest virtual address >> PAGE_SHIFT */
785 unsigned long frame_number; /* Guest physical address >> PAGE_SHIFT */
786 uint32_t pfec; /* PF error code of the lookup that filled this
787 * entry. A pfec of zero means the slot is empty
788 * (since that would require us to re-try anyway) */
789 };
791 /* Call whenever the guest flushes hit actual TLB */
792 static inline void vtlb_flush(struct vcpu *v)
793 {
794 spin_lock(&v->arch.paging.vtlb_lock);
795 memset(v->arch.paging.vtlb, 0, VTLB_ENTRIES * sizeof (struct shadow_vtlb));
796 spin_unlock(&v->arch.paging.vtlb_lock);
797 }
799 static inline int vtlb_hash(unsigned long page_number)
800 {
801 return page_number % VTLB_ENTRIES;
802 }
804 /* Put a translation into the vTLB, potentially clobbering an old one */
805 static inline void vtlb_insert(struct vcpu *v, unsigned long page,
806 unsigned long frame, uint32_t pfec)
807 {
808 struct shadow_vtlb entry =
809 { .page_number = page, .frame_number = frame, .pfec = pfec };
810 spin_lock(&v->arch.paging.vtlb_lock);
811 v->arch.paging.vtlb[vtlb_hash(page)] = entry;
812 spin_unlock(&v->arch.paging.vtlb_lock);
813 }
815 /* Look a translation up in the vTLB. Returns INVALID_GFN if not found. */
816 static inline unsigned long vtlb_lookup(struct vcpu *v,
817 unsigned long va, uint32_t pfec)
818 {
819 unsigned long page_number = va >> PAGE_SHIFT;
820 unsigned long frame_number = INVALID_GFN;
821 int i = vtlb_hash(page_number);
823 spin_lock(&v->arch.paging.vtlb_lock);
824 if ( v->arch.paging.vtlb[i].pfec != 0
825 && v->arch.paging.vtlb[i].page_number == page_number
826 /* Any successful walk that had at least these pfec bits is OK */
827 && (v->arch.paging.vtlb[i].pfec & pfec) == pfec )
828 {
829 frame_number = v->arch.paging.vtlb[i].frame_number;
830 }
831 spin_unlock(&v->arch.paging.vtlb_lock);
832 return frame_number;
833 }
834 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
837 #endif /* _XEN_SHADOW_PRIVATE_H */
839 /*
840 * Local variables:
841 * mode: C
842 * c-set-style: "BSD"
843 * c-basic-offset: 4
844 * indent-tabs-mode: nil
845 * End:
846 */