ia64/xen-unstable

view xen/arch/x86/mm/shadow/private.h @ 16626:d9ab9eb2bfee

HVM: support unaligned and page-crossing writes in the shadow emulator
so that we can use it to support guests that clear CR0.WP.

Signed-off-by: Tim Deegan <Tim.Deegan@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Sat Dec 15 18:29:27 2007 +0000 (2007-12-15)
parents 8e98c3d6a55f
children 9d447ba0c99a
line source
1 /******************************************************************************
2 * arch/x86/mm/shadow/private.h
3 *
4 * Shadow code that is private, and does not need to be multiply compiled.
5 * Parts of this code are Copyright (c) 2006 by XenSource Inc.
6 * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
7 * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 */
24 #ifndef _XEN_SHADOW_PRIVATE_H
25 #define _XEN_SHADOW_PRIVATE_H
27 // In order to override the definition of mfn_to_page, we make sure page.h has
28 // been included...
29 #include <asm/page.h>
30 #include <xen/domain_page.h>
31 #include <asm/x86_emulate.h>
32 #include <asm/hvm/support.h>
35 /******************************************************************************
36 * Levels of self-test and paranoia
37 */
39 #define SHADOW_AUDIT_HASH 0x01 /* Check current hash bucket */
40 #define SHADOW_AUDIT_HASH_FULL 0x02 /* Check every hash bucket */
41 #define SHADOW_AUDIT_ENTRIES 0x04 /* Check this walk's shadows */
42 #define SHADOW_AUDIT_ENTRIES_FULL 0x08 /* Check every shadow */
43 #define SHADOW_AUDIT_ENTRIES_MFNS 0x10 /* Check gfn-mfn map in shadows */
45 #ifdef NDEBUG
46 #define SHADOW_AUDIT 0
47 #define SHADOW_AUDIT_ENABLE 0
48 #else
49 #define SHADOW_AUDIT 0x15 /* Basic audit of all */
50 #define SHADOW_AUDIT_ENABLE shadow_audit_enable
51 extern int shadow_audit_enable;
52 #endif
54 /******************************************************************************
55 * Levels of optimization
56 */
58 #define SHOPT_WRITABLE_HEURISTIC 0x01 /* Guess at RW PTEs via linear maps */
59 #define SHOPT_EARLY_UNSHADOW 0x02 /* Unshadow l1s on fork or exit */
60 #define SHOPT_FAST_FAULT_PATH 0x04 /* Fast-path MMIO and not-present */
61 #define SHOPT_PREFETCH 0x08 /* Shadow multiple entries per fault */
62 #define SHOPT_LINUX_L3_TOPLEVEL 0x10 /* Pin l3es on early 64bit linux */
63 #define SHOPT_SKIP_VERIFY 0x20 /* Skip PTE v'fy when safe to do so */
64 #define SHOPT_VIRTUAL_TLB 0x40 /* Cache guest v->p translations */
66 #define SHADOW_OPTIMIZATIONS 0x7f
69 /******************************************************************************
70 * Debug and error-message output
71 */
73 #define SHADOW_PRINTK(_f, _a...) \
74 debugtrace_printk("sh: %s(): " _f, __func__, ##_a)
75 #define SHADOW_ERROR(_f, _a...) \
76 printk("sh error: %s(): " _f, __func__, ##_a)
77 #define SHADOW_DEBUG(flag, _f, _a...) \
78 do { \
79 if (SHADOW_DEBUG_ ## flag) \
80 debugtrace_printk("shdebug: %s(): " _f, __func__, ##_a); \
81 } while (0)
83 // The flags for use with SHADOW_DEBUG:
84 #define SHADOW_DEBUG_PROPAGATE 1
85 #define SHADOW_DEBUG_MAKE_SHADOW 1
86 #define SHADOW_DEBUG_DESTROY_SHADOW 1
87 #define SHADOW_DEBUG_A_AND_D 1
88 #define SHADOW_DEBUG_EMULATE 1
89 #define SHADOW_DEBUG_P2M 1
90 #define SHADOW_DEBUG_LOGDIRTY 0
92 /******************************************************************************
93 * The shadow lock.
94 *
95 * This lock is per-domain. It is intended to allow us to make atomic
96 * updates to the software TLB that the shadow tables provide.
97 *
98 * Specifically, it protects:
99 * - all changes to shadow page table pages
100 * - the shadow hash table
101 * - the shadow page allocator
102 * - all changes to guest page table pages
103 * - all changes to the page_info->tlbflush_timestamp
104 * - the page_info->count fields on shadow pages
105 * - the shadow dirty bit array and count
106 */
107 #ifndef CONFIG_SMP
108 #error shadow.h currently requires CONFIG_SMP
109 #endif
111 #define shadow_lock_init(_d) \
112 do { \
113 spin_lock_init(&(_d)->arch.paging.shadow.lock); \
114 (_d)->arch.paging.shadow.locker = -1; \
115 (_d)->arch.paging.shadow.locker_function = "nobody"; \
116 } while (0)
118 #define shadow_locked_by_me(_d) \
119 (current->processor == (_d)->arch.paging.shadow.locker)
121 #define shadow_lock(_d) \
122 do { \
123 if ( unlikely((_d)->arch.paging.shadow.locker == current->processor) )\
124 { \
125 printk("Error: shadow lock held by %s\n", \
126 (_d)->arch.paging.shadow.locker_function); \
127 BUG(); \
128 } \
129 spin_lock(&(_d)->arch.paging.shadow.lock); \
130 ASSERT((_d)->arch.paging.shadow.locker == -1); \
131 (_d)->arch.paging.shadow.locker = current->processor; \
132 (_d)->arch.paging.shadow.locker_function = __func__; \
133 } while (0)
135 #define shadow_unlock(_d) \
136 do { \
137 ASSERT((_d)->arch.paging.shadow.locker == current->processor); \
138 (_d)->arch.paging.shadow.locker = -1; \
139 (_d)->arch.paging.shadow.locker_function = "nobody"; \
140 spin_unlock(&(_d)->arch.paging.shadow.lock); \
141 } while (0)
145 /******************************************************************************
146 * Auditing routines
147 */
149 #if SHADOW_AUDIT & SHADOW_AUDIT_ENTRIES_FULL
150 extern void shadow_audit_tables(struct vcpu *v);
151 #else
152 #define shadow_audit_tables(_v) do {} while(0)
153 #endif
155 /******************************************************************************
156 * Macro for dealing with the naming of the internal names of the
157 * shadow code's external entry points.
158 */
159 #define SHADOW_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels) \
160 name ## __shadow_ ## shadow_levels ## _guest_ ## guest_levels
161 #define SHADOW_INTERNAL_NAME(name, shadow_levels, guest_levels) \
162 SHADOW_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels)
164 #if CONFIG_PAGING_LEVELS == 2
165 #define GUEST_LEVELS 2
166 #define SHADOW_LEVELS 2
167 #include "multi.h"
168 #undef GUEST_LEVELS
169 #undef SHADOW_LEVELS
170 #endif /* CONFIG_PAGING_LEVELS == 2 */
172 #if CONFIG_PAGING_LEVELS == 3
173 #define GUEST_LEVELS 2
174 #define SHADOW_LEVELS 3
175 #include "multi.h"
176 #undef GUEST_LEVELS
177 #undef SHADOW_LEVELS
179 #define GUEST_LEVELS 3
180 #define SHADOW_LEVELS 3
181 #include "multi.h"
182 #undef GUEST_LEVELS
183 #undef SHADOW_LEVELS
184 #endif /* CONFIG_PAGING_LEVELS == 3 */
186 #if CONFIG_PAGING_LEVELS == 4
187 #define GUEST_LEVELS 2
188 #define SHADOW_LEVELS 3
189 #include "multi.h"
190 #undef GUEST_LEVELS
191 #undef SHADOW_LEVELS
193 #define GUEST_LEVELS 3
194 #define SHADOW_LEVELS 3
195 #include "multi.h"
196 #undef GUEST_LEVELS
197 #undef SHADOW_LEVELS
199 #define GUEST_LEVELS 3
200 #define SHADOW_LEVELS 4
201 #include "multi.h"
202 #undef GUEST_LEVELS
203 #undef SHADOW_LEVELS
205 #define GUEST_LEVELS 4
206 #define SHADOW_LEVELS 4
207 #include "multi.h"
208 #undef GUEST_LEVELS
209 #undef SHADOW_LEVELS
210 #endif /* CONFIG_PAGING_LEVELS == 4 */
212 /******************************************************************************
213 * Page metadata for shadow pages.
214 */
216 struct shadow_page_info
217 {
218 union {
219 /* When in use, guest page we're a shadow of */
220 unsigned long backpointer;
221 /* When free, order of the freelist we're on */
222 unsigned int order;
223 };
224 union {
225 /* When in use, next shadow in this hash chain */
226 struct shadow_page_info *next_shadow;
227 /* When free, TLB flush time when freed */
228 u32 tlbflush_timestamp;
229 };
230 struct {
231 unsigned int type:4; /* What kind of shadow is this? */
232 unsigned int pinned:1; /* Is the shadow pinned? */
233 unsigned int count:27; /* Reference count */
234 u32 mbz; /* Must be zero: this is where the owner
235 * field lives in a non-shadow page */
236 } __attribute__((packed));
237 union {
238 /* For unused shadow pages, a list of pages of this order;
239 * for pinnable shadows, if pinned, a list of other pinned shadows
240 * (see sh_type_is_pinnable() below for the definition of
241 * "pinnable" shadow types). */
242 struct list_head list;
243 /* For non-pinnable shadows, a higher entry that points at us */
244 paddr_t up;
245 };
246 };
248 /* The structure above *must* be the same size as a struct page_info
249 * from mm.h, since we'll be using the same space in the frametable.
250 * Also, the mbz field must line up with the owner field of normal
251 * pages, so they look properly like anonymous/xen pages. */
252 static inline void shadow_check_page_struct_offsets(void) {
253 BUILD_BUG_ON(sizeof (struct shadow_page_info)
254 != sizeof (struct page_info));
255 BUILD_BUG_ON(offsetof(struct shadow_page_info, mbz)
256 != offsetof(struct page_info, u.inuse._domain));
257 };
259 /* Shadow type codes */
260 #define SH_type_none (0U) /* on the shadow free list */
261 #define SH_type_min_shadow (1U)
262 #define SH_type_l1_32_shadow (1U) /* shadowing a 32-bit L1 guest page */
263 #define SH_type_fl1_32_shadow (2U) /* L1 shadow for a 32b 4M superpage */
264 #define SH_type_l2_32_shadow (3U) /* shadowing a 32-bit L2 guest page */
265 #define SH_type_l1_pae_shadow (4U) /* shadowing a pae L1 page */
266 #define SH_type_fl1_pae_shadow (5U) /* L1 shadow for pae 2M superpg */
267 #define SH_type_l2_pae_shadow (6U) /* shadowing a pae L2-low page */
268 #define SH_type_l2h_pae_shadow (7U) /* shadowing a pae L2-high page */
269 #define SH_type_l1_64_shadow (8U) /* shadowing a 64-bit L1 page */
270 #define SH_type_fl1_64_shadow (9U) /* L1 shadow for 64-bit 2M superpg */
271 #define SH_type_l2_64_shadow (10U) /* shadowing a 64-bit L2 page */
272 #define SH_type_l2h_64_shadow (11U) /* shadowing a compat PAE L2 high page */
273 #define SH_type_l3_64_shadow (12U) /* shadowing a 64-bit L3 page */
274 #define SH_type_l4_64_shadow (13U) /* shadowing a 64-bit L4 page */
275 #define SH_type_max_shadow (13U)
276 #define SH_type_p2m_table (14U) /* in use as the p2m table */
277 #define SH_type_monitor_table (15U) /* in use as a monitor table */
278 #define SH_type_unused (16U)
280 /*
281 * What counts as a pinnable shadow?
282 */
284 static inline int sh_type_is_pinnable(struct vcpu *v, unsigned int t)
285 {
286 /* Top-level shadow types in each mode can be pinned, so that they
287 * persist even when not currently in use in a guest CR3 */
288 if ( t == SH_type_l2_32_shadow
289 || t == SH_type_l2_pae_shadow
290 || t == SH_type_l2h_pae_shadow
291 || t == SH_type_l4_64_shadow )
292 return 1;
294 #if (SHADOW_OPTIMIZATIONS & SHOPT_LINUX_L3_TOPLEVEL)
295 /* Early 64-bit linux used three levels of pagetables for the guest
296 * and context switched by changing one l4 entry in a per-cpu l4
297 * page. When we're shadowing those kernels, we have to pin l3
298 * shadows so they don't just evaporate on every context switch.
299 * For all other guests, we'd rather use the up-pointer field in l3s. */
300 if ( unlikely((v->domain->arch.paging.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL)
301 && CONFIG_PAGING_LEVELS >= 4
302 && t == SH_type_l3_64_shadow) )
303 return 1;
304 #endif
306 /* Everything else is not pinnable, and can use the "up" pointer */
307 return 0;
308 }
310 /*
311 * Definitions for the shadow_flags field in page_info.
312 * These flags are stored on *guest* pages...
313 * Bits 1-13 are encodings for the shadow types.
314 */
315 #define SHF_page_type_mask \
316 (((1u << (SH_type_max_shadow + 1u)) - 1u) - \
317 ((1u << SH_type_min_shadow) - 1u))
319 #define SHF_L1_32 (1u << SH_type_l1_32_shadow)
320 #define SHF_FL1_32 (1u << SH_type_fl1_32_shadow)
321 #define SHF_L2_32 (1u << SH_type_l2_32_shadow)
322 #define SHF_L1_PAE (1u << SH_type_l1_pae_shadow)
323 #define SHF_FL1_PAE (1u << SH_type_fl1_pae_shadow)
324 #define SHF_L2_PAE (1u << SH_type_l2_pae_shadow)
325 #define SHF_L2H_PAE (1u << SH_type_l2h_pae_shadow)
326 #define SHF_L1_64 (1u << SH_type_l1_64_shadow)
327 #define SHF_FL1_64 (1u << SH_type_fl1_64_shadow)
328 #define SHF_L2_64 (1u << SH_type_l2_64_shadow)
329 #define SHF_L2H_64 (1u << SH_type_l2h_64_shadow)
330 #define SHF_L3_64 (1u << SH_type_l3_64_shadow)
331 #define SHF_L4_64 (1u << SH_type_l4_64_shadow)
333 #define SHF_32 (SHF_L1_32|SHF_FL1_32|SHF_L2_32)
334 #define SHF_PAE (SHF_L1_PAE|SHF_FL1_PAE|SHF_L2_PAE|SHF_L2H_PAE)
335 #define SHF_64 (SHF_L1_64|SHF_FL1_64|SHF_L2_64|SHF_L2H_64|SHF_L3_64|SHF_L4_64)
337 /* Used for hysteresis when automatically unhooking mappings on fork/exit */
338 #define SHF_unhooked_mappings (1u<<31)
341 /******************************************************************************
342 * Various function declarations
343 */
345 /* Hash table functions */
346 mfn_t shadow_hash_lookup(struct vcpu *v, unsigned long n, unsigned int t);
347 void shadow_hash_insert(struct vcpu *v,
348 unsigned long n, unsigned int t, mfn_t smfn);
349 void shadow_hash_delete(struct vcpu *v,
350 unsigned long n, unsigned int t, mfn_t smfn);
352 /* shadow promotion */
353 void shadow_promote(struct vcpu *v, mfn_t gmfn, u32 type);
354 void shadow_demote(struct vcpu *v, mfn_t gmfn, u32 type);
356 /* Shadow page allocation functions */
357 void shadow_prealloc(struct domain *d, u32 shadow_type, unsigned int count);
358 mfn_t shadow_alloc(struct domain *d,
359 u32 shadow_type,
360 unsigned long backpointer);
361 void shadow_free(struct domain *d, mfn_t smfn);
363 /* Install the xen mappings in various flavours of shadow */
364 void sh_install_xen_entries_in_l4(struct vcpu *v, mfn_t gl4mfn, mfn_t sl4mfn);
365 void sh_install_xen_entries_in_l2(struct vcpu *v, mfn_t gl2mfn, mfn_t sl2mfn);
367 /* Update the shadows in response to a pagetable write from Xen */
368 int sh_validate_guest_entry(struct vcpu *v, mfn_t gmfn, void *entry, u32 size);
370 /* Update the shadows in response to a pagetable write from a HVM guest */
371 void sh_validate_guest_pt_write(struct vcpu *v, mfn_t gmfn,
372 void *entry, u32 size);
374 /* Remove all writeable mappings of a guest frame from the shadows.
375 * Returns non-zero if we need to flush TLBs.
376 * level and fault_addr desribe how we found this to be a pagetable;
377 * level==0 means we have some other reason for revoking write access. */
378 extern int sh_remove_write_access(struct vcpu *v, mfn_t readonly_mfn,
379 unsigned int level,
380 unsigned long fault_addr);
382 /* Functions that atomically write PT/P2M entries and update state */
383 void shadow_write_p2m_entry(struct vcpu *v, unsigned long gfn,
384 l1_pgentry_t *p, mfn_t table_mfn,
385 l1_pgentry_t new, unsigned int level);
386 int shadow_write_guest_entry(struct vcpu *v, intpte_t *p,
387 intpte_t new, mfn_t gmfn);
388 int shadow_cmpxchg_guest_entry(struct vcpu *v, intpte_t *p,
389 intpte_t *old, intpte_t new, mfn_t gmfn);
393 /******************************************************************************
394 * Flags used in the return value of the shadow_set_lXe() functions...
395 */
397 /* We actually wrote something new to the shadow */
398 #define SHADOW_SET_CHANGED 0x1
399 /* Caller should flush TLBs to clear the old entry */
400 #define SHADOW_SET_FLUSH 0x2
401 /* Something went wrong: the shadow entry was invalid or refcount failed */
402 #define SHADOW_SET_ERROR 0x4
405 /******************************************************************************
406 * MFN/page-info handling
407 */
409 // Override mfn_to_page from asm/page.h, which was #include'd above,
410 // in order to make it work with our mfn type.
411 #undef mfn_to_page
412 #define mfn_to_page(_m) (frame_table + mfn_x(_m))
413 #define mfn_to_shadow_page(_m) ((struct shadow_page_info *)mfn_to_page(_m))
415 // Override page_to_mfn from asm/page.h, which was #include'd above,
416 // in order to make it work with our mfn type.
417 #undef page_to_mfn
418 #define page_to_mfn(_pg) (_mfn((_pg) - frame_table))
419 #define shadow_page_to_mfn(_spg) (page_to_mfn((struct page_info *)_spg))
421 // Override mfn_valid from asm/page.h, which was #include'd above,
422 // in order to make it work with our mfn type.
423 #undef mfn_valid
424 #define mfn_valid(_mfn) (mfn_x(_mfn) < max_page)
426 /* Override pagetable_t <-> struct page_info conversions to work with mfn_t */
427 #undef pagetable_get_page
428 #define pagetable_get_page(x) mfn_to_page(pagetable_get_mfn(x))
429 #undef pagetable_from_page
430 #define pagetable_from_page(pg) pagetable_from_mfn(page_to_mfn(pg))
432 static inline int
433 sh_mfn_is_a_page_table(mfn_t gmfn)
434 {
435 struct page_info *page = mfn_to_page(gmfn);
436 struct domain *owner;
437 unsigned long type_info;
439 if ( !mfn_valid(gmfn) )
440 return 0;
442 owner = page_get_owner(page);
443 if ( owner && shadow_mode_refcounts(owner)
444 && (page->count_info & PGC_page_table) )
445 return 1;
447 type_info = page->u.inuse.type_info & PGT_type_mask;
448 return type_info && (type_info <= PGT_l4_page_table);
449 }
451 // Provide mfn_t-aware versions of common xen functions
452 static inline void *
453 sh_map_domain_page(mfn_t mfn)
454 {
455 return map_domain_page(mfn_x(mfn));
456 }
458 static inline void
459 sh_unmap_domain_page(void *p)
460 {
461 unmap_domain_page(p);
462 }
464 static inline void *
465 sh_map_domain_page_global(mfn_t mfn)
466 {
467 return map_domain_page_global(mfn_x(mfn));
468 }
470 static inline void
471 sh_unmap_domain_page_global(void *p)
472 {
473 unmap_domain_page_global(p);
474 }
476 /******************************************************************************
477 * Log-dirty mode bitmap handling
478 */
480 extern void sh_mark_dirty(struct domain *d, mfn_t gmfn);
482 static inline int
483 sh_mfn_is_dirty(struct domain *d, mfn_t gmfn)
484 /* Is this guest page dirty? Call only in log-dirty mode. */
485 {
486 unsigned long pfn;
487 mfn_t mfn, *l4, *l3, *l2;
488 uint8_t *l1;
489 int rv;
491 ASSERT(shadow_mode_log_dirty(d));
492 ASSERT(mfn_valid(d->arch.paging.log_dirty.top));
494 /* We /really/ mean PFN here, even for non-translated guests. */
495 pfn = get_gpfn_from_mfn(mfn_x(gmfn));
496 if ( unlikely(!VALID_M2P(pfn)) )
497 return 0;
499 if ( d->arch.paging.log_dirty.failed_allocs > 0 )
500 /* If we have any failed allocations our dirty log is bogus.
501 * Since we can't signal an error here, be conservative and
502 * report "dirty" in this case. (The only current caller,
503 * _sh_propagate, leaves known-dirty pages writable, preventing
504 * subsequent dirty-logging faults from them.)
505 */
506 return 1;
508 l4 = map_domain_page(mfn_x(d->arch.paging.log_dirty.top));
509 mfn = l4[L4_LOGDIRTY_IDX(pfn)];
510 unmap_domain_page(l4);
511 if ( !mfn_valid(mfn) )
512 return 0;
514 l3 = map_domain_page(mfn_x(mfn));
515 mfn = l3[L3_LOGDIRTY_IDX(pfn)];
516 unmap_domain_page(l3);
517 if ( !mfn_valid(mfn) )
518 return 0;
520 l2 = map_domain_page(mfn_x(mfn));
521 mfn = l2[L2_LOGDIRTY_IDX(pfn)];
522 unmap_domain_page(l2);
523 if ( !mfn_valid(mfn) )
524 return 0;
526 l1 = map_domain_page(mfn_x(mfn));
527 rv = test_bit(L1_LOGDIRTY_IDX(pfn), l1);
528 unmap_domain_page(l1);
530 return rv;
531 }
534 /**************************************************************************/
535 /* Shadow-page refcounting. */
537 void sh_destroy_shadow(struct vcpu *v, mfn_t smfn);
539 /* Increase the refcount of a shadow page. Arguments are the mfn to refcount,
540 * and the physical address of the shadow entry that holds the ref (or zero
541 * if the ref is held by something else).
542 * Returns 0 for failure, 1 for success. */
543 static inline int sh_get_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
544 {
545 u32 x, nx;
546 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
548 ASSERT(mfn_valid(smfn));
550 x = sp->count;
551 nx = x + 1;
553 if ( unlikely(nx >= 1U<<26) )
554 {
555 SHADOW_PRINTK("shadow ref overflow, gmfn=%" PRtype_info " smfn=%lx\n",
556 sp->backpointer, mfn_x(smfn));
557 return 0;
558 }
560 /* Guarded by the shadow lock, so no need for atomic update */
561 sp->count = nx;
563 /* We remember the first shadow entry that points to each shadow. */
564 if ( entry_pa != 0
565 && !sh_type_is_pinnable(v, sp->type)
566 && sp->up == 0 )
567 sp->up = entry_pa;
569 return 1;
570 }
573 /* Decrease the refcount of a shadow page. As for get_ref, takes the
574 * physical address of the shadow entry that held this reference. */
575 static inline void sh_put_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
576 {
577 u32 x, nx;
578 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
580 ASSERT(mfn_valid(smfn));
581 ASSERT(sp->mbz == 0);
583 /* If this is the entry in the up-pointer, remove it */
584 if ( entry_pa != 0
585 && !sh_type_is_pinnable(v, sp->type)
586 && sp->up == entry_pa )
587 sp->up = 0;
589 x = sp->count;
590 nx = x - 1;
592 if ( unlikely(x == 0) )
593 {
594 SHADOW_ERROR("shadow ref underflow, smfn=%lx oc=%08x t=%#x\n",
595 mfn_x(smfn), sp->count, sp->type);
596 BUG();
597 }
599 /* Guarded by the shadow lock, so no need for atomic update */
600 sp->count = nx;
602 if ( unlikely(nx == 0) )
603 sh_destroy_shadow(v, smfn);
604 }
607 /* Pin a shadow page: take an extra refcount, set the pin bit,
608 * and put the shadow at the head of the list of pinned shadows.
609 * Returns 0 for failure, 1 for success. */
610 static inline int sh_pin(struct vcpu *v, mfn_t smfn)
611 {
612 struct shadow_page_info *sp;
614 ASSERT(mfn_valid(smfn));
615 sp = mfn_to_shadow_page(smfn);
616 ASSERT(sh_type_is_pinnable(v, sp->type));
617 if ( sp->pinned )
618 {
619 /* Already pinned: take it out of the pinned-list so it can go
620 * at the front */
621 list_del(&sp->list);
622 }
623 else
624 {
625 /* Not pinned: pin it! */
626 if ( !sh_get_ref(v, smfn, 0) )
627 return 0;
628 sp->pinned = 1;
629 }
630 /* Put it at the head of the list of pinned shadows */
631 list_add(&sp->list, &v->domain->arch.paging.shadow.pinned_shadows);
632 return 1;
633 }
635 /* Unpin a shadow page: unset the pin bit, take the shadow off the list
636 * of pinned shadows, and release the extra ref. */
637 static inline void sh_unpin(struct vcpu *v, mfn_t smfn)
638 {
639 struct shadow_page_info *sp;
641 ASSERT(mfn_valid(smfn));
642 sp = mfn_to_shadow_page(smfn);
643 ASSERT(sh_type_is_pinnable(v, sp->type));
644 if ( sp->pinned )
645 {
646 sp->pinned = 0;
647 list_del(&sp->list);
648 sp->up = 0; /* in case this stops being a pinnable type in future */
649 sh_put_ref(v, smfn, 0);
650 }
651 }
654 /**************************************************************************/
655 /* PTE-write emulation. */
657 struct sh_emulate_ctxt {
658 struct x86_emulate_ctxt ctxt;
660 /* Cache of up to 31 bytes of instruction. */
661 uint8_t insn_buf[31];
662 uint8_t insn_buf_bytes;
663 unsigned long insn_buf_eip;
665 /* Cache of segment registers already gathered for this emulation. */
666 unsigned int valid_seg_regs;
667 struct segment_register seg_reg[6];
669 /* MFNs being written to in write/cmpxchg callbacks */
670 mfn_t mfn1, mfn2;
672 #if (SHADOW_OPTIMIZATIONS & SHOPT_SKIP_VERIFY)
673 /* Special case for avoiding having to verify writes: remember
674 * whether the old value had its low bit (_PAGE_PRESENT) clear. */
675 int low_bit_was_clear:1;
676 #endif
677 };
679 struct x86_emulate_ops *shadow_init_emulation(
680 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
681 void shadow_continue_emulation(
682 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
685 #if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
686 /**************************************************************************/
687 /* Virtual TLB entries
688 *
689 * We keep a cache of virtual-to-physical translations that we have seen
690 * since the last TLB flush. This is safe to use for frame translations,
691 * but callers that use the rights need to re-check the actual guest tables
692 * before triggering a fault.
693 *
694 * Lookups and updates are protected by a per-vTLB (and hence per-vcpu)
695 * lock. This lock is held *only* while reading or writing the table,
696 * so it is safe to take in any non-interrupt context. Most lookups
697 * happen with v==current, so we expect contention to be low.
698 */
700 #define VTLB_ENTRIES 13
702 struct shadow_vtlb {
703 unsigned long page_number; /* Guest virtual address >> PAGE_SHIFT */
704 unsigned long frame_number; /* Guest physical address >> PAGE_SHIFT */
705 uint32_t pfec; /* Pagefault code for the lookup that filled this entry */
706 uint32_t flags; /* Accumulated guest pte flags, or 0 for an empty slot. */
707 };
709 /* Call whenever the guest flushes hit actual TLB */
710 static inline void vtlb_flush(struct vcpu *v)
711 {
712 spin_lock(&v->arch.paging.vtlb_lock);
713 memset(v->arch.paging.vtlb, 0, VTLB_ENTRIES * sizeof (struct shadow_vtlb));
714 spin_unlock(&v->arch.paging.vtlb_lock);
715 }
717 static inline int vtlb_hash(unsigned long page_number)
718 {
719 return page_number % VTLB_ENTRIES;
720 }
722 /* Put a translation into the vTLB, potentially clobbering an old one */
723 static inline void vtlb_insert(struct vcpu *v, struct shadow_vtlb entry)
724 {
725 spin_lock(&v->arch.paging.vtlb_lock);
726 v->arch.paging.vtlb[vtlb_hash(entry.page_number)] = entry;
727 spin_unlock(&v->arch.paging.vtlb_lock);
728 }
730 /* Look a translation up in the vTLB. Returns 0 if not found. */
731 static inline int vtlb_lookup(struct vcpu *v, unsigned long va, uint32_t pfec,
732 struct shadow_vtlb *result)
733 {
734 unsigned long page_number = va >> PAGE_SHIFT;
735 int rv = 0;
736 int i = vtlb_hash(page_number);
738 spin_lock(&v->arch.paging.vtlb_lock);
739 if ( v->arch.paging.vtlb[i].flags != 0
740 && v->arch.paging.vtlb[i].page_number == page_number
741 /* Any successful walk that had at least these pfec bits is OK */
742 && (v->arch.paging.vtlb[i].pfec & pfec) == pfec )
743 {
744 rv = 1;
745 result[0] = v->arch.paging.vtlb[i];
746 }
747 spin_unlock(&v->arch.paging.vtlb_lock);
748 return rv;
749 }
750 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
753 #endif /* _XEN_SHADOW_PRIVATE_H */
755 /*
756 * Local variables:
757 * mode: C
758 * c-set-style: "BSD"
759 * c-basic-offset: 4
760 * indent-tabs-mode: nil
761 * End:
762 */