ia64/xen-unstable

view xen/arch/x86/mm/shadow/private.h @ 15812:86a154e1ef5d

[HVM] Shadow: don't shadow the p2m table.
For HVM vcpus with paging disabled, we used to shadow the p2m table,
and skip the p2m lookup to go from gfn to mfn. Instead, we now
provide a simple pagetable that gives a one-to-one mapping of 4GB, and
shadow that, making the translations from gfn to mfn via the p2m.
This removes the paging-disabled special-case code from the shadow
fault handler, and allows us to expand the p2m interface, since all HVM
translations now go through the same p2m lookups.
Signed-off-by: Tim Deegan <Tim.Deegan@xensource.com>
author Tim Deegan <Tim.Deegan@xensource.com>
date Fri Aug 31 11:06:22 2007 +0100 (2007-08-31)
parents 9fd5becfba6b
children 19a843def5fd
line source
1 /******************************************************************************
2 * arch/x86/mm/shadow/private.h
3 *
4 * Shadow code that is private, and does not need to be multiply compiled.
5 * Parts of this code are Copyright (c) 2006 by XenSource Inc.
6 * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
7 * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 */
24 #ifndef _XEN_SHADOW_PRIVATE_H
25 #define _XEN_SHADOW_PRIVATE_H
27 // In order to override the definition of mfn_to_page, we make sure page.h has
28 // been included...
29 #include <asm/page.h>
30 #include <xen/domain_page.h>
31 #include <asm/x86_emulate.h>
32 #include <asm/hvm/support.h>
35 /******************************************************************************
36 * Levels of self-test and paranoia
37 */
39 #define SHADOW_AUDIT_HASH 0x01 /* Check current hash bucket */
40 #define SHADOW_AUDIT_HASH_FULL 0x02 /* Check every hash bucket */
41 #define SHADOW_AUDIT_ENTRIES 0x04 /* Check this walk's shadows */
42 #define SHADOW_AUDIT_ENTRIES_FULL 0x08 /* Check every shadow */
43 #define SHADOW_AUDIT_ENTRIES_MFNS 0x10 /* Check gfn-mfn map in shadows */
45 #ifdef NDEBUG
46 #define SHADOW_AUDIT 0
47 #define SHADOW_AUDIT_ENABLE 0
48 #else
49 #define SHADOW_AUDIT 0x15 /* Basic audit of all */
50 #define SHADOW_AUDIT_ENABLE shadow_audit_enable
51 extern int shadow_audit_enable;
52 #endif
54 /******************************************************************************
55 * Levels of optimization
56 */
58 #define SHOPT_WRITABLE_HEURISTIC 0x01 /* Guess at RW PTEs via linear maps */
59 #define SHOPT_EARLY_UNSHADOW 0x02 /* Unshadow l1s on fork or exit */
60 #define SHOPT_FAST_FAULT_PATH 0x04 /* Fast-path MMIO and not-present */
61 #define SHOPT_PREFETCH 0x08 /* Shadow multiple entries per fault */
62 #define SHOPT_LINUX_L3_TOPLEVEL 0x10 /* Pin l3es on early 64bit linux */
63 #define SHOPT_SKIP_VERIFY 0x20 /* Skip PTE v'fy when safe to do so */
64 #define SHOPT_VIRTUAL_TLB 0x40 /* Cache guest v->p translations */
66 #define SHADOW_OPTIMIZATIONS 0x7f
69 /******************************************************************************
70 * Debug and error-message output
71 */
73 #define SHADOW_PRINTK(_f, _a...) \
74 debugtrace_printk("sh: %s(): " _f, __func__, ##_a)
75 #define SHADOW_ERROR(_f, _a...) \
76 printk("sh error: %s(): " _f, __func__, ##_a)
77 #define SHADOW_DEBUG(flag, _f, _a...) \
78 do { \
79 if (SHADOW_DEBUG_ ## flag) \
80 debugtrace_printk("shdebug: %s(): " _f, __func__, ##_a); \
81 } while (0)
83 // The flags for use with SHADOW_DEBUG:
84 #define SHADOW_DEBUG_PROPAGATE 1
85 #define SHADOW_DEBUG_MAKE_SHADOW 1
86 #define SHADOW_DEBUG_DESTROY_SHADOW 1
87 #define SHADOW_DEBUG_A_AND_D 1
88 #define SHADOW_DEBUG_EMULATE 1
89 #define SHADOW_DEBUG_P2M 1
90 #define SHADOW_DEBUG_LOGDIRTY 0
92 /******************************************************************************
93 * The shadow lock.
94 *
95 * This lock is per-domain. It is intended to allow us to make atomic
96 * updates to the software TLB that the shadow tables provide.
97 *
98 * Specifically, it protects:
99 * - all changes to shadow page table pages
100 * - the shadow hash table
101 * - the shadow page allocator
102 * - all changes to guest page table pages
103 * - all changes to the page_info->tlbflush_timestamp
104 * - the page_info->count fields on shadow pages
105 * - the shadow dirty bit array and count
106 */
107 #ifndef CONFIG_SMP
108 #error shadow.h currently requires CONFIG_SMP
109 #endif
111 #define shadow_lock_init(_d) \
112 do { \
113 spin_lock_init(&(_d)->arch.paging.shadow.lock); \
114 (_d)->arch.paging.shadow.locker = -1; \
115 (_d)->arch.paging.shadow.locker_function = "nobody"; \
116 } while (0)
118 #define shadow_locked_by_me(_d) \
119 (current->processor == (_d)->arch.paging.shadow.locker)
121 #define shadow_lock(_d) \
122 do { \
123 if ( unlikely((_d)->arch.paging.shadow.locker == current->processor) )\
124 { \
125 printk("Error: shadow lock held by %s\n", \
126 (_d)->arch.paging.shadow.locker_function); \
127 BUG(); \
128 } \
129 spin_lock(&(_d)->arch.paging.shadow.lock); \
130 ASSERT((_d)->arch.paging.shadow.locker == -1); \
131 (_d)->arch.paging.shadow.locker = current->processor; \
132 (_d)->arch.paging.shadow.locker_function = __func__; \
133 } while (0)
135 #define shadow_unlock(_d) \
136 do { \
137 ASSERT((_d)->arch.paging.shadow.locker == current->processor); \
138 (_d)->arch.paging.shadow.locker = -1; \
139 (_d)->arch.paging.shadow.locker_function = "nobody"; \
140 spin_unlock(&(_d)->arch.paging.shadow.lock); \
141 } while (0)
145 /******************************************************************************
146 * Auditing routines
147 */
149 #if SHADOW_AUDIT & SHADOW_AUDIT_ENTRIES_FULL
150 extern void shadow_audit_tables(struct vcpu *v);
151 #else
152 #define shadow_audit_tables(_v) do {} while(0)
153 #endif
155 /******************************************************************************
156 * Macro for dealing with the naming of the internal names of the
157 * shadow code's external entry points.
158 */
159 #define SHADOW_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels) \
160 name ## __shadow_ ## shadow_levels ## _guest_ ## guest_levels
161 #define SHADOW_INTERNAL_NAME(name, shadow_levels, guest_levels) \
162 SHADOW_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels)
164 #if CONFIG_PAGING_LEVELS == 2
165 #define GUEST_LEVELS 2
166 #define SHADOW_LEVELS 2
167 #include "multi.h"
168 #undef GUEST_LEVELS
169 #undef SHADOW_LEVELS
170 #endif /* CONFIG_PAGING_LEVELS == 2 */
172 #if CONFIG_PAGING_LEVELS == 3
173 #define GUEST_LEVELS 2
174 #define SHADOW_LEVELS 3
175 #include "multi.h"
176 #undef GUEST_LEVELS
177 #undef SHADOW_LEVELS
179 #define GUEST_LEVELS 3
180 #define SHADOW_LEVELS 3
181 #include "multi.h"
182 #undef GUEST_LEVELS
183 #undef SHADOW_LEVELS
184 #endif /* CONFIG_PAGING_LEVELS == 3 */
186 #if CONFIG_PAGING_LEVELS == 4
187 #define GUEST_LEVELS 2
188 #define SHADOW_LEVELS 3
189 #include "multi.h"
190 #undef GUEST_LEVELS
191 #undef SHADOW_LEVELS
193 #define GUEST_LEVELS 3
194 #define SHADOW_LEVELS 3
195 #include "multi.h"
196 #undef GUEST_LEVELS
197 #undef SHADOW_LEVELS
199 #define GUEST_LEVELS 3
200 #define SHADOW_LEVELS 4
201 #include "multi.h"
202 #undef GUEST_LEVELS
203 #undef SHADOW_LEVELS
205 #define GUEST_LEVELS 4
206 #define SHADOW_LEVELS 4
207 #include "multi.h"
208 #undef GUEST_LEVELS
209 #undef SHADOW_LEVELS
210 #endif /* CONFIG_PAGING_LEVELS == 4 */
212 /******************************************************************************
213 * Page metadata for shadow pages.
214 */
216 struct shadow_page_info
217 {
218 union {
219 /* When in use, guest page we're a shadow of */
220 unsigned long backpointer;
221 /* When free, order of the freelist we're on */
222 unsigned int order;
223 };
224 union {
225 /* When in use, next shadow in this hash chain */
226 struct shadow_page_info *next_shadow;
227 /* When free, TLB flush time when freed */
228 u32 tlbflush_timestamp;
229 };
230 struct {
231 unsigned int type:4; /* What kind of shadow is this? */
232 unsigned int pinned:1; /* Is the shadow pinned? */
233 unsigned int count:27; /* Reference count */
234 u32 mbz; /* Must be zero: this is where the owner
235 * field lives in a non-shadow page */
236 } __attribute__((packed));
237 union {
238 /* For unused shadow pages, a list of pages of this order;
239 * for pinnable shadows, if pinned, a list of other pinned shadows
240 * (see sh_type_is_pinnable() below for the definition of
241 * "pinnable" shadow types). */
242 struct list_head list;
243 /* For non-pinnable shadows, a higher entry that points at us */
244 paddr_t up;
245 };
246 };
248 /* The structure above *must* be the same size as a struct page_info
249 * from mm.h, since we'll be using the same space in the frametable.
250 * Also, the mbz field must line up with the owner field of normal
251 * pages, so they look properly like anonymous/xen pages. */
252 static inline void shadow_check_page_struct_offsets(void) {
253 BUILD_BUG_ON(sizeof (struct shadow_page_info)
254 != sizeof (struct page_info));
255 BUILD_BUG_ON(offsetof(struct shadow_page_info, mbz)
256 != offsetof(struct page_info, u.inuse._domain));
257 };
259 /* Shadow type codes */
260 #define SH_type_none (0U) /* on the shadow free list */
261 #define SH_type_min_shadow (1U)
262 #define SH_type_l1_32_shadow (1U) /* shadowing a 32-bit L1 guest page */
263 #define SH_type_fl1_32_shadow (2U) /* L1 shadow for a 32b 4M superpage */
264 #define SH_type_l2_32_shadow (3U) /* shadowing a 32-bit L2 guest page */
265 #define SH_type_l1_pae_shadow (4U) /* shadowing a pae L1 page */
266 #define SH_type_fl1_pae_shadow (5U) /* L1 shadow for pae 2M superpg */
267 #define SH_type_l2_pae_shadow (6U) /* shadowing a pae L2-low page */
268 #define SH_type_l2h_pae_shadow (7U) /* shadowing a pae L2-high page */
269 #define SH_type_l1_64_shadow (8U) /* shadowing a 64-bit L1 page */
270 #define SH_type_fl1_64_shadow (9U) /* L1 shadow for 64-bit 2M superpg */
271 #define SH_type_l2_64_shadow (10U) /* shadowing a 64-bit L2 page */
272 #define SH_type_l2h_64_shadow (11U) /* shadowing a compat PAE L2 high page */
273 #define SH_type_l3_64_shadow (12U) /* shadowing a 64-bit L3 page */
274 #define SH_type_l4_64_shadow (13U) /* shadowing a 64-bit L4 page */
275 #define SH_type_max_shadow (13U)
276 #define SH_type_p2m_table (14U) /* in use as the p2m table */
277 #define SH_type_monitor_table (15U) /* in use as a monitor table */
278 #define SH_type_unused (16U)
280 /*
281 * What counts as a pinnable shadow?
282 */
284 static inline int sh_type_is_pinnable(struct vcpu *v, unsigned int t)
285 {
286 /* Top-level shadow types in each mode can be pinned, so that they
287 * persist even when not currently in use in a guest CR3 */
288 if ( t == SH_type_l2_32_shadow
289 || t == SH_type_l2_pae_shadow
290 || t == SH_type_l2h_pae_shadow
291 || t == SH_type_l4_64_shadow )
292 return 1;
294 #if (SHADOW_OPTIMIZATIONS & SHOPT_LINUX_L3_TOPLEVEL)
295 /* Early 64-bit linux used three levels of pagetables for the guest
296 * and context switched by changing one l4 entry in a per-cpu l4
297 * page. When we're shadowing those kernels, we have to pin l3
298 * shadows so they don't just evaporate on every context switch.
299 * For all other guests, we'd rather use the up-pointer field in l3s. */
300 if ( unlikely((v->domain->arch.paging.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL)
301 && CONFIG_PAGING_LEVELS >= 4
302 && t == SH_type_l3_64_shadow) )
303 return 1;
304 #endif
306 /* Everything else is not pinnable, and can use the "up" pointer */
307 return 0;
308 }
310 /*
311 * Definitions for the shadow_flags field in page_info.
312 * These flags are stored on *guest* pages...
313 * Bits 1-13 are encodings for the shadow types.
314 */
315 #define SHF_page_type_mask \
316 (((1u << (SH_type_max_shadow + 1u)) - 1u) - \
317 ((1u << SH_type_min_shadow) - 1u))
319 #define SHF_L1_32 (1u << SH_type_l1_32_shadow)
320 #define SHF_FL1_32 (1u << SH_type_fl1_32_shadow)
321 #define SHF_L2_32 (1u << SH_type_l2_32_shadow)
322 #define SHF_L1_PAE (1u << SH_type_l1_pae_shadow)
323 #define SHF_FL1_PAE (1u << SH_type_fl1_pae_shadow)
324 #define SHF_L2_PAE (1u << SH_type_l2_pae_shadow)
325 #define SHF_L2H_PAE (1u << SH_type_l2h_pae_shadow)
326 #define SHF_L1_64 (1u << SH_type_l1_64_shadow)
327 #define SHF_FL1_64 (1u << SH_type_fl1_64_shadow)
328 #define SHF_L2_64 (1u << SH_type_l2_64_shadow)
329 #define SHF_L2H_64 (1u << SH_type_l2h_64_shadow)
330 #define SHF_L3_64 (1u << SH_type_l3_64_shadow)
331 #define SHF_L4_64 (1u << SH_type_l4_64_shadow)
333 #define SHF_32 (SHF_L1_32|SHF_FL1_32|SHF_L2_32)
334 #define SHF_PAE (SHF_L1_PAE|SHF_FL1_PAE|SHF_L2_PAE|SHF_L2H_PAE)
335 #define SHF_64 (SHF_L1_64|SHF_FL1_64|SHF_L2_64|SHF_L2H_64|SHF_L3_64|SHF_L4_64)
337 /* Used for hysteresis when automatically unhooking mappings on fork/exit */
338 #define SHF_unhooked_mappings (1u<<31)
341 /******************************************************************************
342 * Various function declarations
343 */
345 /* Hash table functions */
346 mfn_t shadow_hash_lookup(struct vcpu *v, unsigned long n, unsigned int t);
347 void shadow_hash_insert(struct vcpu *v,
348 unsigned long n, unsigned int t, mfn_t smfn);
349 void shadow_hash_delete(struct vcpu *v,
350 unsigned long n, unsigned int t, mfn_t smfn);
352 /* shadow promotion */
353 void shadow_promote(struct vcpu *v, mfn_t gmfn, u32 type);
354 void shadow_demote(struct vcpu *v, mfn_t gmfn, u32 type);
356 /* Shadow page allocation functions */
357 void shadow_prealloc(struct domain *d, unsigned int order);
358 mfn_t shadow_alloc(struct domain *d,
359 u32 shadow_type,
360 unsigned long backpointer);
361 void shadow_free(struct domain *d, mfn_t smfn);
363 /* Dispatcher function: call the per-mode function that will unhook the
364 * non-Xen mappings in this top-level shadow mfn */
365 void shadow_unhook_mappings(struct vcpu *v, mfn_t smfn);
367 /* Install the xen mappings in various flavours of shadow */
368 void sh_install_xen_entries_in_l4(struct vcpu *v, mfn_t gl4mfn, mfn_t sl4mfn);
369 void sh_install_xen_entries_in_l2(struct vcpu *v, mfn_t gl2mfn, mfn_t sl2mfn);
371 /* Update the shadows in response to a pagetable write from Xen */
372 int sh_validate_guest_entry(struct vcpu *v, mfn_t gmfn, void *entry, u32 size);
374 /* Update the shadows in response to a pagetable write from a HVM guest */
375 void sh_validate_guest_pt_write(struct vcpu *v, mfn_t gmfn,
376 void *entry, u32 size);
378 /* Remove all writeable mappings of a guest frame from the shadows.
379 * Returns non-zero if we need to flush TLBs.
380 * level and fault_addr desribe how we found this to be a pagetable;
381 * level==0 means we have some other reason for revoking write access. */
382 extern int sh_remove_write_access(struct vcpu *v, mfn_t readonly_mfn,
383 unsigned int level,
384 unsigned long fault_addr);
386 /* Allocate/free functions for passing to the P2M code. */
387 struct page_info *shadow_alloc_p2m_page(struct domain *d);
388 void shadow_free_p2m_page(struct domain *d, struct page_info *pg);
390 /* Functions that atomically write PT/P2M entries and update state */
391 void shadow_write_p2m_entry(struct vcpu *v, unsigned long gfn,
392 l1_pgentry_t *p, mfn_t table_mfn,
393 l1_pgentry_t new, unsigned int level);
394 int shadow_write_guest_entry(struct vcpu *v, intpte_t *p,
395 intpte_t new, mfn_t gmfn);
396 int shadow_cmpxchg_guest_entry(struct vcpu *v, intpte_t *p,
397 intpte_t *old, intpte_t new, mfn_t gmfn);
401 /******************************************************************************
402 * Flags used in the return value of the shadow_set_lXe() functions...
403 */
405 /* We actually wrote something new to the shadow */
406 #define SHADOW_SET_CHANGED 0x1
407 /* Caller should flush TLBs to clear the old entry */
408 #define SHADOW_SET_FLUSH 0x2
409 /* Something went wrong: the shadow entry was invalid or refcount failed */
410 #define SHADOW_SET_ERROR 0x4
413 /******************************************************************************
414 * MFN/page-info handling
415 */
417 // Override mfn_to_page from asm/page.h, which was #include'd above,
418 // in order to make it work with our mfn type.
419 #undef mfn_to_page
420 #define mfn_to_page(_m) (frame_table + mfn_x(_m))
421 #define mfn_to_shadow_page(_m) ((struct shadow_page_info *)mfn_to_page(_m))
423 // Override page_to_mfn from asm/page.h, which was #include'd above,
424 // in order to make it work with our mfn type.
425 #undef page_to_mfn
426 #define page_to_mfn(_pg) (_mfn((_pg) - frame_table))
427 #define shadow_page_to_mfn(_spg) (page_to_mfn((struct page_info *)_spg))
429 // Override mfn_valid from asm/page.h, which was #include'd above,
430 // in order to make it work with our mfn type.
431 #undef mfn_valid
432 #define mfn_valid(_mfn) (mfn_x(_mfn) < max_page)
434 /* Override pagetable_t <-> struct page_info conversions to work with mfn_t */
435 #undef pagetable_get_page
436 #define pagetable_get_page(x) mfn_to_page(pagetable_get_mfn(x))
437 #undef pagetable_from_page
438 #define pagetable_from_page(pg) pagetable_from_mfn(page_to_mfn(pg))
441 #if GUEST_PAGING_LEVELS >= 3
442 # define is_lo_pte(_vaddr) (((_vaddr)&0x4)==0)
443 #else
444 # define is_lo_pte(_vaddr) (1)
445 #endif
447 static inline int
448 sh_mfn_is_a_page_table(mfn_t gmfn)
449 {
450 struct page_info *page = mfn_to_page(gmfn);
451 struct domain *owner;
452 unsigned long type_info;
454 if ( !mfn_valid(gmfn) )
455 return 0;
457 owner = page_get_owner(page);
458 if ( owner && shadow_mode_refcounts(owner)
459 && (page->count_info & PGC_page_table) )
460 return 1;
462 type_info = page->u.inuse.type_info & PGT_type_mask;
463 return type_info && (type_info <= PGT_l4_page_table);
464 }
466 // Provide mfn_t-aware versions of common xen functions
467 static inline void *
468 sh_map_domain_page(mfn_t mfn)
469 {
470 return map_domain_page(mfn_x(mfn));
471 }
473 static inline void
474 sh_unmap_domain_page(void *p)
475 {
476 unmap_domain_page(p);
477 }
479 static inline void *
480 sh_map_domain_page_global(mfn_t mfn)
481 {
482 return map_domain_page_global(mfn_x(mfn));
483 }
485 static inline void
486 sh_unmap_domain_page_global(void *p)
487 {
488 unmap_domain_page_global(p);
489 }
491 /******************************************************************************
492 * Log-dirty mode bitmap handling
493 */
495 extern void sh_mark_dirty(struct domain *d, mfn_t gmfn);
497 static inline int
498 sh_mfn_is_dirty(struct domain *d, mfn_t gmfn)
499 /* Is this guest page dirty? Call only in log-dirty mode. */
500 {
501 unsigned long pfn;
502 ASSERT(shadow_mode_log_dirty(d));
503 ASSERT(d->arch.paging.log_dirty.bitmap != NULL);
505 /* We /really/ mean PFN here, even for non-translated guests. */
506 pfn = get_gpfn_from_mfn(mfn_x(gmfn));
507 if ( likely(VALID_M2P(pfn))
508 && likely(pfn < d->arch.paging.log_dirty.bitmap_size)
509 && test_bit(pfn, d->arch.paging.log_dirty.bitmap) )
510 return 1;
512 return 0;
513 }
516 /**************************************************************************/
517 /* Shadow-page refcounting. */
519 void sh_destroy_shadow(struct vcpu *v, mfn_t smfn);
521 /* Increase the refcount of a shadow page. Arguments are the mfn to refcount,
522 * and the physical address of the shadow entry that holds the ref (or zero
523 * if the ref is held by something else).
524 * Returns 0 for failure, 1 for success. */
525 static inline int sh_get_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
526 {
527 u32 x, nx;
528 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
530 ASSERT(mfn_valid(smfn));
532 x = sp->count;
533 nx = x + 1;
535 if ( unlikely(nx >= 1U<<26) )
536 {
537 SHADOW_PRINTK("shadow ref overflow, gmfn=%" PRtype_info " smfn=%lx\n",
538 sp->backpointer, mfn_x(smfn));
539 return 0;
540 }
542 /* Guarded by the shadow lock, so no need for atomic update */
543 sp->count = nx;
545 /* We remember the first shadow entry that points to each shadow. */
546 if ( entry_pa != 0
547 && !sh_type_is_pinnable(v, sp->type)
548 && sp->up == 0 )
549 sp->up = entry_pa;
551 return 1;
552 }
555 /* Decrease the refcount of a shadow page. As for get_ref, takes the
556 * physical address of the shadow entry that held this reference. */
557 static inline void sh_put_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
558 {
559 u32 x, nx;
560 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
562 ASSERT(mfn_valid(smfn));
563 ASSERT(sp->mbz == 0);
565 /* If this is the entry in the up-pointer, remove it */
566 if ( entry_pa != 0
567 && !sh_type_is_pinnable(v, sp->type)
568 && sp->up == entry_pa )
569 sp->up = 0;
571 x = sp->count;
572 nx = x - 1;
574 if ( unlikely(x == 0) )
575 {
576 SHADOW_ERROR("shadow ref underflow, smfn=%lx oc=%08x t=%#x\n",
577 mfn_x(smfn), sp->count, sp->type);
578 BUG();
579 }
581 /* Guarded by the shadow lock, so no need for atomic update */
582 sp->count = nx;
584 if ( unlikely(nx == 0) )
585 sh_destroy_shadow(v, smfn);
586 }
589 /* Pin a shadow page: take an extra refcount, set the pin bit,
590 * and put the shadow at the head of the list of pinned shadows.
591 * Returns 0 for failure, 1 for success. */
592 static inline int sh_pin(struct vcpu *v, mfn_t smfn)
593 {
594 struct shadow_page_info *sp;
596 ASSERT(mfn_valid(smfn));
597 sp = mfn_to_shadow_page(smfn);
598 ASSERT(sh_type_is_pinnable(v, sp->type));
599 if ( sp->pinned )
600 {
601 /* Already pinned: take it out of the pinned-list so it can go
602 * at the front */
603 list_del(&sp->list);
604 }
605 else
606 {
607 /* Not pinned: pin it! */
608 if ( !sh_get_ref(v, smfn, 0) )
609 return 0;
610 sp->pinned = 1;
611 }
612 /* Put it at the head of the list of pinned shadows */
613 list_add(&sp->list, &v->domain->arch.paging.shadow.pinned_shadows);
614 return 1;
615 }
617 /* Unpin a shadow page: unset the pin bit, take the shadow off the list
618 * of pinned shadows, and release the extra ref. */
619 static inline void sh_unpin(struct vcpu *v, mfn_t smfn)
620 {
621 struct shadow_page_info *sp;
623 ASSERT(mfn_valid(smfn));
624 sp = mfn_to_shadow_page(smfn);
625 ASSERT(sh_type_is_pinnable(v, sp->type));
626 if ( sp->pinned )
627 {
628 sp->pinned = 0;
629 list_del(&sp->list);
630 sp->up = 0; /* in case this stops being a pinnable type in future */
631 sh_put_ref(v, smfn, 0);
632 }
633 }
636 /**************************************************************************/
637 /* PTE-write emulation. */
639 struct sh_emulate_ctxt {
640 struct x86_emulate_ctxt ctxt;
642 /* [HVM] Cache of up to 31 bytes of instruction. */
643 uint8_t insn_buf[31];
644 uint8_t insn_buf_bytes;
645 unsigned long insn_buf_eip;
647 /* [HVM] Cache of segment registers already gathered for this emulation. */
648 unsigned int valid_seg_regs;
649 struct segment_register seg_reg[6];
650 };
652 struct x86_emulate_ops *shadow_init_emulation(
653 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
654 void shadow_continue_emulation(
655 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
658 #if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
659 /**************************************************************************/
660 /* Virtual TLB entries
661 *
662 * We keep a cache of virtual-to-physical translations that we have seen
663 * since the last TLB flush. This is safe to use for frame translations,
664 * but callers that use the rights need to re-check the actual guest tables
665 * before triggering a fault.
666 *
667 * Lookups and updates are protected by a per-vTLB (and hence per-vcpu)
668 * lock. This lock is held *only* while reading or writing the table,
669 * so it is safe to take in any non-interrupt context. Most lookups
670 * happen with v==current, so we expect contention to be low.
671 */
673 #define VTLB_ENTRIES 13
675 struct shadow_vtlb {
676 unsigned long page_number; /* Guest virtual address >> PAGE_SHIFT */
677 unsigned long frame_number; /* Guest physical address >> PAGE_SHIFT */
678 u32 flags; /* Accumulated guest pte flags, or 0 for an empty slot. */
679 };
681 /* Call whenever the guest flushes hit actual TLB */
682 static inline void vtlb_flush(struct vcpu *v)
683 {
684 spin_lock(&v->arch.paging.vtlb_lock);
685 memset(v->arch.paging.vtlb, 0, VTLB_ENTRIES * sizeof (struct shadow_vtlb));
686 spin_unlock(&v->arch.paging.vtlb_lock);
687 }
689 static inline int vtlb_hash(unsigned long page_number)
690 {
691 return page_number % VTLB_ENTRIES;
692 }
694 /* Put a translation into the vTLB, potentially clobbering an old one */
695 static inline void vtlb_insert(struct vcpu *v, struct shadow_vtlb entry)
696 {
697 spin_lock(&v->arch.paging.vtlb_lock);
698 v->arch.paging.vtlb[vtlb_hash(entry.page_number)] = entry;
699 spin_unlock(&v->arch.paging.vtlb_lock);
700 }
702 /* Look a translation up in the vTLB. Returns 0 if not found. */
703 static inline int vtlb_lookup(struct vcpu *v, unsigned long va,
704 struct shadow_vtlb *result)
705 {
706 unsigned long page_number = va >> PAGE_SHIFT;
707 int rv = 0;
708 int i = vtlb_hash(page_number);
710 spin_lock(&v->arch.paging.vtlb_lock);
711 if ( v->arch.paging.vtlb[i].flags != 0
712 && v->arch.paging.vtlb[i].page_number == page_number )
713 {
714 rv = 1;
715 result[0] = v->arch.paging.vtlb[i];
716 }
717 spin_unlock(&v->arch.paging.vtlb_lock);
718 return rv;
719 }
720 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
723 #endif /* _XEN_SHADOW_PRIVATE_H */
725 /*
726 * Local variables:
727 * mode: C
728 * c-set-style: "BSD"
729 * c-basic-offset: 4
730 * indent-tabs-mode: nil
731 * End:
732 */