ia64/xen-unstable

view xen/arch/x86/mm/shadow/private.h @ 17571:b6aa55ca599e

shadow: track video RAM dirty bits

This adds a new HVM op that enables tracking dirty bits of a range of
video RAM. The idea is to optimize just for the most common case
(only one guest mapping, with sometimes some temporary other
mappings), which permits to keep the overhead on shadow as low as
possible.

Signed-off-by: Samuel Thibault <samuel.thibault@eu.citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri May 02 15:08:27 2008 +0100 (2008-05-02)
parents af33f2054f47
children 0ac957f9d42e
line source
1 /******************************************************************************
2 * arch/x86/mm/shadow/private.h
3 *
4 * Shadow code that is private, and does not need to be multiply compiled.
5 * Parts of this code are Copyright (c) 2006 by XenSource Inc.
6 * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
7 * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 */
24 #ifndef _XEN_SHADOW_PRIVATE_H
25 #define _XEN_SHADOW_PRIVATE_H
27 // In order to override the definition of mfn_to_page, we make sure page.h has
28 // been included...
29 #include <asm/page.h>
30 #include <xen/domain_page.h>
31 #include <asm/x86_emulate.h>
32 #include <asm/hvm/support.h>
35 /******************************************************************************
36 * Levels of self-test and paranoia
37 */
39 #define SHADOW_AUDIT_HASH 0x01 /* Check current hash bucket */
40 #define SHADOW_AUDIT_HASH_FULL 0x02 /* Check every hash bucket */
41 #define SHADOW_AUDIT_ENTRIES 0x04 /* Check this walk's shadows */
42 #define SHADOW_AUDIT_ENTRIES_FULL 0x08 /* Check every shadow */
43 #define SHADOW_AUDIT_ENTRIES_MFNS 0x10 /* Check gfn-mfn map in shadows */
45 #ifdef NDEBUG
46 #define SHADOW_AUDIT 0
47 #define SHADOW_AUDIT_ENABLE 0
48 #else
49 #define SHADOW_AUDIT 0x15 /* Basic audit of all */
50 #define SHADOW_AUDIT_ENABLE shadow_audit_enable
51 extern int shadow_audit_enable;
52 #endif
54 /******************************************************************************
55 * Levels of optimization
56 */
58 #define SHOPT_WRITABLE_HEURISTIC 0x01 /* Guess at RW PTEs via linear maps */
59 #define SHOPT_EARLY_UNSHADOW 0x02 /* Unshadow l1s on fork or exit */
60 #define SHOPT_FAST_FAULT_PATH 0x04 /* Fast-path MMIO and not-present */
61 #define SHOPT_PREFETCH 0x08 /* Shadow multiple entries per fault */
62 #define SHOPT_LINUX_L3_TOPLEVEL 0x10 /* Pin l3es on early 64bit linux */
63 #define SHOPT_SKIP_VERIFY 0x20 /* Skip PTE v'fy when safe to do so */
64 #define SHOPT_VIRTUAL_TLB 0x40 /* Cache guest v->p translations */
65 #define SHOPT_FAST_EMULATION 0x80 /* Fast write emulation */
67 #define SHADOW_OPTIMIZATIONS 0xff
70 /******************************************************************************
71 * Debug and error-message output
72 */
74 #define SHADOW_PRINTK(_f, _a...) \
75 debugtrace_printk("sh: %s(): " _f, __func__, ##_a)
76 #define SHADOW_ERROR(_f, _a...) \
77 printk("sh error: %s(): " _f, __func__, ##_a)
78 #define SHADOW_DEBUG(flag, _f, _a...) \
79 do { \
80 if (SHADOW_DEBUG_ ## flag) \
81 debugtrace_printk("shdebug: %s(): " _f, __func__, ##_a); \
82 } while (0)
84 // The flags for use with SHADOW_DEBUG:
85 #define SHADOW_DEBUG_PROPAGATE 1
86 #define SHADOW_DEBUG_MAKE_SHADOW 1
87 #define SHADOW_DEBUG_DESTROY_SHADOW 1
88 #define SHADOW_DEBUG_A_AND_D 1
89 #define SHADOW_DEBUG_EMULATE 1
90 #define SHADOW_DEBUG_P2M 1
91 #define SHADOW_DEBUG_LOGDIRTY 0
93 /******************************************************************************
94 * The shadow lock.
95 *
96 * This lock is per-domain. It is intended to allow us to make atomic
97 * updates to the software TLB that the shadow tables provide.
98 *
99 * Specifically, it protects:
100 * - all changes to shadow page table pages
101 * - the shadow hash table
102 * - the shadow page allocator
103 * - all changes to guest page table pages
104 * - all changes to the page_info->tlbflush_timestamp
105 * - the page_info->count fields on shadow pages
106 * - the shadow dirty bit array and count
107 */
108 #ifndef CONFIG_SMP
109 #error shadow.h currently requires CONFIG_SMP
110 #endif
112 #define shadow_lock_init(_d) \
113 do { \
114 spin_lock_init(&(_d)->arch.paging.shadow.lock); \
115 (_d)->arch.paging.shadow.locker = -1; \
116 (_d)->arch.paging.shadow.locker_function = "nobody"; \
117 } while (0)
119 #define shadow_locked_by_me(_d) \
120 (current->processor == (_d)->arch.paging.shadow.locker)
122 #define shadow_lock(_d) \
123 do { \
124 if ( unlikely((_d)->arch.paging.shadow.locker == current->processor) )\
125 { \
126 printk("Error: shadow lock held by %s\n", \
127 (_d)->arch.paging.shadow.locker_function); \
128 BUG(); \
129 } \
130 spin_lock(&(_d)->arch.paging.shadow.lock); \
131 ASSERT((_d)->arch.paging.shadow.locker == -1); \
132 (_d)->arch.paging.shadow.locker = current->processor; \
133 (_d)->arch.paging.shadow.locker_function = __func__; \
134 } while (0)
136 #define shadow_unlock(_d) \
137 do { \
138 ASSERT((_d)->arch.paging.shadow.locker == current->processor); \
139 (_d)->arch.paging.shadow.locker = -1; \
140 (_d)->arch.paging.shadow.locker_function = "nobody"; \
141 spin_unlock(&(_d)->arch.paging.shadow.lock); \
142 } while (0)
146 /******************************************************************************
147 * Auditing routines
148 */
150 #if SHADOW_AUDIT & SHADOW_AUDIT_ENTRIES_FULL
151 extern void shadow_audit_tables(struct vcpu *v);
152 #else
153 #define shadow_audit_tables(_v) do {} while(0)
154 #endif
156 /******************************************************************************
157 * Macro for dealing with the naming of the internal names of the
158 * shadow code's external entry points.
159 */
160 #define SHADOW_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels) \
161 name ## __shadow_ ## shadow_levels ## _guest_ ## guest_levels
162 #define SHADOW_INTERNAL_NAME(name, shadow_levels, guest_levels) \
163 SHADOW_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels)
165 #if CONFIG_PAGING_LEVELS == 2
166 #define GUEST_LEVELS 2
167 #define SHADOW_LEVELS 2
168 #include "multi.h"
169 #undef GUEST_LEVELS
170 #undef SHADOW_LEVELS
171 #endif /* CONFIG_PAGING_LEVELS == 2 */
173 #if CONFIG_PAGING_LEVELS == 3
174 #define GUEST_LEVELS 2
175 #define SHADOW_LEVELS 3
176 #include "multi.h"
177 #undef GUEST_LEVELS
178 #undef SHADOW_LEVELS
180 #define GUEST_LEVELS 3
181 #define SHADOW_LEVELS 3
182 #include "multi.h"
183 #undef GUEST_LEVELS
184 #undef SHADOW_LEVELS
185 #endif /* CONFIG_PAGING_LEVELS == 3 */
187 #if CONFIG_PAGING_LEVELS == 4
188 #define GUEST_LEVELS 2
189 #define SHADOW_LEVELS 3
190 #include "multi.h"
191 #undef GUEST_LEVELS
192 #undef SHADOW_LEVELS
194 #define GUEST_LEVELS 3
195 #define SHADOW_LEVELS 3
196 #include "multi.h"
197 #undef GUEST_LEVELS
198 #undef SHADOW_LEVELS
200 #define GUEST_LEVELS 3
201 #define SHADOW_LEVELS 4
202 #include "multi.h"
203 #undef GUEST_LEVELS
204 #undef SHADOW_LEVELS
206 #define GUEST_LEVELS 4
207 #define SHADOW_LEVELS 4
208 #include "multi.h"
209 #undef GUEST_LEVELS
210 #undef SHADOW_LEVELS
211 #endif /* CONFIG_PAGING_LEVELS == 4 */
213 /******************************************************************************
214 * Page metadata for shadow pages.
215 */
217 struct shadow_page_info
218 {
219 union {
220 /* When in use, guest page we're a shadow of */
221 unsigned long backpointer;
222 /* When free, order of the freelist we're on */
223 unsigned int order;
224 };
225 union {
226 /* When in use, next shadow in this hash chain */
227 struct shadow_page_info *next_shadow;
228 /* When free, TLB flush time when freed */
229 u32 tlbflush_timestamp;
230 };
231 struct {
232 unsigned int type:4; /* What kind of shadow is this? */
233 unsigned int pinned:1; /* Is the shadow pinned? */
234 unsigned int count:27; /* Reference count */
235 u32 mbz; /* Must be zero: this is where the owner
236 * field lives in a non-shadow page */
237 } __attribute__((packed));
238 union {
239 /* For unused shadow pages, a list of pages of this order;
240 * for pinnable shadows, if pinned, a list of other pinned shadows
241 * (see sh_type_is_pinnable() below for the definition of
242 * "pinnable" shadow types). */
243 struct list_head list;
244 /* For non-pinnable shadows, a higher entry that points at us */
245 paddr_t up;
246 };
247 };
249 /* The structure above *must* be the same size as a struct page_info
250 * from mm.h, since we'll be using the same space in the frametable.
251 * Also, the mbz field must line up with the owner field of normal
252 * pages, so they look properly like anonymous/xen pages. */
253 static inline void shadow_check_page_struct_offsets(void) {
254 BUILD_BUG_ON(sizeof (struct shadow_page_info)
255 != sizeof (struct page_info));
256 BUILD_BUG_ON(offsetof(struct shadow_page_info, mbz)
257 != offsetof(struct page_info, u.inuse._domain));
258 };
260 /* Shadow type codes */
261 #define SH_type_none (0U) /* on the shadow free list */
262 #define SH_type_min_shadow (1U)
263 #define SH_type_l1_32_shadow (1U) /* shadowing a 32-bit L1 guest page */
264 #define SH_type_fl1_32_shadow (2U) /* L1 shadow for a 32b 4M superpage */
265 #define SH_type_l2_32_shadow (3U) /* shadowing a 32-bit L2 guest page */
266 #define SH_type_l1_pae_shadow (4U) /* shadowing a pae L1 page */
267 #define SH_type_fl1_pae_shadow (5U) /* L1 shadow for pae 2M superpg */
268 #define SH_type_l2_pae_shadow (6U) /* shadowing a pae L2-low page */
269 #define SH_type_l2h_pae_shadow (7U) /* shadowing a pae L2-high page */
270 #define SH_type_l1_64_shadow (8U) /* shadowing a 64-bit L1 page */
271 #define SH_type_fl1_64_shadow (9U) /* L1 shadow for 64-bit 2M superpg */
272 #define SH_type_l2_64_shadow (10U) /* shadowing a 64-bit L2 page */
273 #define SH_type_l2h_64_shadow (11U) /* shadowing a compat PAE L2 high page */
274 #define SH_type_l3_64_shadow (12U) /* shadowing a 64-bit L3 page */
275 #define SH_type_l4_64_shadow (13U) /* shadowing a 64-bit L4 page */
276 #define SH_type_max_shadow (13U)
277 #define SH_type_p2m_table (14U) /* in use as the p2m table */
278 #define SH_type_monitor_table (15U) /* in use as a monitor table */
279 #define SH_type_unused (16U)
281 /*
282 * What counts as a pinnable shadow?
283 */
285 static inline int sh_type_is_pinnable(struct vcpu *v, unsigned int t)
286 {
287 /* Top-level shadow types in each mode can be pinned, so that they
288 * persist even when not currently in use in a guest CR3 */
289 if ( t == SH_type_l2_32_shadow
290 || t == SH_type_l2_pae_shadow
291 || t == SH_type_l2h_pae_shadow
292 || t == SH_type_l4_64_shadow )
293 return 1;
295 #if (SHADOW_OPTIMIZATIONS & SHOPT_LINUX_L3_TOPLEVEL)
296 /* Early 64-bit linux used three levels of pagetables for the guest
297 * and context switched by changing one l4 entry in a per-cpu l4
298 * page. When we're shadowing those kernels, we have to pin l3
299 * shadows so they don't just evaporate on every context switch.
300 * For all other guests, we'd rather use the up-pointer field in l3s. */
301 if ( unlikely((v->domain->arch.paging.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL)
302 && CONFIG_PAGING_LEVELS >= 4
303 && t == SH_type_l3_64_shadow) )
304 return 1;
305 #endif
307 /* Everything else is not pinnable, and can use the "up" pointer */
308 return 0;
309 }
311 /*
312 * Definitions for the shadow_flags field in page_info.
313 * These flags are stored on *guest* pages...
314 * Bits 1-13 are encodings for the shadow types.
315 */
316 #define SHF_page_type_mask \
317 (((1u << (SH_type_max_shadow + 1u)) - 1u) - \
318 ((1u << SH_type_min_shadow) - 1u))
320 #define SHF_L1_32 (1u << SH_type_l1_32_shadow)
321 #define SHF_FL1_32 (1u << SH_type_fl1_32_shadow)
322 #define SHF_L2_32 (1u << SH_type_l2_32_shadow)
323 #define SHF_L1_PAE (1u << SH_type_l1_pae_shadow)
324 #define SHF_FL1_PAE (1u << SH_type_fl1_pae_shadow)
325 #define SHF_L2_PAE (1u << SH_type_l2_pae_shadow)
326 #define SHF_L2H_PAE (1u << SH_type_l2h_pae_shadow)
327 #define SHF_L1_64 (1u << SH_type_l1_64_shadow)
328 #define SHF_FL1_64 (1u << SH_type_fl1_64_shadow)
329 #define SHF_L2_64 (1u << SH_type_l2_64_shadow)
330 #define SHF_L2H_64 (1u << SH_type_l2h_64_shadow)
331 #define SHF_L3_64 (1u << SH_type_l3_64_shadow)
332 #define SHF_L4_64 (1u << SH_type_l4_64_shadow)
334 #define SHF_32 (SHF_L1_32|SHF_FL1_32|SHF_L2_32)
335 #define SHF_PAE (SHF_L1_PAE|SHF_FL1_PAE|SHF_L2_PAE|SHF_L2H_PAE)
336 #define SHF_64 (SHF_L1_64|SHF_FL1_64|SHF_L2_64|SHF_L2H_64|SHF_L3_64|SHF_L4_64)
339 /******************************************************************************
340 * Various function declarations
341 */
343 /* Hash table functions */
344 mfn_t shadow_hash_lookup(struct vcpu *v, unsigned long n, unsigned int t);
345 void shadow_hash_insert(struct vcpu *v,
346 unsigned long n, unsigned int t, mfn_t smfn);
347 void shadow_hash_delete(struct vcpu *v,
348 unsigned long n, unsigned int t, mfn_t smfn);
350 /* shadow promotion */
351 void shadow_promote(struct vcpu *v, mfn_t gmfn, u32 type);
352 void shadow_demote(struct vcpu *v, mfn_t gmfn, u32 type);
354 /* Shadow page allocation functions */
355 void shadow_prealloc(struct domain *d, u32 shadow_type, unsigned int count);
356 mfn_t shadow_alloc(struct domain *d,
357 u32 shadow_type,
358 unsigned long backpointer);
359 void shadow_free(struct domain *d, mfn_t smfn);
361 /* Install the xen mappings in various flavours of shadow */
362 void sh_install_xen_entries_in_l4(struct vcpu *v, mfn_t gl4mfn, mfn_t sl4mfn);
363 void sh_install_xen_entries_in_l2(struct vcpu *v, mfn_t gl2mfn, mfn_t sl2mfn);
365 /* Update the shadows in response to a pagetable write from Xen */
366 int sh_validate_guest_entry(struct vcpu *v, mfn_t gmfn, void *entry, u32 size);
368 /* Update the shadows in response to a pagetable write from a HVM guest */
369 void sh_validate_guest_pt_write(struct vcpu *v, mfn_t gmfn,
370 void *entry, u32 size);
372 /* Remove all writeable mappings of a guest frame from the shadows.
373 * Returns non-zero if we need to flush TLBs.
374 * level and fault_addr desribe how we found this to be a pagetable;
375 * level==0 means we have some other reason for revoking write access. */
376 extern int sh_remove_write_access(struct vcpu *v, mfn_t readonly_mfn,
377 unsigned int level,
378 unsigned long fault_addr);
380 /* Functions that atomically write PT/P2M entries and update state */
381 void shadow_write_p2m_entry(struct vcpu *v, unsigned long gfn,
382 l1_pgentry_t *p, mfn_t table_mfn,
383 l1_pgentry_t new, unsigned int level);
384 int shadow_write_guest_entry(struct vcpu *v, intpte_t *p,
385 intpte_t new, mfn_t gmfn);
386 int shadow_cmpxchg_guest_entry(struct vcpu *v, intpte_t *p,
387 intpte_t *old, intpte_t new, mfn_t gmfn);
391 /******************************************************************************
392 * Flags used in the return value of the shadow_set_lXe() functions...
393 */
395 /* We actually wrote something new to the shadow */
396 #define SHADOW_SET_CHANGED 0x1
397 /* Caller should flush TLBs to clear the old entry */
398 #define SHADOW_SET_FLUSH 0x2
399 /* Something went wrong: the shadow entry was invalid or refcount failed */
400 #define SHADOW_SET_ERROR 0x4
403 /******************************************************************************
404 * MFN/page-info handling
405 */
407 // Override mfn_to_page from asm/page.h, which was #include'd above,
408 // in order to make it work with our mfn type.
409 #undef mfn_to_page
410 #define mfn_to_page(_m) (frame_table + mfn_x(_m))
411 #define mfn_to_shadow_page(_m) ((struct shadow_page_info *)mfn_to_page(_m))
413 // Override page_to_mfn from asm/page.h, which was #include'd above,
414 // in order to make it work with our mfn type.
415 #undef page_to_mfn
416 #define page_to_mfn(_pg) (_mfn((_pg) - frame_table))
417 #define shadow_page_to_mfn(_spg) (page_to_mfn((struct page_info *)_spg))
419 // Override mfn_valid from asm/page.h, which was #include'd above,
420 // in order to make it work with our mfn type.
421 #undef mfn_valid
422 #define mfn_valid(_mfn) (mfn_x(_mfn) < max_page)
424 /* Override pagetable_t <-> struct page_info conversions to work with mfn_t */
425 #undef pagetable_get_page
426 #define pagetable_get_page(x) mfn_to_page(pagetable_get_mfn(x))
427 #undef pagetable_from_page
428 #define pagetable_from_page(pg) pagetable_from_mfn(page_to_mfn(pg))
430 static inline int
431 sh_mfn_is_a_page_table(mfn_t gmfn)
432 {
433 struct page_info *page = mfn_to_page(gmfn);
434 struct domain *owner;
435 unsigned long type_info;
437 if ( !mfn_valid(gmfn) )
438 return 0;
440 owner = page_get_owner(page);
441 if ( owner && shadow_mode_refcounts(owner)
442 && (page->count_info & PGC_page_table) )
443 return 1;
445 type_info = page->u.inuse.type_info & PGT_type_mask;
446 return type_info && (type_info <= PGT_l4_page_table);
447 }
449 // Provide mfn_t-aware versions of common xen functions
450 static inline void *
451 sh_map_domain_page(mfn_t mfn)
452 {
453 return map_domain_page(mfn_x(mfn));
454 }
456 static inline void
457 sh_unmap_domain_page(void *p)
458 {
459 unmap_domain_page(p);
460 }
462 static inline void *
463 sh_map_domain_page_global(mfn_t mfn)
464 {
465 return map_domain_page_global(mfn_x(mfn));
466 }
468 static inline void
469 sh_unmap_domain_page_global(void *p)
470 {
471 unmap_domain_page_global(p);
472 }
474 /******************************************************************************
475 * Log-dirty mode bitmap handling
476 */
478 extern void sh_mark_dirty(struct domain *d, mfn_t gmfn);
480 static inline int
481 sh_mfn_is_dirty(struct domain *d, mfn_t gmfn)
482 /* Is this guest page dirty? Call only in log-dirty mode. */
483 {
484 unsigned long pfn;
485 mfn_t mfn, *l4, *l3, *l2;
486 unsigned long *l1;
487 int rv;
489 ASSERT(shadow_mode_log_dirty(d));
490 ASSERT(mfn_valid(d->arch.paging.log_dirty.top));
492 /* We /really/ mean PFN here, even for non-translated guests. */
493 pfn = get_gpfn_from_mfn(mfn_x(gmfn));
494 if ( unlikely(!VALID_M2P(pfn)) )
495 return 0;
497 if ( d->arch.paging.log_dirty.failed_allocs > 0 )
498 /* If we have any failed allocations our dirty log is bogus.
499 * Since we can't signal an error here, be conservative and
500 * report "dirty" in this case. (The only current caller,
501 * _sh_propagate, leaves known-dirty pages writable, preventing
502 * subsequent dirty-logging faults from them.)
503 */
504 return 1;
506 l4 = map_domain_page(mfn_x(d->arch.paging.log_dirty.top));
507 mfn = l4[L4_LOGDIRTY_IDX(pfn)];
508 unmap_domain_page(l4);
509 if ( !mfn_valid(mfn) )
510 return 0;
512 l3 = map_domain_page(mfn_x(mfn));
513 mfn = l3[L3_LOGDIRTY_IDX(pfn)];
514 unmap_domain_page(l3);
515 if ( !mfn_valid(mfn) )
516 return 0;
518 l2 = map_domain_page(mfn_x(mfn));
519 mfn = l2[L2_LOGDIRTY_IDX(pfn)];
520 unmap_domain_page(l2);
521 if ( !mfn_valid(mfn) )
522 return 0;
524 l1 = map_domain_page(mfn_x(mfn));
525 rv = test_bit(L1_LOGDIRTY_IDX(pfn), l1);
526 unmap_domain_page(l1);
528 return rv;
529 }
531 /**************************************************************************/
532 /* VRAM dirty tracking support */
534 struct sh_dirty_vram {
535 unsigned long begin_pfn;
536 unsigned long end_pfn;
537 paddr_t *sl1ma;
538 uint8_t *dirty_bitmap;
539 };
541 /**************************************************************************/
542 /* Shadow-page refcounting. */
544 void sh_destroy_shadow(struct vcpu *v, mfn_t smfn);
546 /* Increase the refcount of a shadow page. Arguments are the mfn to refcount,
547 * and the physical address of the shadow entry that holds the ref (or zero
548 * if the ref is held by something else).
549 * Returns 0 for failure, 1 for success. */
550 static inline int sh_get_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
551 {
552 u32 x, nx;
553 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
555 ASSERT(mfn_valid(smfn));
557 x = sp->count;
558 nx = x + 1;
560 if ( unlikely(nx >= 1U<<26) )
561 {
562 SHADOW_PRINTK("shadow ref overflow, gmfn=%" PRtype_info " smfn=%lx\n",
563 sp->backpointer, mfn_x(smfn));
564 return 0;
565 }
567 /* Guarded by the shadow lock, so no need for atomic update */
568 sp->count = nx;
570 /* We remember the first shadow entry that points to each shadow. */
571 if ( entry_pa != 0
572 && !sh_type_is_pinnable(v, sp->type)
573 && sp->up == 0 )
574 sp->up = entry_pa;
576 return 1;
577 }
580 /* Decrease the refcount of a shadow page. As for get_ref, takes the
581 * physical address of the shadow entry that held this reference. */
582 static inline void sh_put_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
583 {
584 u32 x, nx;
585 struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
587 ASSERT(mfn_valid(smfn));
588 ASSERT(sp->mbz == 0);
590 /* If this is the entry in the up-pointer, remove it */
591 if ( entry_pa != 0
592 && !sh_type_is_pinnable(v, sp->type)
593 && sp->up == entry_pa )
594 sp->up = 0;
596 x = sp->count;
597 nx = x - 1;
599 if ( unlikely(x == 0) )
600 {
601 SHADOW_ERROR("shadow ref underflow, smfn=%lx oc=%08x t=%#x\n",
602 mfn_x(smfn), sp->count, sp->type);
603 BUG();
604 }
606 /* Guarded by the shadow lock, so no need for atomic update */
607 sp->count = nx;
609 if ( unlikely(nx == 0) )
610 sh_destroy_shadow(v, smfn);
611 }
614 /* Pin a shadow page: take an extra refcount, set the pin bit,
615 * and put the shadow at the head of the list of pinned shadows.
616 * Returns 0 for failure, 1 for success. */
617 static inline int sh_pin(struct vcpu *v, mfn_t smfn)
618 {
619 struct shadow_page_info *sp;
621 ASSERT(mfn_valid(smfn));
622 sp = mfn_to_shadow_page(smfn);
623 ASSERT(sh_type_is_pinnable(v, sp->type));
624 if ( sp->pinned )
625 {
626 /* Already pinned: take it out of the pinned-list so it can go
627 * at the front */
628 list_del(&sp->list);
629 }
630 else
631 {
632 /* Not pinned: pin it! */
633 if ( !sh_get_ref(v, smfn, 0) )
634 return 0;
635 sp->pinned = 1;
636 }
637 /* Put it at the head of the list of pinned shadows */
638 list_add(&sp->list, &v->domain->arch.paging.shadow.pinned_shadows);
639 return 1;
640 }
642 /* Unpin a shadow page: unset the pin bit, take the shadow off the list
643 * of pinned shadows, and release the extra ref. */
644 static inline void sh_unpin(struct vcpu *v, mfn_t smfn)
645 {
646 struct shadow_page_info *sp;
648 ASSERT(mfn_valid(smfn));
649 sp = mfn_to_shadow_page(smfn);
650 ASSERT(sh_type_is_pinnable(v, sp->type));
651 if ( sp->pinned )
652 {
653 sp->pinned = 0;
654 list_del(&sp->list);
655 sp->up = 0; /* in case this stops being a pinnable type in future */
656 sh_put_ref(v, smfn, 0);
657 }
658 }
661 /**************************************************************************/
662 /* PTE-write emulation. */
664 struct sh_emulate_ctxt {
665 struct x86_emulate_ctxt ctxt;
667 /* Cache of up to 31 bytes of instruction. */
668 uint8_t insn_buf[31];
669 uint8_t insn_buf_bytes;
670 unsigned long insn_buf_eip;
672 /* Cache of segment registers already gathered for this emulation. */
673 unsigned int valid_seg_regs;
674 struct segment_register seg_reg[6];
676 /* MFNs being written to in write/cmpxchg callbacks */
677 mfn_t mfn1, mfn2;
679 #if (SHADOW_OPTIMIZATIONS & SHOPT_SKIP_VERIFY)
680 /* Special case for avoiding having to verify writes: remember
681 * whether the old value had its low bit (_PAGE_PRESENT) clear. */
682 int low_bit_was_clear:1;
683 #endif
684 };
686 struct x86_emulate_ops *shadow_init_emulation(
687 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
688 void shadow_continue_emulation(
689 struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
690 struct segment_register *hvm_get_seg_reg(
691 enum x86_segment seg, struct sh_emulate_ctxt *sh_ctxt);
693 #if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
694 /**************************************************************************/
695 /* Virtual TLB entries
696 *
697 * We keep a cache of virtual-to-physical translations that we have seen
698 * since the last TLB flush. This is safe to use for frame translations,
699 * but callers need to re-check the actual guest tables if the lookup fails.
700 *
701 * Lookups and updates are protected by a per-vTLB (and hence per-vcpu)
702 * lock. This lock is held *only* while reading or writing the table,
703 * so it is safe to take in any non-interrupt context. Most lookups
704 * happen with v==current, so we expect contention to be low.
705 */
707 #define VTLB_ENTRIES 13
709 struct shadow_vtlb {
710 unsigned long page_number; /* Guest virtual address >> PAGE_SHIFT */
711 unsigned long frame_number; /* Guest physical address >> PAGE_SHIFT */
712 uint32_t pfec; /* PF error code of the lookup that filled this
713 * entry. A pfec of zero means the slot is empty
714 * (since that would require us to re-try anyway) */
715 };
717 /* Call whenever the guest flushes hit actual TLB */
718 static inline void vtlb_flush(struct vcpu *v)
719 {
720 spin_lock(&v->arch.paging.vtlb_lock);
721 memset(v->arch.paging.vtlb, 0, VTLB_ENTRIES * sizeof (struct shadow_vtlb));
722 spin_unlock(&v->arch.paging.vtlb_lock);
723 }
725 static inline int vtlb_hash(unsigned long page_number)
726 {
727 return page_number % VTLB_ENTRIES;
728 }
730 /* Put a translation into the vTLB, potentially clobbering an old one */
731 static inline void vtlb_insert(struct vcpu *v, unsigned long page,
732 unsigned long frame, uint32_t pfec)
733 {
734 struct shadow_vtlb entry =
735 { .page_number = page, .frame_number = frame, .pfec = pfec };
736 spin_lock(&v->arch.paging.vtlb_lock);
737 v->arch.paging.vtlb[vtlb_hash(page)] = entry;
738 spin_unlock(&v->arch.paging.vtlb_lock);
739 }
741 /* Look a translation up in the vTLB. Returns INVALID_GFN if not found. */
742 static inline unsigned long vtlb_lookup(struct vcpu *v,
743 unsigned long va, uint32_t pfec)
744 {
745 unsigned long page_number = va >> PAGE_SHIFT;
746 unsigned long frame_number = INVALID_GFN;
747 int i = vtlb_hash(page_number);
749 spin_lock(&v->arch.paging.vtlb_lock);
750 if ( v->arch.paging.vtlb[i].pfec != 0
751 && v->arch.paging.vtlb[i].page_number == page_number
752 /* Any successful walk that had at least these pfec bits is OK */
753 && (v->arch.paging.vtlb[i].pfec & pfec) == pfec )
754 {
755 frame_number = v->arch.paging.vtlb[i].frame_number;
756 }
757 spin_unlock(&v->arch.paging.vtlb_lock);
758 return frame_number;
759 }
760 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
763 #endif /* _XEN_SHADOW_PRIVATE_H */
765 /*
766 * Local variables:
767 * mode: C
768 * c-set-style: "BSD"
769 * c-basic-offset: 4
770 * indent-tabs-mode: nil
771 * End:
772 */