ia64/xen-unstable

view xen/include/asm-x86/shadow.h @ 2741:524ade91f41f

bitkeeper revision 1.1159.1.280 (417d9c2dsQCkPThidkzK1AZFnLMZ8Q)

Merge labyrinth.cl.cam.ac.uk:/auto/groups/xeno/BK/xeno.bk
into labyrinth.cl.cam.ac.uk:/auto/anfs/scratch/labyrinth/iap10/xeno-clone/xeno.bk
author iap10@labyrinth.cl.cam.ac.uk
date Tue Oct 26 00:37:01 2004 +0000 (2004-10-26)
parents 855925dd3bae 9813b32b51f6
children fe5933507ca5 20087d779fb6 4644bea63898
line source
1 /* -*- Mode:C; c-basic-offset:4; tab-width:4 -*- */
3 #ifndef _XEN_SHADOW_H
4 #define _XEN_SHADOW_H
6 #include <xen/config.h>
7 #include <xen/types.h>
8 #include <xen/perfc.h>
9 #include <asm/processor.h>
11 /* Shadow PT flag bits in pfn_info */
12 #define PSH_shadowed (1<<31) /* page has a shadow. PFN points to shadow */
13 #define PSH_pfn_mask ((1<<21)-1)
15 /* Shadow PT operation mode : shadowmode variable in mm_struct */
16 #define SHM_test (1) /* just run domain on shadow PTs */
17 #define SHM_logdirty (2) /* log pages that are dirtied */
18 #define SHM_translate (3) /* lookup machine pages in translation table */
19 #define SHM_cow (4) /* copy on write all dirtied pages */
21 #define shadow_linear_pg_table ((l1_pgentry_t *)SH_LINEAR_PT_VIRT_START)
22 #define shadow_linear_l2_table ((l2_pgentry_t *)(SH_LINEAR_PT_VIRT_START + \
23 (SH_LINEAR_PT_VIRT_START >> (L2_PAGETABLE_SHIFT - L1_PAGETABLE_SHIFT))))
25 #define shadow_mode(_d) ((_d)->mm.shadow_mode)
26 #define shadow_lock_init(_d) spin_lock_init(&(_d)->mm.shadow_lock)
27 #define shadow_lock(_m) spin_lock(&(_m)->shadow_lock)
28 #define shadow_unlock(_m) spin_unlock(&(_m)->shadow_lock)
30 extern void shadow_mode_init(void);
31 extern int shadow_mode_control(struct domain *p, dom0_shadow_control_t *sc);
32 extern int shadow_fault(unsigned long va, long error_code);
33 extern void shadow_l1_normal_pt_update(
34 unsigned long pa, unsigned long gpte,
35 unsigned long *prev_spfn_ptr, l1_pgentry_t **prev_spl1e_ptr);
36 extern void shadow_l2_normal_pt_update(unsigned long pa, unsigned long gpte);
37 extern void unshadow_table(unsigned long gpfn, unsigned int type);
38 extern int shadow_mode_enable(struct domain *p, unsigned int mode);
40 extern void __shadow_mode_disable(struct domain *d);
41 static inline void shadow_mode_disable(struct domain *d)
42 {
43 if ( shadow_mode(d) )
44 __shadow_mode_disable(d);
45 }
47 extern unsigned long shadow_l2_table(
48 struct mm_struct *m, unsigned long gpfn);
50 #define SHADOW_DEBUG 0
51 #define SHADOW_HASH_DEBUG 0
53 struct shadow_status {
54 unsigned long pfn; /* Guest pfn. */
55 unsigned long spfn_and_flags; /* Shadow pfn plus flags. */
56 struct shadow_status *next; /* Pull-to-front list. */
57 };
59 #define shadow_ht_extra_size 128
60 #define shadow_ht_buckets 256
62 #ifdef VERBOSE
63 #define SH_LOG(_f, _a...) \
64 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
65 current->id , __LINE__ , ## _a )
66 #else
67 #define SH_LOG(_f, _a...)
68 #endif
70 #if SHADOW_DEBUG
71 #define SH_VLOG(_f, _a...) \
72 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
73 current->id , __LINE__ , ## _a )
74 #else
75 #define SH_VLOG(_f, _a...)
76 #endif
78 #if 0
79 #define SH_VVLOG(_f, _a...) \
80 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
81 current->id , __LINE__ , ## _a )
82 #else
83 #define SH_VVLOG(_f, _a...)
84 #endif
87 /************************************************************************/
89 static inline int __mark_dirty( struct mm_struct *m, unsigned int mfn)
90 {
91 unsigned long pfn;
92 int rc = 0;
94 ASSERT(spin_is_locked(&m->shadow_lock));
95 ASSERT(m->shadow_dirty_bitmap != NULL);
97 pfn = machine_to_phys_mapping[mfn];
99 /*
100 * Values with the MSB set denote MFNs that aren't really part of the
101 * domain's pseudo-physical memory map (e.g., the shared info frame).
102 * Nothing to do here...
103 */
104 if ( unlikely(pfn & 0x80000000UL) )
105 return rc;
107 if ( likely(pfn < m->shadow_dirty_bitmap_size) )
108 {
109 /* N.B. Can use non-atomic TAS because protected by shadow_lock. */
110 if ( !__test_and_set_bit(pfn, m->shadow_dirty_bitmap) )
111 {
112 m->shadow_dirty_count++;
113 rc = 1;
114 }
115 }
116 #ifndef NDEBUG
117 else if ( mfn < max_page )
118 {
119 SH_LOG("mark_dirty OOR! mfn=%x pfn=%lx max=%x (mm %p)",
120 mfn, pfn, m->shadow_dirty_bitmap_size, m );
121 SH_LOG("dom=%p caf=%08x taf=%08x\n",
122 frame_table[mfn].u.inuse.domain,
123 frame_table[mfn].count_info,
124 frame_table[mfn].u.inuse.type_info );
125 {
126 extern void show_trace(unsigned long *esp);
127 unsigned long *esp;
128 __asm__ __volatile__ ("movl %%esp,%0" : "=r" (esp) : );
129 show_trace(esp);
130 }
131 }
132 #endif
134 return rc;
135 }
138 static inline int mark_dirty(struct mm_struct *m, unsigned int mfn)
139 {
140 int rc;
141 shadow_lock(m);
142 rc = __mark_dirty(m, mfn);
143 shadow_unlock(m);
144 return rc;
145 }
148 /************************************************************************/
150 static inline void l1pte_write_fault(
151 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
152 {
153 unsigned long gpte = *gpte_p;
154 unsigned long spte = *spte_p;
156 ASSERT(gpte & _PAGE_RW);
158 gpte |= _PAGE_DIRTY | _PAGE_ACCESSED;
160 switch ( m->shadow_mode )
161 {
162 case SHM_test:
163 spte = gpte | _PAGE_RW;
164 break;
166 case SHM_logdirty:
167 spte = gpte | _PAGE_RW;
168 __mark_dirty(m, gpte >> PAGE_SHIFT);
169 break;
170 }
172 *gpte_p = gpte;
173 *spte_p = spte;
174 }
176 static inline void l1pte_read_fault(
177 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
178 {
179 unsigned long gpte = *gpte_p;
180 unsigned long spte = *spte_p;
182 gpte |= _PAGE_ACCESSED;
184 switch ( m->shadow_mode )
185 {
186 case SHM_test:
187 spte = (gpte & _PAGE_DIRTY) ? gpte : (gpte & ~_PAGE_RW);
188 break;
190 case SHM_logdirty:
191 spte = gpte & ~_PAGE_RW;
192 break;
193 }
195 *gpte_p = gpte;
196 *spte_p = spte;
197 }
199 static inline void l1pte_propagate_from_guest(
200 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
201 {
202 unsigned long gpte = *gpte_p;
203 unsigned long spte = *spte_p;
205 switch ( m->shadow_mode )
206 {
207 case SHM_test:
208 spte = 0;
209 if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
210 (_PAGE_PRESENT|_PAGE_ACCESSED) )
211 spte = (gpte & _PAGE_DIRTY) ? gpte : (gpte & ~_PAGE_RW);
212 break;
214 case SHM_logdirty:
215 spte = 0;
216 if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
217 (_PAGE_PRESENT|_PAGE_ACCESSED) )
218 spte = gpte & ~_PAGE_RW;
219 break;
220 }
222 *gpte_p = gpte;
223 *spte_p = spte;
224 }
226 static inline void l2pde_general(
227 struct mm_struct *m,
228 unsigned long *gpde_p,
229 unsigned long *spde_p,
230 unsigned long sl1pfn)
231 {
232 unsigned long gpde = *gpde_p;
233 unsigned long spde = *spde_p;
235 spde = 0;
237 if ( sl1pfn != 0 )
238 {
239 spde = (gpde & ~PAGE_MASK) | (sl1pfn << PAGE_SHIFT) |
240 _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY;
241 gpde |= _PAGE_ACCESSED | _PAGE_DIRTY;
243 /* Detect linear p.t. mappings and write-protect them. */
244 if ( (frame_table[sl1pfn].u.inuse.type_info & PGT_type_mask) ==
245 PGT_l2_page_table )
246 spde = gpde & ~_PAGE_RW;
247 }
249 *gpde_p = gpde;
250 *spde_p = spde;
251 }
253 /*********************************************************************/
255 #if SHADOW_HASH_DEBUG
256 static void shadow_audit(struct mm_struct *m, int print)
257 {
258 int live = 0, free = 0, j = 0, abs;
259 struct shadow_status *a;
261 for ( j = 0; j < shadow_ht_buckets; j++ )
262 {
263 a = &m->shadow_ht[j];
264 if ( a->pfn ) { live++; ASSERT(a->spfn_and_flags & PSH_pfn_mask); }
265 ASSERT(a->pfn < 0x00100000UL);
266 a = a->next;
267 while ( a && (live < 9999) )
268 {
269 live++;
270 if ( (a->pfn == 0) || (a->spfn_and_flags == 0) )
271 {
272 printk("XXX live=%d pfn=%08lx sp=%08lx next=%p\n",
273 live, a->pfn, a->spfn_and_flags, a->next);
274 BUG();
275 }
276 ASSERT(a->pfn < 0x00100000UL);
277 ASSERT(a->spfn_and_flags & PSH_pfn_mask);
278 a = a->next;
279 }
280 ASSERT(live < 9999);
281 }
283 for ( a = m->shadow_ht_free; a != NULL; a = a->next )
284 free++;
286 if ( print)
287 printk("Xlive=%d free=%d\n",live,free);
289 abs = (perfc_value(shadow_l1_pages) + perfc_value(shadow_l2_pages)) - live;
290 if ( (abs < -1) || (abs > 1) )
291 {
292 printk("live=%d free=%d l1=%d l2=%d\n",live,free,
293 perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages) );
294 BUG();
295 }
296 }
297 #else
298 #define shadow_audit(p, print) ((void)0)
299 #endif
303 static inline struct shadow_status *hash_bucket(
304 struct mm_struct *m, unsigned int gpfn)
305 {
306 return &m->shadow_ht[gpfn % shadow_ht_buckets];
307 }
310 static inline unsigned long __shadow_status(
311 struct mm_struct *m, unsigned int gpfn)
312 {
313 struct shadow_status *p, *x, *head;
315 x = head = hash_bucket(m, gpfn);
316 p = NULL;
318 SH_VVLOG("lookup gpfn=%08x bucket=%p", gpfn, x);
319 shadow_audit(m, 0);
321 do
322 {
323 ASSERT(x->pfn || ((x == head) && (x->next == NULL)));
325 if ( x->pfn == gpfn )
326 {
327 /* Pull-to-front if 'x' isn't already the head item. */
328 if ( unlikely(x != head) )
329 {
330 /* Delete 'x' from list and reinsert immediately after head. */
331 p->next = x->next;
332 x->next = head->next;
333 head->next = x;
335 /* Swap 'x' contents with head contents. */
336 SWAP(head->pfn, x->pfn);
337 SWAP(head->spfn_and_flags, x->spfn_and_flags);
338 }
340 return head->spfn_and_flags;
341 }
343 p = x;
344 x = x->next;
345 }
346 while ( x != NULL );
348 return 0;
349 }
351 /*
352 * N.B. We can make this locking more fine grained (e.g., per shadow page) if
353 * it ever becomes a problem, but since we need a spin lock on the hash table
354 * anyway it's probably not worth being too clever.
355 */
356 static inline unsigned long get_shadow_status(
357 struct mm_struct *m, unsigned int gpfn )
358 {
359 unsigned long res;
361 ASSERT(m->shadow_mode);
363 /*
364 * If we get here we know that some sort of update has happened to the
365 * underlying page table page: either a PTE has been updated, or the page
366 * has changed type. If we're in log dirty mode, we should set the
367 * appropriate bit in the dirty bitmap.
368 * N.B. The VA update path doesn't use this and is handled independently.
369 */
371 shadow_lock(m);
373 if ( m->shadow_mode == SHM_logdirty )
374 __mark_dirty( m, gpfn );
376 if ( !(res = __shadow_status(m, gpfn)) )
377 shadow_unlock(m);
379 return res;
380 }
383 static inline void put_shadow_status(
384 struct mm_struct *m)
385 {
386 shadow_unlock(m);
387 }
390 static inline void delete_shadow_status(
391 struct mm_struct *m, unsigned int gpfn)
392 {
393 struct shadow_status *p, *x, *n, *head;
395 ASSERT(spin_is_locked(&m->shadow_lock));
396 ASSERT(gpfn != 0);
398 head = hash_bucket(m, gpfn);
400 SH_VVLOG("delete gpfn=%08x bucket=%p", gpfn, b);
401 shadow_audit(m, 0);
403 /* Match on head item? */
404 if ( head->pfn == gpfn )
405 {
406 if ( (n = head->next) != NULL )
407 {
408 /* Overwrite head with contents of following node. */
409 head->pfn = n->pfn;
410 head->spfn_and_flags = n->spfn_and_flags;
412 /* Delete following node. */
413 head->next = n->next;
415 /* Add deleted node to the free list. */
416 n->pfn = 0;
417 n->spfn_and_flags = 0;
418 n->next = m->shadow_ht_free;
419 m->shadow_ht_free = n;
420 }
421 else
422 {
423 /* This bucket is now empty. Initialise the head node. */
424 head->pfn = 0;
425 head->spfn_and_flags = 0;
426 }
428 goto found;
429 }
431 p = head;
432 x = head->next;
434 do
435 {
436 if ( x->pfn == gpfn )
437 {
438 /* Delete matching node. */
439 p->next = x->next;
441 /* Add deleted node to the free list. */
442 x->pfn = 0;
443 x->spfn_and_flags = 0;
444 x->next = m->shadow_ht_free;
445 m->shadow_ht_free = x;
447 goto found;
448 }
450 p = x;
451 x = x->next;
452 }
453 while ( x != NULL );
455 /* If we got here, it wasn't in the list! */
456 BUG();
458 found:
459 shadow_audit(m, 0);
460 }
463 static inline void set_shadow_status(
464 struct mm_struct *m, unsigned int gpfn, unsigned long s)
465 {
466 struct shadow_status *x, *head, *extra;
467 int i;
469 ASSERT(spin_is_locked(&m->shadow_lock));
470 ASSERT(gpfn != 0);
471 ASSERT(s & PSH_shadowed);
473 x = head = hash_bucket(m, gpfn);
475 SH_VVLOG("set gpfn=%08x s=%08lx bucket=%p(%p)", gpfn, s, b, b->next);
476 shadow_audit(m, 0);
478 /*
479 * STEP 1. If page is already in the table, update it in place.
480 */
482 do
483 {
484 if ( x->pfn == gpfn )
485 {
486 x->spfn_and_flags = s;
487 goto done;
488 }
490 x = x->next;
491 }
492 while ( x != NULL );
494 /*
495 * STEP 2. The page must be inserted into the table.
496 */
498 /* If the bucket is empty then insert the new page as the head item. */
499 if ( head->pfn == 0 )
500 {
501 head->pfn = gpfn;
502 head->spfn_and_flags = s;
503 ASSERT(head->next == NULL);
504 goto done;
505 }
507 /* We need to allocate a new node. Ensure the quicklist is non-empty. */
508 if ( unlikely(m->shadow_ht_free == NULL) )
509 {
510 SH_LOG("Allocate more shadow hashtable blocks.");
512 extra = xmalloc(
513 sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
515 /* XXX Should be more graceful here. */
516 if ( extra == NULL )
517 BUG();
519 memset(extra, 0, sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
521 /* Record the allocation block so it can be correctly freed later. */
522 m->shadow_extras_count++;
523 *((struct shadow_status **)&extra[shadow_ht_extra_size]) =
524 m->shadow_ht_extras;
525 m->shadow_ht_extras = &extra[0];
527 /* Thread a free chain through the newly-allocated nodes. */
528 for ( i = 0; i < (shadow_ht_extra_size - 1); i++ )
529 extra[i].next = &extra[i+1];
530 extra[i].next = NULL;
532 /* Add the new nodes to the free list. */
533 m->shadow_ht_free = &extra[0];
534 }
536 /* Allocate a new node from the quicklist. */
537 x = m->shadow_ht_free;
538 m->shadow_ht_free = x->next;
540 /* Initialise the new node and insert directly after the head item. */
541 x->pfn = gpfn;
542 x->spfn_and_flags = s;
543 x->next = head->next;
544 head->next = x;
546 done:
547 shadow_audit(m, 0);
548 }
550 static inline void __shadow_mk_pagetable(struct mm_struct *mm)
551 {
552 unsigned long gpfn = pagetable_val(mm->pagetable) >> PAGE_SHIFT;
553 unsigned long spfn = __shadow_status(mm, gpfn);
555 if ( unlikely(spfn == 0) )
556 spfn = shadow_l2_table(mm, gpfn);
558 mm->shadow_table = mk_pagetable(spfn << PAGE_SHIFT);
559 }
561 static inline void shadow_mk_pagetable(struct mm_struct *mm)
562 {
563 SH_VVLOG("shadow_mk_pagetable( gptbase=%08lx, mode=%d )",
564 pagetable_val(mm->pagetable), mm->shadow_mode );
566 if ( unlikely(mm->shadow_mode) )
567 {
568 shadow_lock(mm);
569 __shadow_mk_pagetable(mm);
570 shadow_unlock(mm);
571 }
573 SH_VVLOG("leaving shadow_mk_pagetable( gptbase=%08lx, mode=%d ) sh=%08lx",
574 pagetable_val(mm->pagetable), mm->shadow_mode,
575 pagetable_val(mm->shadow_table) );
576 }
578 #if SHADOW_DEBUG
579 extern int check_pagetable(struct mm_struct *m, pagetable_t pt, char *s);
580 #else
581 #define check_pagetable(m, pt, s) ((void)0)
582 #endif
584 #endif /* XEN_SHADOW_H */