ia64/xen-unstable

view xen/include/asm-x86/shadow.h @ 3814:58be428f51a8

bitkeeper revision 1.1159.258.4 (420e086eRqMDAepFfVSrE8gAuTZ1dg)

Merge maf46@ssh-relay1.cl.cam.ac.uk:/usr/groups/xeno/BK/xen-2.0-testing.bk
into fleming.research:/scratch/fleming/mafetter/xen-2.0-testing.bk
author mafetter@fleming.research
date Sat Feb 12 13:45:18 2005 +0000 (2005-02-12)
parents 4644bea63898 4b4a77f35103
children 2e88f4d9217f 0dc3b8b8c298
line source
1 /* -*- Mode:C; c-basic-offset:4; tab-width:4 -*- */
3 #ifndef _XEN_SHADOW_H
4 #define _XEN_SHADOW_H
6 #include <xen/config.h>
7 #include <xen/types.h>
8 #include <xen/perfc.h>
9 #include <asm/processor.h>
11 /* Shadow PT flag bits in pfn_info */
12 #define PSH_shadowed (1<<31) /* page has a shadow. PFN points to shadow */
13 #define PSH_pfn_mask ((1<<21)-1)
15 /* Shadow PT operation mode : shadowmode variable in mm_struct */
16 #define SHM_test (1) /* just run domain on shadow PTs */
17 #define SHM_logdirty (2) /* log pages that are dirtied */
18 #define SHM_translate (3) /* lookup machine pages in translation table */
19 #define SHM_cow (4) /* copy on write all dirtied pages */
21 #define shadow_linear_pg_table ((l1_pgentry_t *)SH_LINEAR_PT_VIRT_START)
22 #define shadow_linear_l2_table ((l2_pgentry_t *)(SH_LINEAR_PT_VIRT_START + \
23 (SH_LINEAR_PT_VIRT_START >> (L2_PAGETABLE_SHIFT - L1_PAGETABLE_SHIFT))))
25 #define shadow_mode(_d) ((_d)->mm.shadow_mode)
26 #define shadow_lock_init(_d) spin_lock_init(&(_d)->mm.shadow_lock)
27 #define shadow_lock(_m) spin_lock(&(_m)->shadow_lock)
28 #define shadow_unlock(_m) spin_unlock(&(_m)->shadow_lock)
30 extern void shadow_mode_init(void);
31 extern int shadow_mode_control(struct domain *p, dom0_shadow_control_t *sc);
32 extern int shadow_fault(unsigned long va, long error_code);
33 extern void shadow_l1_normal_pt_update(
34 unsigned long pa, unsigned long gpte,
35 unsigned long *prev_spfn_ptr, l1_pgentry_t **prev_spl1e_ptr);
36 extern void shadow_l2_normal_pt_update(unsigned long pa, unsigned long gpte);
37 extern void unshadow_table(unsigned long gpfn, unsigned int type);
38 extern int shadow_mode_enable(struct domain *p, unsigned int mode);
40 extern void __shadow_mode_disable(struct domain *d);
41 static inline void shadow_mode_disable(struct domain *d)
42 {
43 if ( shadow_mode(d) )
44 __shadow_mode_disable(d);
45 }
47 extern unsigned long shadow_l2_table(
48 struct mm_struct *m, unsigned long gpfn);
50 #define SHADOW_DEBUG 0
51 #define SHADOW_HASH_DEBUG 0
53 struct shadow_status {
54 unsigned long pfn; /* Guest pfn. */
55 unsigned long spfn_and_flags; /* Shadow pfn plus flags. */
56 struct shadow_status *next; /* Pull-to-front list. */
57 };
59 #define shadow_ht_extra_size 128
60 #define shadow_ht_buckets 256
62 #ifdef VERBOSE
63 #define SH_LOG(_f, _a...) \
64 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
65 current->id , __LINE__ , ## _a )
66 #else
67 #define SH_LOG(_f, _a...)
68 #endif
70 #if SHADOW_DEBUG
71 #define SH_VLOG(_f, _a...) \
72 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
73 current->id , __LINE__ , ## _a )
74 #else
75 #define SH_VLOG(_f, _a...)
76 #endif
78 #if 0
79 #define SH_VVLOG(_f, _a...) \
80 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
81 current->id , __LINE__ , ## _a )
82 #else
83 #define SH_VVLOG(_f, _a...)
84 #endif
87 /************************************************************************/
89 static inline int __mark_dirty( struct mm_struct *m, unsigned int mfn)
90 {
91 unsigned long pfn;
92 int rc = 0;
94 ASSERT(spin_is_locked(&m->shadow_lock));
95 ASSERT(m->shadow_dirty_bitmap != NULL);
97 pfn = machine_to_phys_mapping[mfn];
99 /*
100 * Values with the MSB set denote MFNs that aren't really part of the
101 * domain's pseudo-physical memory map (e.g., the shared info frame).
102 * Nothing to do here...
103 */
104 if ( unlikely(pfn & 0x80000000UL) )
105 return rc;
107 if ( likely(pfn < m->shadow_dirty_bitmap_size) )
108 {
109 /* N.B. Can use non-atomic TAS because protected by shadow_lock. */
110 if ( !__test_and_set_bit(pfn, m->shadow_dirty_bitmap) )
111 {
112 m->shadow_dirty_count++;
113 rc = 1;
114 }
115 }
116 #ifndef NDEBUG
117 else if ( mfn < max_page )
118 {
119 unsigned long *esp;
120 SH_LOG("mark_dirty OOR! mfn=%x pfn=%lx max=%x (mm %p)",
121 mfn, pfn, m->shadow_dirty_bitmap_size, m );
122 SH_LOG("dom=%p caf=%08x taf=%08x\n",
123 frame_table[mfn].u.inuse.domain,
124 frame_table[mfn].count_info,
125 frame_table[mfn].u.inuse.type_info );
126 __asm__ __volatile__ ("movl %%esp,%0" : "=r" (esp) : );
127 show_trace(esp);
128 }
129 #endif
131 return rc;
132 }
135 static inline int mark_dirty(struct mm_struct *m, unsigned int mfn)
136 {
137 int rc;
138 shadow_lock(m);
139 rc = __mark_dirty(m, mfn);
140 shadow_unlock(m);
141 return rc;
142 }
145 /************************************************************************/
147 static inline void l1pte_write_fault(
148 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
149 {
150 unsigned long gpte = *gpte_p;
151 unsigned long spte = *spte_p;
153 ASSERT(gpte & _PAGE_RW);
155 gpte |= _PAGE_DIRTY | _PAGE_ACCESSED;
157 switch ( m->shadow_mode )
158 {
159 case SHM_test:
160 spte = gpte | _PAGE_RW;
161 break;
163 case SHM_logdirty:
164 spte = gpte | _PAGE_RW;
165 __mark_dirty(m, gpte >> PAGE_SHIFT);
166 break;
167 }
169 *gpte_p = gpte;
170 *spte_p = spte;
171 }
173 static inline void l1pte_read_fault(
174 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
175 {
176 unsigned long gpte = *gpte_p;
177 unsigned long spte = *spte_p;
179 gpte |= _PAGE_ACCESSED;
181 switch ( m->shadow_mode )
182 {
183 case SHM_test:
184 spte = (gpte & _PAGE_DIRTY) ? gpte : (gpte & ~_PAGE_RW);
185 break;
187 case SHM_logdirty:
188 spte = gpte & ~_PAGE_RW;
189 break;
190 }
192 *gpte_p = gpte;
193 *spte_p = spte;
194 }
196 static inline void l1pte_propagate_from_guest(
197 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
198 {
199 unsigned long gpte = *gpte_p;
200 unsigned long spte = *spte_p;
202 switch ( m->shadow_mode )
203 {
204 case SHM_test:
205 spte = 0;
206 if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
207 (_PAGE_PRESENT|_PAGE_ACCESSED) )
208 spte = (gpte & _PAGE_DIRTY) ? gpte : (gpte & ~_PAGE_RW);
209 break;
211 case SHM_logdirty:
212 spte = 0;
213 if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
214 (_PAGE_PRESENT|_PAGE_ACCESSED) )
215 spte = gpte & ~_PAGE_RW;
216 break;
217 }
219 *gpte_p = gpte;
220 *spte_p = spte;
221 }
223 static inline void l2pde_general(
224 struct mm_struct *m,
225 unsigned long *gpde_p,
226 unsigned long *spde_p,
227 unsigned long sl1pfn)
228 {
229 unsigned long gpde = *gpde_p;
230 unsigned long spde = *spde_p;
232 spde = 0;
234 if ( sl1pfn != 0 )
235 {
236 spde = (gpde & ~PAGE_MASK) | (sl1pfn << PAGE_SHIFT) |
237 _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY;
238 gpde |= _PAGE_ACCESSED | _PAGE_DIRTY;
240 /* Detect linear p.t. mappings and write-protect them. */
241 if ( (frame_table[sl1pfn].u.inuse.type_info & PGT_type_mask) ==
242 PGT_l2_page_table )
243 spde = gpde & ~_PAGE_RW;
244 }
246 *gpde_p = gpde;
247 *spde_p = spde;
248 }
250 /*********************************************************************/
252 #if SHADOW_HASH_DEBUG
253 static void shadow_audit(struct mm_struct *m, int print)
254 {
255 int live = 0, free = 0, j = 0, abs;
256 struct shadow_status *a;
258 for ( j = 0; j < shadow_ht_buckets; j++ )
259 {
260 a = &m->shadow_ht[j];
261 if ( a->pfn ) { live++; ASSERT(a->spfn_and_flags & PSH_pfn_mask); }
262 ASSERT(a->pfn < 0x00100000UL);
263 a = a->next;
264 while ( a && (live < 9999) )
265 {
266 live++;
267 if ( (a->pfn == 0) || (a->spfn_and_flags == 0) )
268 {
269 printk("XXX live=%d pfn=%08lx sp=%08lx next=%p\n",
270 live, a->pfn, a->spfn_and_flags, a->next);
271 BUG();
272 }
273 ASSERT(a->pfn < 0x00100000UL);
274 ASSERT(a->spfn_and_flags & PSH_pfn_mask);
275 a = a->next;
276 }
277 ASSERT(live < 9999);
278 }
280 for ( a = m->shadow_ht_free; a != NULL; a = a->next )
281 free++;
283 if ( print)
284 printk("Xlive=%d free=%d\n",live,free);
286 abs = (perfc_value(shadow_l1_pages) + perfc_value(shadow_l2_pages)) - live;
287 if ( (abs < -1) || (abs > 1) )
288 {
289 printk("live=%d free=%d l1=%d l2=%d\n",live,free,
290 perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages) );
291 BUG();
292 }
293 }
294 #else
295 #define shadow_audit(p, print) ((void)0)
296 #endif
300 static inline struct shadow_status *hash_bucket(
301 struct mm_struct *m, unsigned int gpfn)
302 {
303 return &m->shadow_ht[gpfn % shadow_ht_buckets];
304 }
307 static inline unsigned long __shadow_status(
308 struct mm_struct *m, unsigned int gpfn)
309 {
310 struct shadow_status *p, *x, *head;
312 x = head = hash_bucket(m, gpfn);
313 p = NULL;
315 SH_VVLOG("lookup gpfn=%08x bucket=%p", gpfn, x);
316 shadow_audit(m, 0);
318 do
319 {
320 ASSERT(x->pfn || ((x == head) && (x->next == NULL)));
322 if ( x->pfn == gpfn )
323 {
324 /* Pull-to-front if 'x' isn't already the head item. */
325 if ( unlikely(x != head) )
326 {
327 /* Delete 'x' from list and reinsert immediately after head. */
328 p->next = x->next;
329 x->next = head->next;
330 head->next = x;
332 /* Swap 'x' contents with head contents. */
333 SWAP(head->pfn, x->pfn);
334 SWAP(head->spfn_and_flags, x->spfn_and_flags);
335 }
337 return head->spfn_and_flags;
338 }
340 p = x;
341 x = x->next;
342 }
343 while ( x != NULL );
345 return 0;
346 }
348 /*
349 * N.B. We can make this locking more fine grained (e.g., per shadow page) if
350 * it ever becomes a problem, but since we need a spin lock on the hash table
351 * anyway it's probably not worth being too clever.
352 */
353 static inline unsigned long get_shadow_status(
354 struct mm_struct *m, unsigned int gpfn )
355 {
356 unsigned long res;
358 ASSERT(m->shadow_mode);
360 /*
361 * If we get here we know that some sort of update has happened to the
362 * underlying page table page: either a PTE has been updated, or the page
363 * has changed type. If we're in log dirty mode, we should set the
364 * appropriate bit in the dirty bitmap.
365 * N.B. The VA update path doesn't use this and is handled independently.
366 */
368 shadow_lock(m);
370 if ( m->shadow_mode == SHM_logdirty )
371 __mark_dirty( m, gpfn );
373 if ( !(res = __shadow_status(m, gpfn)) )
374 shadow_unlock(m);
376 return res;
377 }
380 static inline void put_shadow_status(
381 struct mm_struct *m)
382 {
383 shadow_unlock(m);
384 }
387 static inline void delete_shadow_status(
388 struct mm_struct *m, unsigned int gpfn)
389 {
390 struct shadow_status *p, *x, *n, *head;
392 ASSERT(spin_is_locked(&m->shadow_lock));
393 ASSERT(gpfn != 0);
395 head = hash_bucket(m, gpfn);
397 SH_VVLOG("delete gpfn=%08x bucket=%p", gpfn, b);
398 shadow_audit(m, 0);
400 /* Match on head item? */
401 if ( head->pfn == gpfn )
402 {
403 if ( (n = head->next) != NULL )
404 {
405 /* Overwrite head with contents of following node. */
406 head->pfn = n->pfn;
407 head->spfn_and_flags = n->spfn_and_flags;
409 /* Delete following node. */
410 head->next = n->next;
412 /* Add deleted node to the free list. */
413 n->pfn = 0;
414 n->spfn_and_flags = 0;
415 n->next = m->shadow_ht_free;
416 m->shadow_ht_free = n;
417 }
418 else
419 {
420 /* This bucket is now empty. Initialise the head node. */
421 head->pfn = 0;
422 head->spfn_and_flags = 0;
423 }
425 goto found;
426 }
428 p = head;
429 x = head->next;
431 do
432 {
433 if ( x->pfn == gpfn )
434 {
435 /* Delete matching node. */
436 p->next = x->next;
438 /* Add deleted node to the free list. */
439 x->pfn = 0;
440 x->spfn_and_flags = 0;
441 x->next = m->shadow_ht_free;
442 m->shadow_ht_free = x;
444 goto found;
445 }
447 p = x;
448 x = x->next;
449 }
450 while ( x != NULL );
452 /* If we got here, it wasn't in the list! */
453 BUG();
455 found:
456 shadow_audit(m, 0);
457 }
460 static inline void set_shadow_status(
461 struct mm_struct *m, unsigned int gpfn, unsigned long s)
462 {
463 struct shadow_status *x, *head, *extra;
464 int i;
466 ASSERT(spin_is_locked(&m->shadow_lock));
467 ASSERT(gpfn != 0);
468 ASSERT(s & PSH_shadowed);
470 x = head = hash_bucket(m, gpfn);
472 SH_VVLOG("set gpfn=%08x s=%08lx bucket=%p(%p)", gpfn, s, b, b->next);
473 shadow_audit(m, 0);
475 /*
476 * STEP 1. If page is already in the table, update it in place.
477 */
479 do
480 {
481 if ( x->pfn == gpfn )
482 {
483 x->spfn_and_flags = s;
484 goto done;
485 }
487 x = x->next;
488 }
489 while ( x != NULL );
491 /*
492 * STEP 2. The page must be inserted into the table.
493 */
495 /* If the bucket is empty then insert the new page as the head item. */
496 if ( head->pfn == 0 )
497 {
498 head->pfn = gpfn;
499 head->spfn_and_flags = s;
500 ASSERT(head->next == NULL);
501 goto done;
502 }
504 /* We need to allocate a new node. Ensure the quicklist is non-empty. */
505 if ( unlikely(m->shadow_ht_free == NULL) )
506 {
507 SH_LOG("Allocate more shadow hashtable blocks.");
509 extra = xmalloc(
510 sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
512 /* XXX Should be more graceful here. */
513 if ( extra == NULL )
514 BUG();
516 memset(extra, 0, sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
518 /* Record the allocation block so it can be correctly freed later. */
519 m->shadow_extras_count++;
520 *((struct shadow_status **)&extra[shadow_ht_extra_size]) =
521 m->shadow_ht_extras;
522 m->shadow_ht_extras = &extra[0];
524 /* Thread a free chain through the newly-allocated nodes. */
525 for ( i = 0; i < (shadow_ht_extra_size - 1); i++ )
526 extra[i].next = &extra[i+1];
527 extra[i].next = NULL;
529 /* Add the new nodes to the free list. */
530 m->shadow_ht_free = &extra[0];
531 }
533 /* Allocate a new node from the quicklist. */
534 x = m->shadow_ht_free;
535 m->shadow_ht_free = x->next;
537 /* Initialise the new node and insert directly after the head item. */
538 x->pfn = gpfn;
539 x->spfn_and_flags = s;
540 x->next = head->next;
541 head->next = x;
543 done:
544 shadow_audit(m, 0);
545 }
547 static inline void __shadow_mk_pagetable(struct mm_struct *mm)
548 {
549 unsigned long gpfn = pagetable_val(mm->pagetable) >> PAGE_SHIFT;
550 unsigned long spfn = __shadow_status(mm, gpfn);
552 if ( unlikely(spfn == 0) )
553 spfn = shadow_l2_table(mm, gpfn);
555 mm->shadow_table = mk_pagetable(spfn << PAGE_SHIFT);
556 }
558 static inline void shadow_mk_pagetable(struct mm_struct *mm)
559 {
560 SH_VVLOG("shadow_mk_pagetable( gptbase=%08lx, mode=%d )",
561 pagetable_val(mm->pagetable), mm->shadow_mode );
563 if ( unlikely(mm->shadow_mode) )
564 {
565 shadow_lock(mm);
566 __shadow_mk_pagetable(mm);
567 shadow_unlock(mm);
568 }
570 SH_VVLOG("leaving shadow_mk_pagetable( gptbase=%08lx, mode=%d ) sh=%08lx",
571 pagetable_val(mm->pagetable), mm->shadow_mode,
572 pagetable_val(mm->shadow_table) );
573 }
575 #if SHADOW_DEBUG
576 extern int _check_pagetable(struct mm_struct *m, pagetable_t pt, char *s);
577 #define check_pagetable(m, pt, s) _check_pagetable(m, pt, s)
578 #else
579 #define check_pagetable(m, pt, s) ((void)0)
580 #endif
582 #endif /* XEN_SHADOW_H */