ia64/xen-unstable

view xen/arch/x86/shadow_public.c @ 11158:b127e557ee74

[HVM] Fix the HVM hypercall issue when paging is not enabled yet on 64bit host.
Signed-off-by: Xin Li <xin.b.li@intel.com>
author kfraser@localhost.localdomain
date Wed Aug 16 11:55:26 2006 +0100 (2006-08-16)
parents 88e6bd5e2b54
children 716ef8e8bddc
line source
1 /******************************************************************************
2 * arch/x86/shadow_public.c
3 *
4 * Copyright (c) 2005 Michael A Fetterman
5 * Based on an earlier implementation by Ian Pratt et al
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 */
23 #include <xen/config.h>
24 #include <xen/types.h>
25 #include <xen/mm.h>
26 #include <xen/domain_page.h>
27 #include <asm/shadow.h>
28 #include <asm/page.h>
29 #include <xen/event.h>
30 #include <xen/sched.h>
31 #include <xen/trace.h>
32 #include <xen/guest_access.h>
33 #include <asm/shadow_64.h>
35 static int alloc_p2m_table(struct domain *d);
36 static void free_p2m_table(struct domain *d);
38 #define SHADOW_MAX_GUEST32(_encoded) ((L1_PAGETABLE_ENTRIES_32 - 1) - ((_encoded) >> 16))
41 int shadow_direct_map_init(struct domain *d)
42 {
43 struct page_info *page;
44 l3_pgentry_t *root;
46 if ( !(page = alloc_domheap_pages(NULL, 0, MEMF_dma)) )
47 return 0;
49 root = map_domain_page(page_to_mfn(page));
50 memset(root, 0, PAGE_SIZE);
51 root[PAE_SHADOW_SELF_ENTRY] = l3e_from_page(page, __PAGE_HYPERVISOR);
53 d->arch.phys_table = pagetable_from_page(page);
55 unmap_domain_page(root);
56 return 1;
57 }
59 void shadow_direct_map_clean(struct domain *d)
60 {
61 unsigned long mfn;
62 l2_pgentry_t *l2e;
63 l3_pgentry_t *l3e;
64 int i, j;
66 mfn = pagetable_get_pfn(d->arch.phys_table);
68 /*
69 * We may fail very early before direct map is built.
70 */
71 if ( !mfn )
72 return;
74 l3e = (l3_pgentry_t *)map_domain_page(mfn);
76 for ( i = 0; i < PAE_L3_PAGETABLE_ENTRIES; i++ )
77 {
78 if ( l3e_get_flags(l3e[i]) & _PAGE_PRESENT )
79 {
80 l2e = map_domain_page(l3e_get_pfn(l3e[i]));
82 for ( j = 0; j < L2_PAGETABLE_ENTRIES; j++ )
83 {
84 if ( l2e_get_flags(l2e[j]) & _PAGE_PRESENT )
85 free_domheap_page(mfn_to_page(l2e_get_pfn(l2e[j])));
86 }
87 unmap_domain_page(l2e);
88 free_domheap_page(mfn_to_page(l3e_get_pfn(l3e[i])));
89 }
90 }
91 free_domheap_page(mfn_to_page(mfn));
93 unmap_domain_page(l3e);
95 d->arch.phys_table = pagetable_null();
96 }
98 /****************************************************************************/
99 /************* export interface functions ***********************************/
100 /****************************************************************************/
101 void free_shadow_pages(struct domain *d);
103 int shadow_set_guest_paging_levels(struct domain *d, int levels)
104 {
105 struct vcpu *v = current;
107 /*
108 * Need to wait for VCPU0 to complete the on-going shadow ops.
109 */
111 if ( v->domain == d && v->vcpu_id )
112 return 1;
114 shadow_lock(d);
116 switch(levels) {
117 #if CONFIG_PAGING_LEVELS == 4
118 case 4:
119 if ( d->arch.ops != &MODE_64_4_HANDLER )
120 d->arch.ops = &MODE_64_4_HANDLER;
121 shadow_unlock(d);
122 return 1;
123 #endif
124 #if CONFIG_PAGING_LEVELS == 3
125 case 3:
126 if ( d->arch.ops == NULL ||
127 shadow_mode_log_dirty(d) )
128 {
129 if ( d->arch.ops != &MODE_64_3_HANDLER )
130 d->arch.ops = &MODE_64_3_HANDLER;
131 }
132 else
133 {
134 if ( d->arch.ops == &MODE_64_2_HANDLER )
135 free_shadow_pages(d);
136 if ( d->arch.ops != &MODE_64_PAE_HANDLER )
137 d->arch.ops = &MODE_64_PAE_HANDLER;
138 }
139 shadow_unlock(d);
140 return 1;
141 #endif
142 #if CONFIG_PAGING_LEVELS == 4
143 case 3:
144 if ( d->arch.ops == &MODE_64_2_HANDLER )
145 free_shadow_pages(d);
146 if ( d->arch.ops != &MODE_64_PAE_HANDLER )
147 d->arch.ops = &MODE_64_PAE_HANDLER;
148 shadow_unlock(d);
149 return 1;
150 #endif
151 case 2:
152 #if CONFIG_PAGING_LEVELS == 2
153 if ( d->arch.ops != &MODE_32_2_HANDLER )
154 d->arch.ops = &MODE_32_2_HANDLER;
155 #elif CONFIG_PAGING_LEVELS >= 3
156 if ( d->arch.ops != &MODE_64_2_HANDLER )
157 d->arch.ops = &MODE_64_2_HANDLER;
158 #endif
159 shadow_unlock(d);
160 return 1;
161 default:
162 shadow_unlock(d);
163 return 0;
164 }
165 }
167 void shadow_invlpg(struct vcpu *v, unsigned long va)
168 {
169 struct domain *d = current->domain;
170 d->arch.ops->invlpg(v, va);
171 }
173 int shadow_fault(unsigned long va, struct cpu_user_regs *regs)
174 {
175 struct domain *d = current->domain;
176 return d->arch.ops->fault(va, regs);
177 }
179 void __update_pagetables(struct vcpu *v)
180 {
181 struct domain *d = v->domain;
182 d->arch.ops->update_pagetables(v);
183 }
185 void __shadow_sync_all(struct domain *d)
186 {
187 d->arch.ops->sync_all(d);
188 }
190 int shadow_remove_all_write_access(
191 struct domain *d, unsigned long readonly_gpfn, unsigned long readonly_gmfn)
192 {
193 return d->arch.ops->remove_all_write_access(d, readonly_gpfn, readonly_gmfn);
194 }
196 int shadow_do_update_va_mapping(unsigned long va,
197 l1_pgentry_t val,
198 struct vcpu *v)
199 {
200 struct domain *d = v->domain;
201 return d->arch.ops->do_update_va_mapping(va, val, v);
202 }
204 struct out_of_sync_entry *
205 shadow_mark_mfn_out_of_sync(struct vcpu *v, unsigned long gpfn,
206 unsigned long mfn)
207 {
208 struct domain *d = v->domain;
209 return d->arch.ops->mark_mfn_out_of_sync(v, gpfn, mfn);
210 }
212 /*
213 * Returns 1 if va's shadow mapping is out-of-sync.
214 * Returns 0 otherwise.
215 */
216 int __shadow_out_of_sync(struct vcpu *v, unsigned long va)
217 {
218 struct domain *d = v->domain;
219 return d->arch.ops->is_out_of_sync(v, va);
220 }
222 unsigned long gva_to_gpa(unsigned long gva)
223 {
224 struct domain *d = current->domain;
225 return d->arch.ops->gva_to_gpa(gva);
226 }
227 /****************************************************************************/
228 /****************************************************************************/
229 #if CONFIG_PAGING_LEVELS >= 3
231 static void inline
232 free_shadow_fl1_table(struct domain *d, unsigned long smfn)
233 {
234 l1_pgentry_t *pl1e = map_domain_page(smfn);
235 int i;
237 for (i = 0; i < L1_PAGETABLE_ENTRIES; i++)
238 put_page_from_l1e(pl1e[i], d);
240 unmap_domain_page(pl1e);
241 }
243 /*
244 * Free l2, l3, l4 shadow tables
245 */
247 void free_fake_shadow_l2(struct domain *d,unsigned long smfn);
249 static void inline
250 free_shadow_tables(struct domain *d, unsigned long smfn, u32 level)
251 {
252 pgentry_64_t *ple = map_domain_page(smfn);
253 int i, external = shadow_mode_external(d);
255 #if CONFIG_PAGING_LEVELS >= 3
256 if ( d->arch.ops->guest_paging_levels == PAGING_L2 )
257 {
258 struct page_info *page = mfn_to_page(smfn);
259 for ( i = 0; i < PAE_L3_PAGETABLE_ENTRIES; i++ )
260 {
261 if ( entry_get_flags(ple[i]) & _PAGE_PRESENT )
262 free_fake_shadow_l2(d, entry_get_pfn(ple[i]));
263 }
265 page = mfn_to_page(entry_get_pfn(ple[0]));
266 free_domheap_pages(page, SL2_ORDER);
267 unmap_domain_page(ple);
268 }
269 else
270 #endif
271 {
272 /*
273 * No Xen mappings in external pages
274 */
275 if ( external )
276 {
277 for ( i = 0; i < PAGETABLE_ENTRIES; i++ ) {
278 if ( entry_get_flags(ple[i]) & _PAGE_PRESENT )
279 put_shadow_ref(entry_get_pfn(ple[i]));
280 if (d->arch.ops->guest_paging_levels == PAGING_L3)
281 {
282 #if CONFIG_PAGING_LEVELS >= 3
283 if ( i == PAE_L3_PAGETABLE_ENTRIES && level == PAGING_L4 )
284 #endif
285 break;
286 }
287 }
288 }
289 else
290 {
291 for ( i = 0; i < PAGETABLE_ENTRIES; i++ )
292 {
293 /*
294 * List the skip/break conditions to avoid freeing
295 * Xen private mappings.
296 */
297 #if CONFIG_PAGING_LEVELS == 2
298 if ( level == PAGING_L2 && !is_guest_l2_slot(0, i) )
299 continue;
300 #endif
301 #if CONFIG_PAGING_LEVELS == 3
302 if ( level == PAGING_L3 && i == L3_PAGETABLE_ENTRIES )
303 break;
304 if ( level == PAGING_L2 )
305 {
306 struct page_info *page = mfn_to_page(smfn);
307 if ( is_xen_l2_slot(page->u.inuse.type_info, i) )
308 continue;
309 }
310 #endif
311 #if CONFIG_PAGING_LEVELS == 4
312 if ( level == PAGING_L4 && !is_guest_l4_slot(i))
313 continue;
314 #endif
315 if ( entry_get_flags(ple[i]) & _PAGE_PRESENT )
316 put_shadow_ref(entry_get_pfn(ple[i]));
317 }
318 }
319 unmap_domain_page(ple);
320 }
321 }
322 #endif
324 #if CONFIG_PAGING_LEVELS == 4
325 static void alloc_monitor_pagetable(struct vcpu *v)
326 {
327 unsigned long mmfn;
328 l4_pgentry_t *mpl4e;
329 struct page_info *mmfn_info;
330 struct domain *d = v->domain;
332 ASSERT(!pagetable_get_paddr(v->arch.monitor_table)); /* we should only get called once */
334 mmfn_info = alloc_domheap_page(NULL);
335 ASSERT( mmfn_info );
336 if (!mmfn_info)
337 {
338 printk("Fail to allocate monitor pagetable\n");
339 domain_crash(v->domain);
340 }
342 mmfn = page_to_mfn(mmfn_info);
343 mpl4e = (l4_pgentry_t *) map_domain_page_global(mmfn);
344 memcpy(mpl4e, idle_pg_table, PAGE_SIZE);
345 mpl4e[l4_table_offset(PERDOMAIN_VIRT_START)] =
346 l4e_from_paddr(__pa(d->arch.mm_perdomain_l3), __PAGE_HYPERVISOR);
348 /* map the phys_to_machine map into the per domain Read-Only MPT space */
350 v->arch.monitor_table = pagetable_from_pfn(mmfn);
351 v->arch.monitor_vtable = (l2_pgentry_t *) mpl4e;
352 mpl4e[l4_table_offset(RO_MPT_VIRT_START)] = l4e_empty();
354 if ( v->vcpu_id == 0 )
355 alloc_p2m_table(d);
356 else
357 {
358 unsigned long mfn;
360 mfn = pagetable_get_pfn(d->vcpu[0]->arch.monitor_table);
361 if ( mfn )
362 {
363 l4_pgentry_t *l4tab;
365 l4tab = map_domain_page(mfn);
367 mpl4e[l4_table_offset(RO_MPT_VIRT_START)] =
368 l4tab[l4_table_offset(RO_MPT_VIRT_START)];
370 unmap_domain_page(l4tab);
371 }
372 }
373 }
375 void free_monitor_pagetable(struct vcpu *v)
376 {
377 unsigned long mfn;
379 /*
380 * free monitor_table.
381 */
382 if ( v->vcpu_id == 0 )
383 free_p2m_table(v->domain);
385 /*
386 * Then free monitor_table.
387 */
388 mfn = pagetable_get_pfn(v->arch.monitor_table);
389 unmap_domain_page_global(v->arch.monitor_vtable);
390 free_domheap_page(mfn_to_page(mfn));
392 v->arch.monitor_table = pagetable_null();
393 v->arch.monitor_vtable = 0;
394 }
395 #elif CONFIG_PAGING_LEVELS == 3
396 static void alloc_monitor_pagetable(struct vcpu *v)
397 {
398 unsigned long m2mfn, m3mfn;
399 l2_pgentry_t *mpl2e;
400 l3_pgentry_t *mpl3e;
401 struct page_info *m2mfn_info, *m3mfn_info;
402 struct domain *d = v->domain;
403 int i;
405 ASSERT(!pagetable_get_paddr(v->arch.monitor_table)); /* we should only get called once */
407 m3mfn_info = alloc_domheap_pages(NULL, 0, MEMF_dma);
408 ASSERT( m3mfn_info );
410 m3mfn = page_to_mfn(m3mfn_info);
411 mpl3e = (l3_pgentry_t *) map_domain_page_global(m3mfn);
412 memset(mpl3e, 0, L3_PAGETABLE_ENTRIES * sizeof(l3_pgentry_t));
414 v->arch.monitor_table = pagetable_from_pfn(m3mfn);
415 v->arch.monitor_vtable = (l2_pgentry_t *) mpl3e;
417 m2mfn_info = alloc_domheap_page(NULL);
418 ASSERT( m2mfn_info );
420 m2mfn = page_to_mfn(m2mfn_info);
421 mpl2e = (l2_pgentry_t *) map_domain_page(m2mfn);
422 memset(mpl2e, 0, PAGE_SIZE);
424 /* Map L2 page into L3 */
425 mpl3e[L3_PAGETABLE_ENTRIES - 1] = l3e_from_pfn(m2mfn, _PAGE_PRESENT);
427 memcpy(&mpl2e[L2_PAGETABLE_FIRST_XEN_SLOT & (L2_PAGETABLE_ENTRIES-1)],
428 &idle_pg_table_l2[L2_PAGETABLE_FIRST_XEN_SLOT],
429 L2_PAGETABLE_XEN_SLOTS * sizeof(l2_pgentry_t));
431 for ( i = 0; i < PDPT_L2_ENTRIES; i++ )
432 mpl2e[l2_table_offset(PERDOMAIN_VIRT_START) + i] =
433 l2e_from_page(
434 virt_to_page(d->arch.mm_perdomain_pt) + i,
435 __PAGE_HYPERVISOR);
436 for ( i = 0; i < (LINEARPT_MBYTES >> (L2_PAGETABLE_SHIFT - 20)); i++ )
437 mpl2e[l2_table_offset(LINEAR_PT_VIRT_START) + i] =
438 (l3e_get_flags(mpl3e[i]) & _PAGE_PRESENT) ?
439 l2e_from_pfn(l3e_get_pfn(mpl3e[i]), __PAGE_HYPERVISOR) :
440 l2e_empty();
441 for ( i = 0; i < (MACHPHYS_MBYTES >> (L2_PAGETABLE_SHIFT - 20)); i++ )
442 mpl2e[l2_table_offset(RO_MPT_VIRT_START) + i] = l2e_empty();
444 if ( v->vcpu_id == 0 )
445 {
446 unsigned long m1mfn;
447 l1_pgentry_t *mpl1e;
448 struct page_info *m1mfn_info;
450 /*
451 * 2 l2 slots are allocated here, so that 4M for p2m table,
452 * with this we can guarantee PCI MMIO p2m entries, especially
453 * Cirrus VGA, can be seen by all other vcpus.
454 */
455 for ( i = 0; i < 2; i++ )
456 {
457 m1mfn_info = alloc_domheap_page(NULL);
458 ASSERT( m1mfn_info );
460 m1mfn = page_to_mfn(m1mfn_info);
461 mpl1e = (l1_pgentry_t *) map_domain_page(m1mfn);
462 memset(mpl1e, 0, PAGE_SIZE);
463 unmap_domain_page(mpl1e);
465 /* Map L1 page into L2 */
466 mpl2e[l2_table_offset(RO_MPT_VIRT_START) + i] =
467 l2e_from_pfn(m1mfn, __PAGE_HYPERVISOR);
468 }
470 alloc_p2m_table(d);
471 }
472 else
473 {
474 unsigned long mfn;
476 mfn = pagetable_get_pfn(d->vcpu[0]->arch.monitor_table);
477 if ( mfn )
478 {
479 l3_pgentry_t *l3tab, l3e;
480 l2_pgentry_t *l2tab;
482 l3tab = map_domain_page(mfn);
483 l3e = l3tab[l3_table_offset(RO_MPT_VIRT_START)];
485 /*
486 * NB: when CONFIG_PAGING_LEVELS == 3,
487 * (entry_get_flags(l3e) & _PAGE_PRESENT) is always true here.
488 * alloc_monitor_pagetable should guarantee this.
489 */
490 if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
491 BUG();
493 l2tab = map_domain_page(l3e_get_pfn(l3e));
495 for ( i = 0; i < (MACHPHYS_MBYTES >> (L2_PAGETABLE_SHIFT - 20)); i++ )
496 mpl2e[l2_table_offset(RO_MPT_VIRT_START) + i] =
497 l2tab[l2_table_offset(RO_MPT_VIRT_START) + i];
499 unmap_domain_page(l2tab);
500 unmap_domain_page(l3tab);
501 }
502 }
504 unmap_domain_page(mpl2e);
505 }
507 void free_monitor_pagetable(struct vcpu *v)
508 {
509 unsigned long m2mfn, m3mfn;
510 /*
511 * free monitor_table.
512 */
513 if ( v->vcpu_id == 0 )
514 free_p2m_table(v->domain);
516 m3mfn = pagetable_get_pfn(v->arch.monitor_table);
517 m2mfn = l2e_get_pfn(v->arch.monitor_vtable[L3_PAGETABLE_ENTRIES - 1]);
519 free_domheap_page(mfn_to_page(m2mfn));
520 unmap_domain_page_global(v->arch.monitor_vtable);
521 free_domheap_page(mfn_to_page(m3mfn));
523 v->arch.monitor_table = pagetable_null();
524 v->arch.monitor_vtable = 0;
525 }
526 #endif
528 static void
529 shadow_free_snapshot(struct domain *d, struct out_of_sync_entry *entry)
530 {
531 void *snapshot;
533 if ( entry->snapshot_mfn == SHADOW_SNAPSHOT_ELSEWHERE )
534 return;
536 // Clear the out_of_sync bit.
537 //
538 clear_bit(_PGC_out_of_sync, &mfn_to_page(entry->gmfn)->count_info);
540 // XXX Need to think about how to protect the domain's
541 // information less expensively.
542 //
543 snapshot = map_domain_page(entry->snapshot_mfn);
544 memset(snapshot, 0, PAGE_SIZE);
545 unmap_domain_page(snapshot);
547 put_shadow_ref(entry->snapshot_mfn);
548 }
550 void
551 release_out_of_sync_entry(struct domain *d, struct out_of_sync_entry *entry)
552 {
553 struct page_info *page;
555 page = mfn_to_page(entry->gmfn);
557 // Decrement ref count of guest & shadow pages
558 //
559 put_page(page);
561 // Only use entries that have low bits clear...
562 //
563 if ( !(entry->writable_pl1e & (sizeof(l1_pgentry_t)-1)) )
564 {
565 put_shadow_ref(entry->writable_pl1e >> PAGE_SHIFT);
566 entry->writable_pl1e = -2;
567 }
568 else
569 ASSERT( entry->writable_pl1e == -1 );
571 // Free the snapshot
572 //
573 shadow_free_snapshot(d, entry);
574 }
576 static void remove_out_of_sync_entries(struct domain *d, unsigned long gmfn)
577 {
578 struct out_of_sync_entry *entry = d->arch.out_of_sync;
579 struct out_of_sync_entry **prev = &d->arch.out_of_sync;
580 struct out_of_sync_entry *found = NULL;
582 // NB: Be careful not to call something that manipulates this list
583 // while walking it. Collect the results into a separate list
584 // first, then walk that list.
585 //
586 while ( entry )
587 {
588 if ( entry->gmfn == gmfn )
589 {
590 // remove from out of sync list
591 *prev = entry->next;
593 // add to found list
594 entry->next = found;
595 found = entry;
597 entry = *prev;
598 continue;
599 }
600 prev = &entry->next;
601 entry = entry->next;
602 }
604 prev = NULL;
605 entry = found;
606 while ( entry )
607 {
608 release_out_of_sync_entry(d, entry);
610 prev = &entry->next;
611 entry = entry->next;
612 }
614 // Add found list to free list
615 if ( prev )
616 {
617 *prev = d->arch.out_of_sync_free;
618 d->arch.out_of_sync_free = found;
619 }
620 }
622 static inline void
623 shadow_demote(struct domain *d, unsigned long gpfn, unsigned long gmfn)
624 {
625 if ( !shadow_mode_refcounts(d) )
626 return;
628 ASSERT(mfn_to_page(gmfn)->count_info & PGC_page_table);
630 if ( shadow_max_pgtable_type(d, gpfn, NULL) == PGT_none )
631 {
632 clear_bit(_PGC_page_table, &mfn_to_page(gmfn)->count_info);
634 if ( page_out_of_sync(mfn_to_page(gmfn)) )
635 {
636 remove_out_of_sync_entries(d, gmfn);
637 }
638 }
639 }
641 static void inline
642 free_shadow_l1_table(struct domain *d, unsigned long smfn)
643 {
644 l1_pgentry_t *pl1e = map_domain_page(smfn);
645 l1_pgentry_t *pl1e_next = 0, *sl1e_p;
646 int i;
647 struct page_info *spage = mfn_to_page(smfn);
648 u32 min_max = spage->tlbflush_timestamp;
649 int min = SHADOW_MIN(min_max);
650 int max;
652 if ( d->arch.ops->guest_paging_levels == PAGING_L2 )
653 {
654 max = SHADOW_MAX_GUEST32(min_max);
655 pl1e_next = map_domain_page(smfn + 1);
656 }
657 else
658 max = SHADOW_MAX(min_max);
660 for ( i = min; i <= max; i++ )
661 {
662 if ( pl1e_next && i >= L1_PAGETABLE_ENTRIES )
663 sl1e_p = &pl1e_next[i - L1_PAGETABLE_ENTRIES];
664 else
665 sl1e_p = &pl1e[i];
667 shadow_put_page_from_l1e(*sl1e_p, d);
668 *sl1e_p = l1e_empty();
669 }
671 unmap_domain_page(pl1e);
672 if ( pl1e_next )
673 unmap_domain_page(pl1e_next);
674 }
676 static void inline
677 free_shadow_hl2_table(struct domain *d, unsigned long smfn)
678 {
679 l1_pgentry_t *hl2 = map_domain_page(smfn);
680 int i, limit;
682 SH_VVLOG("%s: smfn=%lx freed", __func__, smfn);
684 #if CONFIG_PAGING_LEVELS == 2
685 if ( shadow_mode_external(d) )
686 limit = L2_PAGETABLE_ENTRIES;
687 else
688 limit = DOMAIN_ENTRIES_PER_L2_PAGETABLE;
689 #endif
691 for ( i = 0; i < limit; i++ )
692 {
693 if ( l1e_get_flags(hl2[i]) & _PAGE_PRESENT )
694 put_page(mfn_to_page(l1e_get_pfn(hl2[i])));
695 }
697 unmap_domain_page(hl2);
698 }
700 static void inline
701 free_shadow_l2_table(struct domain *d, unsigned long smfn, unsigned int type)
702 {
703 l2_pgentry_t *pl2e = map_domain_page(smfn);
704 int i, external = shadow_mode_external(d);
706 for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
707 if ( external || is_guest_l2_slot(type, i) )
708 if ( l2e_get_flags(pl2e[i]) & _PAGE_PRESENT )
709 put_shadow_ref(l2e_get_pfn(pl2e[i]));
711 if ( (PGT_base_page_table == PGT_l2_page_table) &&
712 shadow_mode_translate(d) && !external )
713 {
714 // free the ref to the hl2
715 //
716 put_shadow_ref(l2e_get_pfn(pl2e[l2_table_offset(LINEAR_PT_VIRT_START)]));
717 }
719 unmap_domain_page(pl2e);
720 }
722 void free_fake_shadow_l2(struct domain *d, unsigned long smfn)
723 {
724 pgentry_64_t *ple = map_domain_page(smfn);
725 int i;
727 for ( i = 0; i < PAGETABLE_ENTRIES; i = i + 2 )
728 if ( entry_get_flags(ple[i]) & _PAGE_PRESENT )
729 put_shadow_ref(entry_get_pfn(ple[i]));
731 unmap_domain_page(ple);
732 }
734 void free_shadow_page(unsigned long smfn)
735 {
736 struct page_info *page = mfn_to_page(smfn);
737 unsigned long gmfn = page->u.inuse.type_info & PGT_mfn_mask;
738 struct domain *d = page_get_owner(mfn_to_page(gmfn));
739 unsigned long gpfn = mfn_to_gmfn(d, gmfn);
740 unsigned long type = page->u.inuse.type_info & PGT_type_mask;
741 u64 index = 0;
743 SH_VVLOG("%s: free'ing smfn=%lx", __func__, smfn);
745 ASSERT( ! IS_INVALID_M2P_ENTRY(gpfn) );
746 #if CONFIG_PAGING_LEVELS >= 4
747 if ( type == PGT_fl1_shadow )
748 {
749 unsigned long mfn;
750 mfn = __shadow_status(d, gpfn, PGT_fl1_shadow);
751 if ( !mfn )
752 gpfn |= (1UL << 63);
753 }
754 #endif
755 #if CONFIG_PAGING_LEVELS >= 3
756 if ( d->arch.ops->guest_paging_levels == PAGING_L3 )
757 {
758 if ( type == PGT_l4_shadow )
759 index = page->tlbflush_timestamp;
760 }
761 #endif
763 delete_shadow_status(d, gpfn, gmfn, type, index);
765 switch ( type )
766 {
767 case PGT_l1_shadow:
768 perfc_decr(shadow_l1_pages);
769 shadow_demote(d, gpfn, gmfn);
770 free_shadow_l1_table(d, smfn);
771 d->arch.shadow_page_count--;
772 break;
773 #if CONFIG_PAGING_LEVELS == 2
774 case PGT_l2_shadow:
775 perfc_decr(shadow_l2_pages);
776 shadow_demote(d, gpfn, gmfn);
777 free_shadow_l2_table(d, smfn, page->u.inuse.type_info);
778 d->arch.shadow_page_count--;
779 break;
781 case PGT_hl2_shadow:
782 perfc_decr(hl2_table_pages);
783 shadow_demote(d, gpfn, gmfn);
784 free_shadow_hl2_table(d, smfn);
785 d->arch.hl2_page_count--;
786 break;
787 #endif
788 #if CONFIG_PAGING_LEVELS >= 3
789 case PGT_l2_shadow:
790 case PGT_l3_shadow:
791 shadow_demote(d, gpfn, gmfn);
792 free_shadow_tables(d, smfn, shadow_type_to_level(type));
793 d->arch.shadow_page_count--;
794 break;
796 case PGT_l4_shadow:
797 gpfn = gpfn & PGT_mfn_mask;
798 if ( d->arch.ops->guest_paging_levels == PAGING_L3 )
799 {
800 /*
801 * Since a single PDPT page can have multiple PDPs, it's possible
802 * that shadow_demote() has been already called for gmfn.
803 */
804 if ( mfn_is_page_table(gmfn) )
805 shadow_demote(d, gpfn, gmfn);
806 } else
807 shadow_demote(d, gpfn, gmfn);
809 free_shadow_tables(d, smfn, shadow_type_to_level(type));
810 d->arch.shadow_page_count--;
811 break;
813 case PGT_fl1_shadow:
814 free_shadow_fl1_table(d, smfn);
815 d->arch.shadow_page_count--;
816 break;
817 #endif
818 case PGT_snapshot:
819 perfc_decr(snapshot_pages);
820 break;
822 default:
823 printk("Free shadow weird page type mfn=%lx type=%" PRtype_info "\n",
824 page_to_mfn(page), page->u.inuse.type_info);
825 break;
826 }
828 // No TLB flushes are needed the next time this page gets allocated.
829 //
830 page->tlbflush_timestamp = 0;
831 page->u.free.cpumask = CPU_MASK_NONE;
833 if ( type == PGT_l1_shadow )
834 {
835 list_add(&page->list, &d->arch.free_shadow_frames);
836 perfc_incr(free_l1_pages);
837 }
838 else
839 free_domheap_page(page);
840 }
842 static void
843 free_writable_pte_predictions(struct domain *d)
844 {
845 int i;
846 struct shadow_status *x;
848 for ( i = 0; i < shadow_ht_buckets; i++ )
849 {
850 u32 count;
851 unsigned long *gpfn_list;
853 /* Skip empty buckets. */
854 if ( d->arch.shadow_ht[i].gpfn_and_flags == 0 )
855 continue;
857 count = 0;
858 for ( x = &d->arch.shadow_ht[i]; x != NULL; x = x->next )
859 if ( (x->gpfn_and_flags & PGT_type_mask) == PGT_writable_pred )
860 count++;
862 gpfn_list = xmalloc_array(unsigned long, count);
863 count = 0;
864 for ( x = &d->arch.shadow_ht[i]; x != NULL; x = x->next )
865 if ( (x->gpfn_and_flags & PGT_type_mask) == PGT_writable_pred )
866 gpfn_list[count++] = x->gpfn_and_flags & PGT_mfn_mask;
868 while ( count )
869 {
870 count--;
871 delete_shadow_status(d, gpfn_list[count], 0, PGT_writable_pred, 0);
872 }
874 xfree(gpfn_list);
875 }
876 }
878 static void free_shadow_ht_entries(struct domain *d)
879 {
880 struct shadow_status *x, *n;
882 SH_VLOG("freed tables count=%d l1=%d l2=%d",
883 d->arch.shadow_page_count, perfc_value(shadow_l1_pages),
884 perfc_value(shadow_l2_pages));
886 n = d->arch.shadow_ht_extras;
887 while ( (x = n) != NULL )
888 {
889 d->arch.shadow_extras_count--;
890 n = *((struct shadow_status **)(&x[shadow_ht_extra_size]));
891 xfree(x);
892 }
894 d->arch.shadow_ht_extras = NULL;
895 d->arch.shadow_ht_free = NULL;
897 ASSERT(d->arch.shadow_extras_count == 0);
898 SH_LOG("freed extras, now %d", d->arch.shadow_extras_count);
900 if ( d->arch.shadow_dirty_bitmap != NULL )
901 {
902 xfree(d->arch.shadow_dirty_bitmap);
903 d->arch.shadow_dirty_bitmap = 0;
904 d->arch.shadow_dirty_bitmap_size = 0;
905 }
907 xfree(d->arch.shadow_ht);
908 d->arch.shadow_ht = NULL;
909 }
911 static void free_out_of_sync_entries(struct domain *d)
912 {
913 struct out_of_sync_entry *x, *n;
915 n = d->arch.out_of_sync_extras;
916 while ( (x = n) != NULL )
917 {
918 d->arch.out_of_sync_extras_count--;
919 n = *((struct out_of_sync_entry **)(&x[out_of_sync_extra_size]));
920 xfree(x);
921 }
923 d->arch.out_of_sync_extras = NULL;
924 d->arch.out_of_sync_free = NULL;
925 d->arch.out_of_sync = NULL;
927 ASSERT(d->arch.out_of_sync_extras_count == 0);
928 FSH_LOG("freed extra out_of_sync entries, now %d",
929 d->arch.out_of_sync_extras_count);
930 }
932 void free_shadow_pages(struct domain *d)
933 {
934 int i;
935 struct shadow_status *x;
936 struct vcpu *v;
937 struct list_head *list_ent, *tmp;
939 /*
940 * WARNING! The shadow page table must not currently be in use!
941 * e.g., You are expected to have paused the domain and synchronized CR3.
942 */
944 if( !d->arch.shadow_ht ) return;
946 shadow_audit(d, 1);
948 // first, remove any outstanding refs from out_of_sync entries...
949 //
950 free_out_of_sync_state(d);
952 // second, remove any outstanding refs from v->arch.shadow_table
953 // and CR3.
954 //
955 for_each_vcpu(d, v)
956 {
957 if ( pagetable_get_paddr(v->arch.shadow_table) )
958 {
959 put_shadow_ref(pagetable_get_pfn(v->arch.shadow_table));
960 v->arch.shadow_table = pagetable_null();
962 if ( shadow_mode_external(d) )
963 {
964 if ( v->arch.shadow_vtable )
965 unmap_domain_page_global(v->arch.shadow_vtable);
966 v->arch.shadow_vtable = NULL;
967 }
968 }
970 if ( v->arch.monitor_shadow_ref )
971 {
972 put_shadow_ref(v->arch.monitor_shadow_ref);
973 v->arch.monitor_shadow_ref = 0;
974 }
975 }
977 #if CONFIG_PAGING_LEVELS == 2
978 // For external shadows, remove the monitor table's refs
979 //
980 if ( shadow_mode_external(d) )
981 {
982 for_each_vcpu(d, v)
983 {
984 l2_pgentry_t *mpl2e = v->arch.monitor_vtable;
986 if ( mpl2e )
987 {
988 l2_pgentry_t hl2e = mpl2e[l2_table_offset(LINEAR_PT_VIRT_START)];
989 l2_pgentry_t smfn = mpl2e[l2_table_offset(SH_LINEAR_PT_VIRT_START)];
991 if ( l2e_get_flags(hl2e) & _PAGE_PRESENT )
992 {
993 put_shadow_ref(l2e_get_pfn(hl2e));
994 mpl2e[l2_table_offset(LINEAR_PT_VIRT_START)] = l2e_empty();
995 }
996 if ( l2e_get_flags(smfn) & _PAGE_PRESENT )
997 {
998 put_shadow_ref(l2e_get_pfn(smfn));
999 mpl2e[l2_table_offset(SH_LINEAR_PT_VIRT_START)] = l2e_empty();
1004 #endif
1005 // Now, the only refs to shadow pages that are left are from the shadow
1006 // pages themselves. We just unpin the pinned pages, and the rest
1007 // should automatically disappear.
1008 //
1009 // NB: Beware: each explicitly or implicit call to free_shadow_page
1010 // can/will result in the hash bucket getting rewritten out from
1011 // under us... First, collect the list of pinned pages, then
1012 // free them.
1013 //
1014 for ( i = 0; i < shadow_ht_buckets; i++ )
1016 u32 count;
1017 unsigned long *mfn_list;
1019 /* Skip empty buckets. */
1020 if ( d->arch.shadow_ht[i].gpfn_and_flags == 0 )
1021 continue;
1023 count = 0;
1024 for ( x = &d->arch.shadow_ht[i]; x != NULL; x = x->next )
1025 if ( MFN_PINNED(x->smfn) )
1026 count++;
1027 if ( !count )
1028 continue;
1030 mfn_list = xmalloc_array(unsigned long, count);
1031 count = 0;
1032 for ( x = &d->arch.shadow_ht[i]; x != NULL; x = x->next )
1033 if ( MFN_PINNED(x->smfn) )
1034 mfn_list[count++] = x->smfn;
1036 while ( count )
1038 shadow_unpin(mfn_list[--count]);
1040 xfree(mfn_list);
1043 /* Now free the pre-zero'ed pages from the domain. */
1044 list_for_each_safe(list_ent, tmp, &d->arch.free_shadow_frames)
1046 struct page_info *page = list_entry(list_ent, struct page_info, list);
1048 list_del(list_ent);
1049 perfc_decr(free_l1_pages);
1051 if (d->arch.ops->guest_paging_levels == PAGING_L2)
1053 #if CONFIG_PAGING_LEVELS >=3
1054 free_domheap_pages(page, SL1_ORDER);
1055 #else
1056 free_domheap_page(page);
1057 #endif
1059 else
1060 free_domheap_page(page);
1063 shadow_audit(d, 0);
1065 SH_LOG("Free shadow table.");
1068 void __shadow_mode_disable(struct domain *d)
1070 struct vcpu *v;
1071 #ifndef NDEBUG
1072 int i;
1073 #endif
1075 if ( unlikely(!shadow_mode_enabled(d)) )
1076 return;
1078 free_shadow_pages(d);
1079 free_writable_pte_predictions(d);
1081 #ifndef NDEBUG
1082 for ( i = 0; i < shadow_ht_buckets; i++ )
1084 if ( d->arch.shadow_ht[i].gpfn_and_flags != 0 )
1086 printk("%s: d->arch.shadow_ht[%x].gpfn_and_flags=%"PRIx64"\n",
1087 __FILE__, i, (u64)d->arch.shadow_ht[i].gpfn_and_flags);
1088 BUG();
1091 #endif
1093 d->arch.shadow_mode = 0;
1095 free_shadow_ht_entries(d);
1096 free_out_of_sync_entries(d);
1098 for_each_vcpu(d, v)
1099 update_pagetables(v);
1103 int __shadow_mode_enable(struct domain *d, unsigned int mode)
1105 struct vcpu *v;
1106 int new_modes = (mode & ~d->arch.shadow_mode);
1107 #if defined(CONFIG_PAGING_LEVELS)
1108 int initial_paging_levels = 3;
1109 #endif
1111 // Gotta be adding something to call this function.
1112 ASSERT(new_modes);
1114 // can't take anything away by calling this function.
1115 ASSERT(!(d->arch.shadow_mode & ~mode));
1117 #if defined(CONFIG_PAGING_LEVELS)
1118 if ( CONFIG_PAGING_LEVELS == 2 )
1119 initial_paging_levels = CONFIG_PAGING_LEVELS;
1120 if ( !shadow_set_guest_paging_levels(d,
1121 initial_paging_levels) ) {
1122 printk("Unsupported guest paging levels\n");
1123 domain_crash_synchronous(); /* need to take a clean path */
1125 #endif
1127 for_each_vcpu(d, v)
1129 invalidate_shadow_ldt(v);
1131 // We need to set these up for __update_pagetables().
1132 // See the comment there.
1134 /*
1135 * arch.guest_vtable
1136 */
1137 if ( v->arch.guest_vtable &&
1138 (v->arch.guest_vtable != __linear_l2_table) )
1140 unmap_domain_page_global(v->arch.guest_vtable);
1142 if ( (mode & (SHM_translate | SHM_external)) == SHM_translate )
1143 v->arch.guest_vtable = __linear_l2_table;
1144 else
1145 v->arch.guest_vtable = NULL;
1147 /*
1148 * arch.shadow_vtable
1149 */
1150 if ( v->arch.shadow_vtable &&
1151 (v->arch.shadow_vtable != __shadow_linear_l2_table) )
1153 unmap_domain_page_global(v->arch.shadow_vtable);
1155 if ( !(mode & SHM_external) && d->arch.ops->guest_paging_levels == 2)
1156 v->arch.shadow_vtable = __shadow_linear_l2_table;
1157 else
1158 v->arch.shadow_vtable = NULL;
1160 #if CONFIG_PAGING_LEVELS == 2
1161 /*
1162 * arch.hl2_vtable
1163 */
1164 if ( v->arch.hl2_vtable &&
1165 (v->arch.hl2_vtable != __linear_hl2_table) )
1167 unmap_domain_page_global(v->arch.hl2_vtable);
1169 if ( (mode & (SHM_translate | SHM_external)) == SHM_translate )
1170 v->arch.hl2_vtable = __linear_hl2_table;
1171 else
1172 v->arch.hl2_vtable = NULL;
1173 #endif
1174 /*
1175 * arch.monitor_table & arch.monitor_vtable
1176 */
1177 if ( v->arch.monitor_vtable )
1179 free_monitor_pagetable(v);
1181 if ( mode & SHM_external )
1183 alloc_monitor_pagetable(v);
1187 if ( new_modes & SHM_enable )
1189 ASSERT( !d->arch.shadow_ht );
1190 d->arch.shadow_ht = xmalloc_array(struct shadow_status, shadow_ht_buckets);
1191 if ( d->arch.shadow_ht == NULL )
1192 goto nomem;
1194 memset(d->arch.shadow_ht, 0,
1195 shadow_ht_buckets * sizeof(struct shadow_status));
1198 if ( new_modes & SHM_log_dirty )
1200 ASSERT( !d->arch.shadow_dirty_bitmap );
1201 d->arch.shadow_dirty_bitmap_size =
1202 (d->shared_info->arch.max_pfn + 63) & ~63;
1203 d->arch.shadow_dirty_bitmap =
1204 xmalloc_array(unsigned long, d->arch.shadow_dirty_bitmap_size /
1205 (8 * sizeof(unsigned long)));
1206 if ( d->arch.shadow_dirty_bitmap == NULL )
1208 d->arch.shadow_dirty_bitmap_size = 0;
1209 goto nomem;
1211 memset(d->arch.shadow_dirty_bitmap, 0,
1212 d->arch.shadow_dirty_bitmap_size/8);
1215 if ( new_modes & SHM_translate )
1217 if ( !(new_modes & SHM_external) )
1219 ASSERT( !pagetable_get_paddr(d->arch.phys_table) );
1220 if ( !alloc_p2m_table(d) )
1222 printk("alloc_p2m_table failed (out-of-memory?)\n");
1223 goto nomem;
1228 // Get rid of any shadow pages from any previous shadow mode.
1229 //
1230 free_shadow_pages(d);
1232 d->arch.shadow_mode = mode;
1234 if ( shadow_mode_refcounts(d) )
1236 struct list_head *list_ent;
1237 struct page_info *page;
1239 /*
1240 * Tear down its counts by disassembling its page-table-based refcounts
1241 * Also remove CR3's gcount/tcount.
1242 * That leaves things like GDTs and LDTs and external refs in tact.
1244 * Most pages will be writable tcount=0.
1245 * Some will still be L1 tcount=0 or L2 tcount=0.
1246 * Maybe some pages will be type none tcount=0.
1247 * Pages granted external writable refs (via grant tables?) will
1248 * still have a non-zero tcount. That's OK.
1250 * gcounts will generally be 1 for PGC_allocated.
1251 * GDTs and LDTs will have additional gcounts.
1252 * Any grant-table based refs will still be in the gcount.
1254 * We attempt to grab writable refs to each page thus setting its type
1255 * Immediately put back those type refs.
1257 * Assert that no pages are left with L1/L2/L3/L4 type.
1258 */
1259 audit_adjust_pgtables(d, -1, 1);
1262 for (list_ent = d->page_list.next; list_ent != &d->page_list;
1263 list_ent = page->list.next) {
1265 page = list_entry(list_ent, struct page_info, list);
1266 if ( !get_page_type(page, PGT_writable_page) )
1267 BUG();
1268 put_page_type(page);
1269 /*
1270 * We use tlbflush_timestamp as back pointer to smfn, and need to
1271 * clean up it.
1272 */
1273 if (shadow_mode_external(d))
1274 page->tlbflush_timestamp = 0;
1277 audit_adjust_pgtables(d, 1, 1);
1281 return 0;
1283 nomem:
1284 if ( (new_modes & SHM_enable) )
1286 xfree(d->arch.shadow_ht);
1287 d->arch.shadow_ht = NULL;
1289 if ( (new_modes & SHM_log_dirty) )
1291 xfree(d->arch.shadow_dirty_bitmap);
1292 d->arch.shadow_dirty_bitmap = NULL;
1295 return -ENOMEM;
1299 int shadow_mode_enable(struct domain *d, unsigned int mode)
1301 int rc;
1302 shadow_lock(d);
1303 rc = __shadow_mode_enable(d, mode);
1304 shadow_unlock(d);
1305 return rc;
1308 static int shadow_mode_table_op(
1309 struct domain *d, dom0_shadow_control_t *sc)
1311 unsigned int op = sc->op;
1312 int i, rc = 0;
1313 struct vcpu *v;
1315 ASSERT(shadow_lock_is_acquired(d));
1317 SH_VLOG("shadow mode table op %lx %lx count %d",
1318 (unsigned long)pagetable_get_pfn(d->vcpu[0]->arch.guest_table), /* XXX SMP */
1319 (unsigned long)pagetable_get_pfn(d->vcpu[0]->arch.shadow_table), /* XXX SMP */
1320 d->arch.shadow_page_count);
1322 shadow_audit(d, 1);
1324 switch ( op )
1326 case DOM0_SHADOW_CONTROL_OP_FLUSH:
1327 free_shadow_pages(d);
1329 d->arch.shadow_fault_count = 0;
1330 d->arch.shadow_dirty_count = 0;
1332 break;
1334 case DOM0_SHADOW_CONTROL_OP_CLEAN:
1335 free_shadow_pages(d);
1337 sc->stats.fault_count = d->arch.shadow_fault_count;
1338 sc->stats.dirty_count = d->arch.shadow_dirty_count;
1340 d->arch.shadow_fault_count = 0;
1341 d->arch.shadow_dirty_count = 0;
1343 if ( guest_handle_is_null(sc->dirty_bitmap) ||
1344 (d->arch.shadow_dirty_bitmap == NULL) )
1346 rc = -EINVAL;
1347 break;
1350 if ( sc->pages > d->arch.shadow_dirty_bitmap_size )
1351 sc->pages = d->arch.shadow_dirty_bitmap_size;
1353 #define chunk (8*1024) /* Transfer and clear in 1kB chunks for L1 cache. */
1354 for ( i = 0; i < sc->pages; i += chunk )
1356 int bytes = ((((sc->pages - i) > chunk) ?
1357 chunk : (sc->pages - i)) + 7) / 8;
1359 if ( copy_to_guest_offset(
1360 sc->dirty_bitmap, i/(8*sizeof(unsigned long)),
1361 d->arch.shadow_dirty_bitmap +(i/(8*sizeof(unsigned long))),
1362 (bytes+sizeof(unsigned long)-1) / sizeof(unsigned long)) )
1364 rc = -EINVAL;
1365 break;
1367 memset(
1368 d->arch.shadow_dirty_bitmap + (i/(8*sizeof(unsigned long))),
1369 0, bytes);
1372 break;
1374 case DOM0_SHADOW_CONTROL_OP_PEEK:
1375 sc->stats.fault_count = d->arch.shadow_fault_count;
1376 sc->stats.dirty_count = d->arch.shadow_dirty_count;
1378 if ( guest_handle_is_null(sc->dirty_bitmap) ||
1379 (d->arch.shadow_dirty_bitmap == NULL) )
1381 rc = -EINVAL;
1382 break;
1385 if ( sc->pages > d->arch.shadow_dirty_bitmap_size )
1386 sc->pages = d->arch.shadow_dirty_bitmap_size;
1388 if ( copy_to_guest(sc->dirty_bitmap,
1389 d->arch.shadow_dirty_bitmap,
1390 (((sc->pages+7)/8)+sizeof(unsigned long)-1) /
1391 sizeof(unsigned long)) )
1393 rc = -EINVAL;
1394 break;
1397 break;
1399 default:
1400 rc = -EINVAL;
1401 break;
1404 SH_VLOG("shadow mode table op : page count %d", d->arch.shadow_page_count);
1405 shadow_audit(d, 1);
1407 for_each_vcpu(d,v)
1408 __update_pagetables(v);
1410 return rc;
1413 int shadow_mode_control(struct domain *d, dom0_shadow_control_t *sc)
1415 unsigned int op = sc->op;
1416 int rc = 0;
1417 struct vcpu *v;
1419 if ( unlikely(d == current->domain) )
1421 DPRINTK("Don't try to do a shadow op on yourself!\n");
1422 return -EINVAL;
1425 domain_pause(d);
1427 shadow_lock(d);
1429 switch ( op )
1431 case DOM0_SHADOW_CONTROL_OP_OFF:
1432 if ( shadow_mode_enabled(d) )
1434 __shadow_sync_all(d);
1435 __shadow_mode_disable(d);
1437 break;
1439 case DOM0_SHADOW_CONTROL_OP_ENABLE_TEST:
1440 free_shadow_pages(d);
1441 rc = __shadow_mode_enable(d, SHM_enable);
1442 break;
1444 case DOM0_SHADOW_CONTROL_OP_ENABLE_LOGDIRTY:
1445 free_shadow_pages(d);
1446 rc = __shadow_mode_enable(
1447 d, d->arch.shadow_mode|SHM_enable|SHM_log_dirty);
1448 break;
1450 case DOM0_SHADOW_CONTROL_OP_ENABLE_TRANSLATE:
1451 free_shadow_pages(d);
1452 rc = __shadow_mode_enable(
1453 d, d->arch.shadow_mode|SHM_enable|SHM_refcounts|SHM_translate);
1454 break;
1456 default:
1457 rc = shadow_mode_enabled(d) ? shadow_mode_table_op(d, sc) : -EINVAL;
1458 break;
1461 shadow_unlock(d);
1463 for_each_vcpu(d,v)
1464 update_pagetables(v);
1466 domain_unpause(d);
1468 return rc;
1471 void shadow_mode_init(void)
1475 int _shadow_mode_refcounts(struct domain *d)
1477 return shadow_mode_refcounts(d);
1480 static int
1481 map_p2m_entry(pgentry_64_t *top_tab, unsigned long gpfn, unsigned long mfn)
1483 #if CONFIG_PAGING_LEVELS >= 4
1484 pgentry_64_t l4e = { 0 };
1485 pgentry_64_t *l3tab = NULL;
1486 #endif
1487 #if CONFIG_PAGING_LEVELS >= 3
1488 pgentry_64_t l3e = { 0 };
1489 #endif
1490 l2_pgentry_t *l2tab = NULL;
1491 l1_pgentry_t *l1tab = NULL;
1492 unsigned long *l0tab = NULL;
1493 l2_pgentry_t l2e = { 0 };
1494 l1_pgentry_t l1e = { 0 };
1495 struct page_info *page;
1496 unsigned long va = RO_MPT_VIRT_START + (gpfn * sizeof(mfn));
1498 #if CONFIG_PAGING_LEVELS >= 4
1499 l4e = top_tab[l4_table_offset(va)];
1500 if ( !(entry_get_flags(l4e) & _PAGE_PRESENT) )
1502 page = alloc_domheap_page(NULL);
1503 if ( !page )
1504 goto nomem;
1506 l3tab = map_domain_page(page_to_mfn(page));
1507 memset(l3tab, 0, PAGE_SIZE);
1508 l4e = top_tab[l4_table_offset(va)] =
1509 entry_from_page(page, __PAGE_HYPERVISOR);
1511 else
1512 l3tab = map_domain_page(entry_get_pfn(l4e));
1514 l3e = l3tab[l3_table_offset(va)];
1515 if ( !(entry_get_flags(l3e) & _PAGE_PRESENT) )
1517 page = alloc_domheap_page(NULL);
1518 if ( !page )
1519 goto nomem;
1521 l2tab = map_domain_page(page_to_mfn(page));
1522 memset(l2tab, 0, PAGE_SIZE);
1523 l3e = l3tab[l3_table_offset(va)] =
1524 entry_from_page(page, __PAGE_HYPERVISOR);
1526 else
1527 l2tab = map_domain_page(entry_get_pfn(l3e));
1529 unmap_domain_page(l3tab);
1530 #else
1531 l3e = top_tab[l3_table_offset(va)];
1533 /*
1534 * NB: when CONFIG_PAGING_LEVELS == 3,
1535 * (entry_get_flags(l3e) & _PAGE_PRESENT) is always true here.
1536 * alloc_monitor_pagetable should guarantee this.
1537 */
1538 if ( !(entry_get_flags(l3e) & _PAGE_PRESENT) )
1539 BUG();
1541 l2tab = map_domain_page(entry_get_pfn(l3e));
1542 #endif
1544 l2e = l2tab[l2_table_offset(va)];
1545 if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
1547 page = alloc_domheap_page(NULL);
1548 if ( !page )
1549 goto nomem;
1551 l1tab = map_domain_page(page_to_mfn(page));
1552 memset(l1tab, 0, PAGE_SIZE);
1553 l2e = l2tab[l2_table_offset(va)] =
1554 l2e_from_page(page, __PAGE_HYPERVISOR);
1556 else
1557 l1tab = map_domain_page(l2e_get_pfn(l2e));
1559 unmap_domain_page(l2tab);
1561 l1e = l1tab[l1_table_offset(va)];
1562 if ( !(l1e_get_flags(l1e) & _PAGE_PRESENT) )
1564 page = alloc_domheap_page(NULL);
1565 if ( !page )
1566 goto nomem;
1568 l0tab = map_domain_page(page_to_mfn(page));
1569 memset(l0tab, 0, PAGE_SIZE);
1570 l1e = l1tab[l1_table_offset(va)] =
1571 l1e_from_page(page, __PAGE_HYPERVISOR);
1573 else
1574 l0tab = map_domain_page(l1e_get_pfn(l1e));
1576 unmap_domain_page(l1tab);
1578 l0tab[gpfn & ((PAGE_SIZE / sizeof(mfn)) - 1)] = mfn;
1580 unmap_domain_page(l0tab);
1582 return 1;
1584 nomem:
1585 return 0;
1588 int
1589 set_p2m_entry(struct domain *d, unsigned long gpfn, unsigned long mfn,
1590 struct domain_mmap_cache *l2cache,
1591 struct domain_mmap_cache *l1cache)
1593 unsigned long tabmfn = pagetable_get_pfn(d->vcpu[0]->arch.monitor_table);
1594 pgentry_64_t *top_tab;
1595 int error;
1597 ASSERT(tabmfn != 0);
1598 ASSERT(shadow_lock_is_acquired(d));
1600 top_tab = map_domain_page_with_cache(tabmfn, l2cache);
1602 if ( !(error = map_p2m_entry(top_tab, gpfn, mfn)) )
1603 domain_crash(d);
1605 unmap_domain_page_with_cache(top_tab, l2cache);
1607 return error;
1610 static int
1611 alloc_p2m_table(struct domain *d)
1613 struct list_head *list_ent;
1614 pgentry_64_t *top_tab = NULL;
1615 unsigned long gpfn, mfn;
1616 int error = 0;
1618 ASSERT( pagetable_get_pfn(d->vcpu[0]->arch.monitor_table) );
1620 top_tab = map_domain_page(
1621 pagetable_get_pfn(d->vcpu[0]->arch.monitor_table));
1623 list_ent = d->page_list.next;
1625 while ( list_ent != &d->page_list )
1627 struct page_info *page;
1629 page = list_entry(list_ent, struct page_info, list);
1630 mfn = page_to_mfn(page);
1632 gpfn = get_gpfn_from_mfn(mfn);
1634 if ( !(error = map_p2m_entry(top_tab, gpfn, mfn)) )
1636 domain_crash(d);
1637 break;
1640 list_ent = page->list.next;
1643 unmap_domain_page(top_tab);
1645 return error;
1648 #if CONFIG_PAGING_LEVELS >= 3
1649 static void
1650 free_p2m_table(struct domain *d)
1652 unsigned long va;
1653 l1_pgentry_t *l1tab;
1654 l1_pgentry_t l1e;
1655 l2_pgentry_t *l2tab;
1656 l2_pgentry_t l2e;
1657 #if CONFIG_PAGING_LEVELS >= 3
1658 l3_pgentry_t *l3tab;
1659 l3_pgentry_t l3e;
1660 #endif
1661 #if CONFIG_PAGING_LEVELS == 4
1662 int i3;
1663 l4_pgentry_t *l4tab;
1664 l4_pgentry_t l4e;
1665 #endif
1667 ASSERT( pagetable_get_pfn(d->vcpu[0]->arch.monitor_table) );
1669 #if CONFIG_PAGING_LEVELS == 4
1670 l4tab = map_domain_page(
1671 pagetable_get_pfn(d->vcpu[0]->arch.monitor_table));
1672 #endif
1673 #if CONFIG_PAGING_LEVELS == 3
1674 l3tab = map_domain_page(
1675 pagetable_get_pfn(d->vcpu[0]->arch.monitor_table));
1677 l3e = l3tab[l3_table_offset(RO_MPT_VIRT_START)];
1679 /*
1680 * NB: when CONFIG_PAGING_LEVELS == 3,
1681 * (entry_get_flags(l3e) & _PAGE_PRESENT) is always true here.
1682 * alloc_monitor_pagetable should guarantee this.
1683 */
1684 if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
1685 BUG();
1687 l2tab = map_domain_page(l3e_get_pfn(l3e));
1688 #endif
1690 for ( va = RO_MPT_VIRT_START; va < RO_MPT_VIRT_END; )
1692 #if CONFIG_PAGING_LEVELS == 4
1693 l4e = l4tab[l4_table_offset(va)];
1695 if ( l4e_get_flags(l4e) & _PAGE_PRESENT )
1697 l3tab = map_domain_page(l4e_get_pfn(l4e));
1699 for ( i3 = 0; i3 < L3_PAGETABLE_ENTRIES; i3++ )
1701 l3e = l3tab[l3_table_offset(va)];
1703 if ( l3e_get_flags(l3e) & _PAGE_PRESENT )
1705 int i2;
1707 l2tab = map_domain_page(l3e_get_pfn(l3e));
1709 for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; i2++ )
1711 #endif
1712 l2e = l2tab[l2_table_offset(va)];
1714 if ( l2e_get_flags(l2e) & _PAGE_PRESENT )
1716 int i1;
1718 l1tab = map_domain_page(l2e_get_pfn(l2e));
1720 /*
1721 * unsigned long phys_to_machine_mapping[]
1722 */
1723 for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++ )
1725 l1e = l1tab[l1_table_offset(va)];
1727 if ( l1e_get_flags(l1e) & _PAGE_PRESENT )
1728 free_domheap_page(mfn_to_page(l1e_get_pfn(l1e)));
1730 va += PAGE_SIZE;
1732 unmap_domain_page(l1tab);
1733 free_domheap_page(mfn_to_page(l2e_get_pfn(l2e)));
1735 else
1736 va += PAGE_SIZE * L1_PAGETABLE_ENTRIES;
1738 #if CONFIG_PAGING_LEVELS == 4
1740 unmap_domain_page(l2tab);
1741 free_domheap_page(mfn_to_page(l3e_get_pfn(l3e)));
1743 else
1744 va += PAGE_SIZE * L1_PAGETABLE_ENTRIES * L2_PAGETABLE_ENTRIES;
1746 unmap_domain_page(l3tab);
1747 free_domheap_page(mfn_to_page(l4e_get_pfn(l4e)));
1749 else
1750 va += PAGE_SIZE *
1751 L1_PAGETABLE_ENTRIES * L2_PAGETABLE_ENTRIES * L3_PAGETABLE_ENTRIES;
1752 #endif
1755 #if CONFIG_PAGING_LEVELS == 4
1756 unmap_domain_page(l4tab);
1757 #endif
1758 #if CONFIG_PAGING_LEVELS == 3
1759 unmap_domain_page(l3tab);
1760 #endif
1762 #endif
1764 void shadow_l1_normal_pt_update(
1765 struct domain *d,
1766 paddr_t pa, l1_pgentry_t gpte,
1767 struct domain_mmap_cache *cache)
1769 unsigned long sl1mfn;
1770 l1_pgentry_t *spl1e, spte;
1772 shadow_lock(d);
1774 sl1mfn = __shadow_status(current->domain, pa >> PAGE_SHIFT, PGT_l1_shadow);
1775 if ( sl1mfn )
1777 SH_VVLOG("shadow_l1_normal_pt_update pa=%p, gpde=%" PRIpte,
1778 (void *)pa, l1e_get_intpte(gpte));
1779 l1pte_propagate_from_guest(current->domain, gpte, &spte);
1781 spl1e = map_domain_page_with_cache(sl1mfn, cache);
1782 spl1e[(pa & ~PAGE_MASK) / sizeof(l1_pgentry_t)] = spte;
1783 unmap_domain_page_with_cache(spl1e, cache);
1786 shadow_unlock(d);
1789 void shadow_l2_normal_pt_update(
1790 struct domain *d,
1791 paddr_t pa, l2_pgentry_t gpde,
1792 struct domain_mmap_cache *cache)
1794 unsigned long sl2mfn;
1795 l2_pgentry_t *spl2e;
1797 shadow_lock(d);
1799 sl2mfn = __shadow_status(current->domain, pa >> PAGE_SHIFT, PGT_l2_shadow);
1800 if ( sl2mfn )
1802 SH_VVLOG("shadow_l2_normal_pt_update pa=%p, gpde=%" PRIpte,
1803 (void *)pa, l2e_get_intpte(gpde));
1804 spl2e = map_domain_page_with_cache(sl2mfn, cache);
1805 validate_pde_change(d, gpde,
1806 &spl2e[(pa & ~PAGE_MASK) / sizeof(l2_pgentry_t)]);
1807 unmap_domain_page_with_cache(spl2e, cache);
1810 shadow_unlock(d);
1813 #if CONFIG_PAGING_LEVELS >= 3
1814 void shadow_l3_normal_pt_update(
1815 struct domain *d,
1816 paddr_t pa, l3_pgentry_t l3e,
1817 struct domain_mmap_cache *cache)
1819 unsigned long sl3mfn;
1820 pgentry_64_t *spl3e;
1822 shadow_lock(d);
1824 sl3mfn = __shadow_status(current->domain, pa >> PAGE_SHIFT, PGT_l3_shadow);
1825 if ( sl3mfn )
1827 SH_VVLOG("shadow_l3_normal_pt_update pa=%p, l3e=%" PRIpte,
1828 (void *)pa, l3e_get_intpte(l3e));
1829 spl3e = (pgentry_64_t *) map_domain_page_with_cache(sl3mfn, cache);
1830 validate_entry_change(d, (pgentry_64_t *) &l3e,
1831 &spl3e[(pa & ~PAGE_MASK) / sizeof(l3_pgentry_t)],
1832 shadow_type_to_level(PGT_l3_shadow));
1833 unmap_domain_page_with_cache(spl3e, cache);
1836 shadow_unlock(d);
1838 #endif
1840 #if CONFIG_PAGING_LEVELS >= 4
1841 void shadow_l4_normal_pt_update(
1842 struct domain *d,
1843 paddr_t pa, l4_pgentry_t l4e,
1844 struct domain_mmap_cache *cache)
1846 unsigned long sl4mfn;
1847 pgentry_64_t *spl4e;
1849 shadow_lock(d);
1851 sl4mfn = __shadow_status(current->domain, pa >> PAGE_SHIFT, PGT_l4_shadow);
1852 if ( sl4mfn )
1854 SH_VVLOG("shadow_l4_normal_pt_update pa=%p, l4e=%" PRIpte,
1855 (void *)pa, l4e_get_intpte(l4e));
1856 spl4e = (pgentry_64_t *)map_domain_page_with_cache(sl4mfn, cache);
1857 validate_entry_change(d, (pgentry_64_t *)&l4e,
1858 &spl4e[(pa & ~PAGE_MASK) / sizeof(l4_pgentry_t)],
1859 shadow_type_to_level(PGT_l4_shadow));
1860 unmap_domain_page_with_cache(spl4e, cache);
1863 shadow_unlock(d);
1865 #endif
1867 static void
1868 translate_l1pgtable(struct domain *d, l1_pgentry_t *p2m, unsigned long l1mfn)
1870 int i;
1871 l1_pgentry_t *l1;
1873 l1 = map_domain_page(l1mfn);
1874 for (i = 0; i < L1_PAGETABLE_ENTRIES; i++)
1876 if ( is_guest_l1_slot(i) &&
1877 (l1e_get_flags(l1[i]) & _PAGE_PRESENT) )
1879 unsigned long mfn = l1e_get_pfn(l1[i]);
1880 unsigned long gpfn = mfn_to_gmfn(d, mfn);
1881 ASSERT(l1e_get_pfn(p2m[gpfn]) == mfn);
1882 l1[i] = l1e_from_pfn(gpfn, l1e_get_flags(l1[i]));
1885 unmap_domain_page(l1);
1888 // This is not general enough to handle arbitrary pagetables
1889 // with shared L1 pages, etc., but it is sufficient for bringing
1890 // up dom0.
1891 //
1892 void
1893 translate_l2pgtable(struct domain *d, l1_pgentry_t *p2m, unsigned long l2mfn,
1894 unsigned int type)
1896 int i;
1897 l2_pgentry_t *l2;
1899 ASSERT(shadow_mode_translate(d) && !shadow_mode_external(d));
1901 l2 = map_domain_page(l2mfn);
1902 for (i = 0; i < L2_PAGETABLE_ENTRIES; i++)
1904 if ( is_guest_l2_slot(type, i) &&
1905 (l2e_get_flags(l2[i]) & _PAGE_PRESENT) )
1907 unsigned long mfn = l2e_get_pfn(l2[i]);
1908 unsigned long gpfn = mfn_to_gmfn(d, mfn);
1909 ASSERT(l1e_get_pfn(p2m[gpfn]) == mfn);
1910 l2[i] = l2e_from_pfn(gpfn, l2e_get_flags(l2[i]));
1911 translate_l1pgtable(d, p2m, mfn);
1914 unmap_domain_page(l2);
1917 void
1918 remove_shadow(struct domain *d, unsigned long gpfn, u32 stype)
1920 unsigned long smfn;
1922 shadow_lock(d);
1924 while ( stype >= PGT_l1_shadow )
1926 smfn = __shadow_status(d, gpfn, stype);
1927 if ( smfn && MFN_PINNED(smfn) )
1928 shadow_unpin(smfn);
1929 stype -= PGT_l1_shadow;
1932 shadow_unlock(d);
1935 unsigned long
1936 get_mfn_from_gpfn_foreign(struct domain *d, unsigned long gpfn)
1938 unsigned long va, tabpfn;
1939 l1_pgentry_t *l1, l1e;
1940 l2_pgentry_t *l2, l2e;
1941 #if CONFIG_PAGING_LEVELS >= 4
1942 pgentry_64_t *l4 = NULL;
1943 pgentry_64_t l4e = { 0 };
1944 #endif
1945 pgentry_64_t *l3 = NULL;
1946 pgentry_64_t l3e = { 0 };
1947 unsigned long *l0tab = NULL;
1948 unsigned long mfn;
1950 ASSERT(shadow_mode_translate(d));
1952 perfc_incrc(get_mfn_from_gpfn_foreign);
1954 va = RO_MPT_VIRT_START + (gpfn * sizeof(mfn));
1956 tabpfn = pagetable_get_pfn(d->vcpu[0]->arch.monitor_table);
1957 if ( !tabpfn )
1958 return INVALID_MFN;
1960 #if CONFIG_PAGING_LEVELS >= 4
1961 l4 = map_domain_page(tabpfn);
1962 l4e = l4[l4_table_offset(va)];
1963 unmap_domain_page(l4);
1964 if ( !(entry_get_flags(l4e) & _PAGE_PRESENT) )
1965 return INVALID_MFN;
1967 l3 = map_domain_page(entry_get_pfn(l4e));
1968 #else
1969 l3 = map_domain_page(tabpfn);
1970 #endif
1971 l3e = l3[l3_table_offset(va)];
1972 unmap_domain_page(l3);
1973 if ( !(entry_get_flags(l3e) & _PAGE_PRESENT) )
1974 return INVALID_MFN;
1975 l2 = map_domain_page(entry_get_pfn(l3e));
1976 l2e = l2[l2_table_offset(va)];
1977 unmap_domain_page(l2);
1978 if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
1979 return INVALID_MFN;
1981 l1 = map_domain_page(l2e_get_pfn(l2e));
1982 l1e = l1[l1_table_offset(va)];
1983 unmap_domain_page(l1);
1984 if ( !(l1e_get_flags(l1e) & _PAGE_PRESENT) )
1985 return INVALID_MFN;
1987 l0tab = map_domain_page(l1e_get_pfn(l1e));
1988 mfn = l0tab[gpfn & ((PAGE_SIZE / sizeof (mfn)) - 1)];
1989 unmap_domain_page(l0tab);
1990 return mfn;
1993 static u32 remove_all_access_in_page(
1994 struct domain *d, unsigned long l1mfn, unsigned long forbidden_gmfn)
1996 l1_pgentry_t *pl1e = map_domain_page(l1mfn);
1997 l1_pgentry_t match, ol2e;
1998 unsigned long flags = _PAGE_PRESENT;
1999 int i;
2000 u32 count = 0;
2001 int is_l1_shadow =
2002 ((mfn_to_page(l1mfn)->u.inuse.type_info & PGT_type_mask) ==
2003 PGT_l1_shadow);
2005 match = l1e_from_pfn(forbidden_gmfn, flags);
2007 for (i = 0; i < L1_PAGETABLE_ENTRIES; i++)
2009 if ( l1e_has_changed(pl1e[i], match, flags) )
2010 continue;
2012 ol2e = pl1e[i];
2013 pl1e[i] = l1e_empty();
2014 count++;
2016 if ( is_l1_shadow )
2017 shadow_put_page_from_l1e(ol2e, d);
2018 else /* must be an hl2 page */
2019 put_page(mfn_to_page(forbidden_gmfn));
2022 unmap_domain_page(pl1e);
2024 return count;
2027 static u32 __shadow_remove_all_access(struct domain *d, unsigned long forbidden_gmfn)
2029 int i;
2030 struct shadow_status *a;
2031 u32 count = 0;
2033 if ( unlikely(!shadow_mode_enabled(d)) )
2034 return 0;
2036 ASSERT(shadow_lock_is_acquired(d));
2037 perfc_incrc(remove_all_access);
2039 for (i = 0; i < shadow_ht_buckets; i++)
2041 a = &d->arch.shadow_ht[i];
2042 while ( a && a->gpfn_and_flags )
2044 switch (a->gpfn_and_flags & PGT_type_mask)
2046 case PGT_l1_shadow:
2047 case PGT_l2_shadow:
2048 case PGT_l3_shadow:
2049 case PGT_l4_shadow:
2050 case PGT_hl2_shadow:
2051 count += remove_all_access_in_page(d, a->smfn, forbidden_gmfn);
2052 break;
2053 case PGT_snapshot:
2054 case PGT_writable_pred:
2055 // these can't hold refs to the forbidden page
2056 break;
2057 default:
2058 BUG();
2061 a = a->next;
2065 return count;
2068 void shadow_drop_references(
2069 struct domain *d, struct page_info *page)
2071 if ( likely(!shadow_mode_refcounts(d)) ||
2072 ((page->u.inuse.type_info & PGT_count_mask) == 0) )
2073 return;
2075 /* XXX This needs more thought... */
2076 printk("%s: needing to call __shadow_remove_all_access for mfn=%lx\n",
2077 __func__, page_to_mfn(page));
2078 printk("Before: mfn=%lx c=%08x t=%" PRtype_info "\n", page_to_mfn(page),
2079 page->count_info, page->u.inuse.type_info);
2081 shadow_lock(d);
2082 __shadow_remove_all_access(d, page_to_mfn(page));
2083 shadow_unlock(d);
2085 printk("After: mfn=%lx c=%08x t=%" PRtype_info "\n", page_to_mfn(page),
2086 page->count_info, page->u.inuse.type_info);
2089 /* XXX Needs more thought. Neither pretty nor fast: a place holder. */
2090 void shadow_sync_and_drop_references(
2091 struct domain *d, struct page_info *page)
2093 if ( likely(!shadow_mode_refcounts(d)) )
2094 return;
2096 shadow_lock(d);
2098 if ( page_out_of_sync(page) )
2099 __shadow_sync_mfn(d, page_to_mfn(page));
2101 __shadow_remove_all_access(d, page_to_mfn(page));
2103 shadow_unlock(d);
2106 void clear_all_shadow_status(struct domain *d)
2108 struct vcpu *v = current;
2110 /*
2111 * Don't clean up while other vcpus are working.
2112 */
2113 if ( v->vcpu_id )
2114 return;
2116 shadow_lock(d);
2118 free_shadow_pages(d);
2119 free_shadow_ht_entries(d);
2120 d->arch.shadow_ht =
2121 xmalloc_array(struct shadow_status, shadow_ht_buckets);
2122 if ( d->arch.shadow_ht == NULL ) {
2123 printk("clear all shadow status:xmalloc fail\n");
2124 domain_crash_synchronous();
2126 memset(d->arch.shadow_ht, 0,
2127 shadow_ht_buckets * sizeof(struct shadow_status));
2129 free_out_of_sync_entries(d);
2131 shadow_unlock(d);
2135 /*
2136 * Local variables:
2137 * mode: C
2138 * c-set-style: "BSD"
2139 * c-basic-offset: 4
2140 * tab-width: 4
2141 * indent-tabs-mode: nil
2142 * End:
2143 */