#endif
}
-/* Pull all the entries on an out-of-sync page back into sync. */
-static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va)
+#define _FIXUP_IDX(_b, _i) ((_b) * SHADOW_OOS_FT_HASH + (_i))
+
+void oos_fixup_add(struct vcpu *v, mfn_t gmfn,
+ mfn_t smfn, unsigned long off)
{
- struct page_info *pg = mfn_to_page(gmfn);
+ int idx, i, free = 0, free_slot = 0;
+ struct oos_fixup *fixups = v->arch.paging.shadow.oos_fixups;
- ASSERT(shadow_locked_by_me(v->domain));
- ASSERT(mfn_is_out_of_sync(gmfn));
- /* Guest page must be shadowed *only* as L1 when out of sync. */
- ASSERT(!(mfn_to_page(gmfn)->shadow_flags & SHF_page_type_mask
- & ~SHF_L1_ANY));
- ASSERT(!sh_page_has_multiple_shadows(mfn_to_page(gmfn)));
+ idx = mfn_x(gmfn) % SHADOW_OOS_FT_HASH;
+ for ( i = 0; i < SHADOW_OOS_FT_ENTRIES; i++ )
+ {
+ if ( !mfn_valid(fixups[_FIXUP_IDX(idx, i)].gmfn)
+ || !mfn_is_out_of_sync(fixups[_FIXUP_IDX(idx, i)].gmfn) )
+ {
+ free = 1;
+ free_slot = _FIXUP_IDX(idx, i);
+ }
+ else if ( (mfn_x(fixups[_FIXUP_IDX(idx, i)].gmfn) == mfn_x(gmfn))
+ && (mfn_x(fixups[_FIXUP_IDX(idx, i)].smfn) == mfn_x(smfn))
+ && (fixups[_FIXUP_IDX(idx, i)].off == off) )
+ {
+ perfc_incr(shadow_oos_fixup_no_add);
+ return;
+ }
+ }
- SHADOW_PRINTK("d=%d, v=%d, gmfn=%05lx, va=%lx\n",
- v->domain->domain_id, v->vcpu_id, mfn_x(gmfn), va);
+ if ( free )
+ {
+ if ( !v->arch.paging.shadow.oos_fixup_used )
+ v->arch.paging.shadow.oos_fixup_used = 1;
+ fixups[free_slot].gmfn = gmfn;
+ fixups[free_slot].smfn = smfn;
+ fixups[free_slot].off = off;
+ perfc_incr(shadow_oos_fixup_add_ok);
+ return;
+ }
+
+
+ perfc_incr(shadow_oos_fixup_add_fail);
+}
+
+void oos_fixup_remove(struct vcpu *v, mfn_t gmfn)
+{
+ int idx, i;
+ struct domain *d = v->domain;
+
+ perfc_incr(shadow_oos_fixup_remove);
+
+ idx = mfn_x(gmfn) % SHADOW_OOS_FT_HASH;
+ for_each_vcpu(d, v)
+ {
+ struct oos_fixup *fixups = v->arch.paging.shadow.oos_fixups;
+ for ( i = 0; i < SHADOW_OOS_FT_ENTRIES; i++ )
+ if ( mfn_x(fixups[_FIXUP_IDX(idx, i)].gmfn) == mfn_x(gmfn) )
+ fixups[_FIXUP_IDX(idx, i)].gmfn = _mfn(INVALID_MFN);
+ }
+}
+
+int oos_fixup_flush(struct vcpu *v)
+{
+ int i, rc = 0;
+ struct oos_fixup *fixups = v->arch.paging.shadow.oos_fixups;
+
+ perfc_incr(shadow_oos_fixup_flush);
+
+ if ( !v->arch.paging.shadow.oos_fixup_used )
+ return 0;
+
+ for ( i = 0; i < SHADOW_OOS_FT_HASH * SHADOW_OOS_FT_ENTRIES; i++ )
+ {
+ if ( mfn_valid(fixups[i].gmfn) )
+ {
+ if ( mfn_is_out_of_sync(fixups[i].gmfn) )
+ rc |= sh_remove_write_access_from_sl1p(v, fixups[i].gmfn,
+ fixups[i].smfn,
+ fixups[i].off);
+ fixups[i].gmfn = _mfn(INVALID_MFN);
+ }
+ }
+
+ v->arch.paging.shadow.oos_fixup_used = 0;
+
+ return rc;
+}
+
+int oos_fixup_flush_gmfn(struct vcpu *v, mfn_t gmfn)
+{
+ int idx, i, rc = 0;
+ struct domain *d = v->domain;
+
+ perfc_incr(shadow_oos_fixup_flush_gmfn);
+
+ idx = mfn_x(gmfn) % SHADOW_OOS_FT_HASH;
+ for_each_vcpu(d, v)
+ {
+ struct oos_fixup *fixups = v->arch.paging.shadow.oos_fixups;
+
+ for ( i = 0; i < SHADOW_OOS_FT_ENTRIES; i++ )
+ {
+ if ( mfn_x(fixups[_FIXUP_IDX(idx, i)].gmfn) != mfn_x(gmfn) )
+ continue;
+
+ rc |= sh_remove_write_access_from_sl1p(v,
+ fixups[_FIXUP_IDX(idx,i)].gmfn,
+ fixups[_FIXUP_IDX(idx,i)].smfn,
+ fixups[_FIXUP_IDX(idx,i)].off);
+
+ fixups[_FIXUP_IDX(idx,i)].gmfn = _mfn(INVALID_MFN);
+ }
+ }
+
+ return rc;
+}
+
+static int oos_remove_write_access(struct vcpu *v, mfn_t gmfn, unsigned long va)
+{
+ int ftlb = 0;
+
+ ftlb |= oos_fixup_flush_gmfn(v, gmfn);
- /* Need to pull write access so the page *stays* in sync.
- * This might be rather slow but we hope that in the common case
- * we're handling this pagetable after a guest walk has pulled
- * write access the fast way. */
switch ( sh_remove_write_access(v, gmfn, 0, va) )
{
default:
break;
case 1:
- flush_tlb_mask(v->domain->domain_dirty_cpumask);
+ ftlb |= 1;
break;
case -1:
/* An unfindable writeable typecount has appeared, probably via a
* grant table entry: can't shoot the mapping, so try to unshadow
* the page. If that doesn't work either, the guest is granting
- * his pagetables and must be killed after all. */
+ * his pagetables and must be killed after all.
+ * This will flush the tlb, so we can return with no worries. */
sh_remove_shadows(v, gmfn, 0 /* Be thorough */, 1 /* Must succeed */);
+ return 1;
+ }
+
+ if ( ftlb )
+ flush_tlb_mask(v->domain->domain_dirty_cpumask);
+
+ return 0;
+}
+
+
+/* Pull all the entries on an out-of-sync page back into sync. */
+static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va)
+{
+ struct page_info *pg = mfn_to_page(gmfn);
+
+ ASSERT(shadow_locked_by_me(v->domain));
+ ASSERT(mfn_is_out_of_sync(gmfn));
+ /* Guest page must be shadowed *only* as L1 when out of sync. */
+ ASSERT(!(mfn_to_page(gmfn)->shadow_flags & SHF_page_type_mask
+ & ~SHF_L1_ANY));
+ ASSERT(!sh_page_has_multiple_shadows(mfn_to_page(gmfn)));
+
+ SHADOW_PRINTK("d=%d, v=%d, gmfn=%05lx, va=%lx\n",
+ v->domain->domain_id, v->vcpu_id, mfn_x(gmfn), va);
+
+ /* Need to pull write access so the page *stays* in sync. */
+ if ( oos_remove_write_access(v, gmfn, va) )
+ {
+ /* Page has been unshadowed. */
return;
}
if ( do_locking )
shadow_lock(v->domain);
+ if ( oos_fixup_flush(v) )
+ flush_tlb_mask(v->domain->domain_dirty_cpumask);
+
/* First: resync all of this vcpu's oos pages */
for ( idx = 0; idx < SHADOW_OOS_PAGES; idx++ )
if ( mfn_valid(oos[idx]) )
#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
/* Was the page out of sync? */
if ( page_is_out_of_sync(page) )
+ {
oos_hash_remove(v, gmfn);
+ oos_fixup_remove(v, gmfn);
+ }
#endif
clear_bit(_PGC_page_table, &page->count_info);
}
#endif /* SHADOW_OPTIMIZATIONS & SHOPT_WRITABLE_HEURISTIC */
/* Brute-force search of all the shadows, by walking the hash */
- perfc_incr(shadow_writeable_bf);
+ if ( level == 0 )
+ perfc_incr(shadow_writeable_bf_1);
+ else
+ perfc_incr(shadow_writeable_bf);
hash_foreach(v, callback_mask, callbacks, gmfn);
/* If that didn't catch the mapping, then there's some non-pagetable
return 1;
}
-
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+int sh_remove_write_access_from_sl1p(struct vcpu *v, mfn_t gmfn,
+ mfn_t smfn, unsigned long off)
+{
+ struct shadow_page_info *sp = mfn_to_shadow_page(smfn);
+
+ ASSERT(mfn_valid(smfn));
+ ASSERT(mfn_valid(gmfn));
+
+ if ( sp->type == SH_type_l1_32_shadow )
+ {
+ return SHADOW_INTERNAL_NAME(sh_rm_write_access_from_sl1p,2)
+ (v, gmfn, smfn, off);
+ }
+#if CONFIG_PAGING_LEVELS >= 3
+ else if ( sp->type == SH_type_l1_pae_shadow )
+ return SHADOW_INTERNAL_NAME(sh_rm_write_access_from_sl1p,3)
+ (v, gmfn, smfn, off);
+#if CONFIG_PAGING_LEVELS >= 4
+ else if ( sp->type == SH_type_l1_64_shadow )
+ return SHADOW_INTERNAL_NAME(sh_rm_write_access_from_sl1p,4)
+ (v, gmfn, smfn, off);
+#endif
+#endif
+
+ return 0;
+}
+#endif
/**************************************************************************/
/* Remove all mappings of a guest frame from the shadow tables.
}
#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ if ( v->arch.paging.shadow.oos_fixups == NULL )
+ {
+ int i;
+ v->arch.paging.shadow.oos_fixups =
+ alloc_xenheap_pages(SHADOW_OOS_FT_ORDER);
+ if ( v->arch.paging.shadow.oos_fixups == NULL )
+ {
+ SHADOW_ERROR("Could not allocate OOS fixup table"
+ " for dom %u vcpu %u\n",
+ v->domain->domain_id, v->vcpu_id);
+ domain_crash(v->domain);
+ return;
+ }
+ for ( i = 0; i < SHADOW_OOS_FT_HASH * SHADOW_OOS_FT_ENTRIES; i++ )
+ v->arch.paging.shadow.oos_fixups[i].gmfn = _mfn(INVALID_MFN);
+ }
+#endif /* OOS */
+
// Valid transitions handled by this function:
// - For PV guests:
// - after a shadow mode has been changed
}
}
-#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+#if (SHADOW_OPTIMIZATIONS & (SHOPT_VIRTUAL_TLB|SHOPT_OUT_OF_SYNC))
/* Free the virtual-TLB array attached to each vcpu */
for_each_vcpu(d, v)
{
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
if ( v->arch.paging.vtlb )
{
xfree(v->arch.paging.vtlb);
v->arch.paging.vtlb = NULL;
}
- }
#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ if ( v->arch.paging.shadow.oos_fixups )
+ {
+ free_xenheap_pages(v->arch.paging.shadow.oos_fixups,
+ SHADOW_OOS_FT_ORDER);
+ }
+#endif /* OOS */
+ }
+#endif /* (SHADOW_OPTIMIZATIONS & (SHOPT_VIRTUAL_TLB|SHOPT_OUT_OF_SYNC)) */
+
list_for_each_safe(entry, n, &d->arch.paging.shadow.p2m_freelist)
{
list_del(entry);
int flags = 0;
struct domain *d = v->domain;
shadow_l1e_t old_sl1e;
+#if SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC
+ mfn_t new_gmfn = shadow_l1e_get_mfn(new_sl1e);
+#endif
ASSERT(sl1e != NULL);
old_sl1e = *sl1e;
/* Doesn't look like a pagetable. */
flags |= SHADOW_SET_ERROR;
new_sl1e = shadow_l1e_empty();
- } else {
+ }
+ else
+ {
shadow_vram_get_l1e(new_sl1e, sl1e, sl1mfn, d);
+#if SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC
+ if ( mfn_valid(new_gmfn) && mfn_oos_may_write(new_gmfn)
+ && (shadow_l1e_get_flags(new_sl1e) & _PAGE_RW) )
+ {
+ oos_fixup_add(v, new_gmfn, sl1mfn, pgentry_ptr_to_slot(sl1e));
+ }
+#endif
+
}
}
}
/**************************************************************************/
/* Functions to revoke guest rights */
+#if SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC
+int sh_rm_write_access_from_sl1p(struct vcpu *v, mfn_t gmfn,
+ mfn_t smfn, unsigned long off)
+{
+ int r;
+ shadow_l1e_t *sl1p, sl1e;
+ struct shadow_page_info *sp;
+
+ ASSERT(mfn_valid(gmfn));
+ ASSERT(mfn_valid(smfn));
+
+ sp = mfn_to_shadow_page(smfn);
+
+ if ( sp->mbz != 0 ||
+#if GUEST_PAGING_LEVELS == 4
+ (sp->type != SH_type_l1_64_shadow)
+#elif GUEST_PAGING_LEVELS == 3
+ (sp->type != SH_type_l1_pae_shadow)
+#elif GUEST_PAGING_LEVELS == 2
+ (sp->type != SH_type_l1_32_shadow)
+#endif
+ )
+ goto fail;
+
+ sl1p = sh_map_domain_page(smfn);
+ sl1p += off;
+ sl1e = *sl1p;
+ if ( ((shadow_l1e_get_flags(sl1e) & (_PAGE_PRESENT|_PAGE_RW))
+ != (_PAGE_PRESENT|_PAGE_RW))
+ || (mfn_x(shadow_l1e_get_mfn(sl1e)) != mfn_x(gmfn)) )
+ {
+ sh_unmap_domain_page(sl1p);
+ goto fail;
+ }
+
+ /* Found it! Need to remove its write permissions. */
+ sl1e = shadow_l1e_remove_flags(sl1e, _PAGE_RW);
+ r = shadow_set_l1e(v, sl1p, sl1e, smfn);
+ ASSERT( !(r & SHADOW_SET_ERROR) );
+
+ sh_unmap_domain_page(sl1p);
+ perfc_incr(shadow_writeable_h_7);
+ return 1;
+
+ fail:
+ perfc_incr(shadow_writeable_h_8);
+ return 0;
+}
+#endif /* OOS */
+
#if SHADOW_OPTIMIZATIONS & SHOPT_WRITABLE_HEURISTIC
static int sh_guess_wrmap(struct vcpu *v, unsigned long vaddr, mfn_t gmfn)
/* Look up this vaddr in the current shadow and see if it's a writeable
projects / xen.git / commitdiff