ia64/xen-unstable

changeset 8029:ba50c9d1271e

Clean up mark_dirty() implementation to check for log-dirty
status internally, with the shadow_lock held for consistency.

Callers no need to check log-dirty status themselves.

Also, add dirty logging to alloc_page_type/free_page_type, so
that we see page tables coming and going during live relocation.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Thu Nov 24 15:38:33 2005 +0100 (2005-11-24)
parents dca4893b0b9f
children 238727cb4c26
files xen/arch/x86/mm.c xen/arch/x86/shadow.c xen/arch/x86/shadow32.c xen/arch/x86/shadow_public.c xen/include/asm-x86/domain.h xen/include/asm-x86/grant_table.h xen/include/asm-x86/shadow.h
line diff
     1.1 --- a/xen/arch/x86/mm.c	Thu Nov 24 15:08:27 2005 +0100
     1.2 +++ b/xen/arch/x86/mm.c	Thu Nov 24 15:38:33 2005 +0100
     1.3 @@ -1289,6 +1289,11 @@ static int mod_l4_entry(l4_pgentry_t *pl
     1.4  
     1.5  int alloc_page_type(struct pfn_info *page, unsigned long type)
     1.6  {
     1.7 +    struct domain *owner = page_get_owner(page);
     1.8 +
     1.9 +    if ( owner != NULL )
    1.10 +        mark_dirty(owner, page_to_pfn(page));
    1.11 +
    1.12      switch ( type & PGT_type_mask )
    1.13      {
    1.14      case PGT_l1_page_table:
    1.15 @@ -1318,16 +1323,14 @@ void free_page_type(struct pfn_info *pag
    1.16      struct domain *owner = page_get_owner(page);
    1.17      unsigned long gpfn;
    1.18  
    1.19 -    if ( owner != NULL )
    1.20 +    if ( unlikely((owner != NULL) && shadow_mode_enabled(owner)) )
    1.21      {
    1.22 +        mark_dirty(owner, page_to_pfn(page));
    1.23          if ( unlikely(shadow_mode_refcounts(owner)) )
    1.24              return;
    1.25 -        if ( unlikely(shadow_mode_enabled(owner)) )
    1.26 -        {
    1.27 -            gpfn = __mfn_to_gpfn(owner, page_to_pfn(page));
    1.28 -            ASSERT(VALID_M2P(gpfn));
    1.29 -            remove_shadow(owner, gpfn, type & PGT_type_mask);
    1.30 -        }
    1.31 +        gpfn = __mfn_to_gpfn(owner, page_to_pfn(page));
    1.32 +        ASSERT(VALID_M2P(gpfn));
    1.33 +        remove_shadow(owner, gpfn, type & PGT_type_mask);
    1.34      }
    1.35  
    1.36      switch ( type & PGT_type_mask )
    1.37 @@ -2203,8 +2206,7 @@ int do_mmu_update(
    1.38                      {
    1.39                          shadow_lock(d);
    1.40  
    1.41 -                        if ( shadow_mode_log_dirty(d) )
    1.42 -                            __mark_dirty(d, mfn);
    1.43 +                        __mark_dirty(d, mfn);
    1.44  
    1.45                          if ( page_is_page_table(page) &&
    1.46                               !page_out_of_sync(page) )
    1.47 @@ -2263,13 +2265,7 @@ int do_mmu_update(
    1.48              set_pfn_from_mfn(mfn, gpfn);
    1.49              okay = 1;
    1.50  
    1.51 -            /*
    1.52 -             * If in log-dirty mode, mark the corresponding
    1.53 -             * page as dirty.
    1.54 -             */
    1.55 -            if ( unlikely(shadow_mode_log_dirty(FOREIGNDOM)) &&
    1.56 -                 mark_dirty(FOREIGNDOM, mfn) )
    1.57 -                FOREIGNDOM->arch.shadow_dirty_block_count++;
    1.58 +            mark_dirty(FOREIGNDOM, mfn);
    1.59  
    1.60              put_page(&frame_table[mfn]);
    1.61              break;
    1.62 @@ -2841,8 +2837,7 @@ long do_update_descriptor(u64 pa, u64 de
    1.63      {
    1.64          shadow_lock(dom);
    1.65  
    1.66 -        if ( shadow_mode_log_dirty(dom) )
    1.67 -            __mark_dirty(dom, mfn);
    1.68 +        __mark_dirty(dom, mfn);
    1.69  
    1.70          if ( page_is_page_table(page) && !page_out_of_sync(page) )
    1.71              shadow_mark_mfn_out_of_sync(current, gpfn, mfn);
     2.1 --- a/xen/arch/x86/shadow.c	Thu Nov 24 15:08:27 2005 +0100
     2.2 +++ b/xen/arch/x86/shadow.c	Thu Nov 24 15:38:33 2005 +0100
     2.3 @@ -1858,8 +1858,7 @@ static inline int l1pte_write_fault(
     2.4      SH_VVLOG("l1pte_write_fault: updating spte=0x%" PRIpte " gpte=0x%" PRIpte,
     2.5               l1e_get_intpte(spte), l1e_get_intpte(gpte));
     2.6  
     2.7 -    if ( shadow_mode_log_dirty(d) )
     2.8 -        __mark_dirty(d, gmfn);
     2.9 +    __mark_dirty(d, gmfn);
    2.10  
    2.11      if ( mfn_is_page_table(gmfn) )
    2.12          shadow_mark_va_out_of_sync(v, gpfn, gmfn, va);
    2.13 @@ -2021,9 +2020,7 @@ static int shadow_fault_32(unsigned long
    2.14              domain_crash_synchronous();
    2.15          }
    2.16  
    2.17 -        // if necessary, record the page table page as dirty
    2.18 -        if ( unlikely(shadow_mode_log_dirty(d)) )
    2.19 -            __mark_dirty(d, __gpfn_to_mfn(d, l2e_get_pfn(gpde)));
    2.20 +        __mark_dirty(d, __gpfn_to_mfn(d, l2e_get_pfn(gpde)));
    2.21      }
    2.22  
    2.23      shadow_set_l1e(va, spte, 1);
    2.24 @@ -2082,8 +2079,7 @@ static int do_update_va_mapping(unsigned
    2.25       * the PTE in the PT-holding page. We need the machine frame number
    2.26       * for this.
    2.27       */
    2.28 -    if ( shadow_mode_log_dirty(d) )
    2.29 -        __mark_dirty(d, va_to_l1mfn(v, va));
    2.30 +    __mark_dirty(d, va_to_l1mfn(v, va));
    2.31  
    2.32      shadow_unlock(d);
    2.33  
    2.34 @@ -3189,11 +3185,7 @@ static inline int l2e_rw_fault(
    2.35                  l1e_remove_flags(sl1e, _PAGE_RW);
    2.36              }
    2.37          } else {
    2.38 -            /* log dirty*/
    2.39 -            /*
    2.40 -               if ( shadow_mode_log_dirty(d) )
    2.41 -               __mark_dirty(d, gmfn);
    2.42 -             */
    2.43 +            /* __mark_dirty(d, gmfn); */
    2.44          }
    2.45         // printk("<%s> gpfn: %lx, mfn: %lx, sl1e: %lx\n", __func__, gpfn, mfn, l1e_get_intpte(sl1e));
    2.46          /* The shadow entrys need setup before shadow_mark_va_out_of_sync()*/
    2.47 @@ -3476,9 +3468,7 @@ check_writeable:
    2.48          if (unlikely(!__guest_set_l1e(v, va, &gl1e))) 
    2.49              domain_crash_synchronous();
    2.50  
    2.51 -        // if necessary, record the page table page as dirty
    2.52 -        if ( unlikely(shadow_mode_log_dirty(d)) )
    2.53 -            __mark_dirty(d, __gpfn_to_mfn(d, l2e_get_pfn(gl2e)));
    2.54 +        __mark_dirty(d, __gpfn_to_mfn(d, l2e_get_pfn(gl2e)));
    2.55      }
    2.56  
    2.57      shadow_set_l1e_64(va, (pgentry_64_t *)&sl1e, 1);
     3.1 --- a/xen/arch/x86/shadow32.c	Thu Nov 24 15:08:27 2005 +0100
     3.2 +++ b/xen/arch/x86/shadow32.c	Thu Nov 24 15:38:33 2005 +0100
     3.3 @@ -1274,8 +1274,6 @@ static int shadow_mode_table_op(
     3.4  
     3.5          d->arch.shadow_fault_count       = 0;
     3.6          d->arch.shadow_dirty_count       = 0;
     3.7 -        d->arch.shadow_dirty_net_count   = 0;
     3.8 -        d->arch.shadow_dirty_block_count = 0;
     3.9  
    3.10          break;
    3.11     
    3.12 @@ -1284,13 +1282,9 @@ static int shadow_mode_table_op(
    3.13  
    3.14          sc->stats.fault_count       = d->arch.shadow_fault_count;
    3.15          sc->stats.dirty_count       = d->arch.shadow_dirty_count;
    3.16 -        sc->stats.dirty_net_count   = d->arch.shadow_dirty_net_count;
    3.17 -        sc->stats.dirty_block_count = d->arch.shadow_dirty_block_count;
    3.18  
    3.19          d->arch.shadow_fault_count       = 0;
    3.20          d->arch.shadow_dirty_count       = 0;
    3.21 -        d->arch.shadow_dirty_net_count   = 0;
    3.22 -        d->arch.shadow_dirty_block_count = 0;
    3.23   
    3.24          if ( (sc->dirty_bitmap == NULL) || 
    3.25               (d->arch.shadow_dirty_bitmap == NULL) )
    3.26 @@ -1327,9 +1321,6 @@ static int shadow_mode_table_op(
    3.27      case DOM0_SHADOW_CONTROL_OP_PEEK:
    3.28          sc->stats.fault_count       = d->arch.shadow_fault_count;
    3.29          sc->stats.dirty_count       = d->arch.shadow_dirty_count;
    3.30 -        sc->stats.dirty_net_count   = d->arch.shadow_dirty_net_count;
    3.31 -        sc->stats.dirty_block_count = d->arch.shadow_dirty_block_count;
    3.32 - 
    3.33  
    3.34          if ( (sc->dirty_bitmap == NULL) || 
    3.35               (d->arch.shadow_dirty_bitmap == NULL) )
    3.36 @@ -2738,9 +2729,7 @@ int shadow_fault(unsigned long va, struc
    3.37              domain_crash_synchronous();
    3.38          }
    3.39  
    3.40 -        // if necessary, record the page table page as dirty
    3.41 -        if ( unlikely(shadow_mode_log_dirty(d)) )
    3.42 -            __mark_dirty(d, __gpfn_to_mfn(d, l2e_get_pfn(gpde)));
    3.43 +        __mark_dirty(d, __gpfn_to_mfn(d, l2e_get_pfn(gpde)));
    3.44      }
    3.45  
    3.46      shadow_set_l1e(va, spte, 1);
    3.47 @@ -2837,8 +2826,6 @@ int shadow_do_update_va_mapping(unsigned
    3.48  
    3.49      shadow_lock(d);
    3.50  
    3.51 -    //printk("%s(va=%p, val=%p)\n", __func__, (void *)va, (void *)l1e_get_intpte(val));
    3.52 -        
    3.53      // This is actually overkill - we don't need to sync the L1 itself,
    3.54      // just everything involved in getting to this L1 (i.e. we need
    3.55      // linear_pg_table[l1_linear_offset(va)] to be in sync)...
    3.56 @@ -2853,10 +2840,8 @@ int shadow_do_update_va_mapping(unsigned
    3.57       * the PTE in the PT-holding page. We need the machine frame number
    3.58       * for this.
    3.59       */
    3.60 -    if ( shadow_mode_log_dirty(d) )
    3.61 -        __mark_dirty(d, va_to_l1mfn(v, va));
    3.62 -
    3.63 -// out:
    3.64 +    __mark_dirty(d, va_to_l1mfn(v, va));
    3.65 +
    3.66      shadow_unlock(d);
    3.67  
    3.68      return rc;
     4.1 --- a/xen/arch/x86/shadow_public.c	Thu Nov 24 15:08:27 2005 +0100
     4.2 +++ b/xen/arch/x86/shadow_public.c	Thu Nov 24 15:38:33 2005 +0100
     4.3 @@ -1183,8 +1183,6 @@ static int shadow_mode_table_op(
     4.4  
     4.5          d->arch.shadow_fault_count       = 0;
     4.6          d->arch.shadow_dirty_count       = 0;
     4.7 -        d->arch.shadow_dirty_net_count   = 0;
     4.8 -        d->arch.shadow_dirty_block_count = 0;
     4.9  
    4.10          break;
    4.11     
    4.12 @@ -1193,15 +1191,10 @@ static int shadow_mode_table_op(
    4.13  
    4.14          sc->stats.fault_count       = d->arch.shadow_fault_count;
    4.15          sc->stats.dirty_count       = d->arch.shadow_dirty_count;
    4.16 -        sc->stats.dirty_net_count   = d->arch.shadow_dirty_net_count;
    4.17 -        sc->stats.dirty_block_count = d->arch.shadow_dirty_block_count;
    4.18  
    4.19          d->arch.shadow_fault_count       = 0;
    4.20          d->arch.shadow_dirty_count       = 0;
    4.21 -        d->arch.shadow_dirty_net_count   = 0;
    4.22 -        d->arch.shadow_dirty_block_count = 0;
    4.23   
    4.24 -
    4.25          if ( (sc->dirty_bitmap == NULL) || 
    4.26               (d->arch.shadow_dirty_bitmap == NULL) )
    4.27          {
    4.28 @@ -1236,8 +1229,6 @@ static int shadow_mode_table_op(
    4.29      case DOM0_SHADOW_CONTROL_OP_PEEK:
    4.30          sc->stats.fault_count       = d->arch.shadow_fault_count;
    4.31          sc->stats.dirty_count       = d->arch.shadow_dirty_count;
    4.32 -        sc->stats.dirty_net_count   = d->arch.shadow_dirty_net_count;
    4.33 -        sc->stats.dirty_block_count = d->arch.shadow_dirty_block_count;
    4.34   
    4.35          if ( (sc->dirty_bitmap == NULL) || 
    4.36               (d->arch.shadow_dirty_bitmap == NULL) )
     5.1 --- a/xen/include/asm-x86/domain.h	Thu Nov 24 15:08:27 2005 +0100
     5.2 +++ b/xen/include/asm-x86/domain.h	Thu Nov 24 15:38:33 2005 +0100
     5.3 @@ -49,8 +49,6 @@ struct arch_domain
     5.4  
     5.5      unsigned int shadow_fault_count;
     5.6      unsigned int shadow_dirty_count;
     5.7 -    unsigned int shadow_dirty_net_count;
     5.8 -    unsigned int shadow_dirty_block_count;
     5.9  
    5.10      /* full shadow mode */
    5.11      struct out_of_sync_entry *out_of_sync; /* list of out-of-sync pages */
     6.1 --- a/xen/include/asm-x86/grant_table.h	Thu Nov 24 15:08:27 2005 +0100
     6.2 +++ b/xen/include/asm-x86/grant_table.h	Thu Nov 24 15:38:33 2005 +0100
     6.3 @@ -33,10 +33,6 @@ int steal_page_for_grant_transfer(
     6.4  #define gnttab_shared_mfn(d, t, i)                      \
     6.5      ((virt_to_phys((t)->shared) >> PAGE_SHIFT) + (i))
     6.6  
     6.7 -#define gnttab_log_dirty(d, f)                          \
     6.8 -    do {                                                \
     6.9 -        if ( unlikely(shadow_mode_log_dirty((d))) )     \
    6.10 -            mark_dirty((d), (f));                       \
    6.11 -    } while ( 0 )
    6.12 +#define gnttab_log_dirty(d, f) mark_dirty((d), (f))
    6.13  
    6.14  #endif /* __ASM_GRANT_TABLE_H__ */
     7.1 --- a/xen/include/asm-x86/shadow.h	Thu Nov 24 15:08:27 2005 +0100
     7.2 +++ b/xen/include/asm-x86/shadow.h	Thu Nov 24 15:38:33 2005 +0100
     7.3 @@ -468,21 +468,18 @@ static inline void shadow_put_page(struc
     7.4  
     7.5  /************************************************************************/
     7.6  
     7.7 -static inline int __mark_dirty(struct domain *d, unsigned long mfn)
     7.8 +static inline void __mark_dirty(struct domain *d, unsigned long mfn)
     7.9  {
    7.10      unsigned long pfn;
    7.11 -    int           rc = 0;
    7.12  
    7.13      ASSERT(shadow_lock_is_acquired(d));
    7.14 +
    7.15 +    if ( likely(!shadow_mode_log_dirty(d)) || !VALID_MFN(mfn) )
    7.16 +        return;
    7.17 +
    7.18      ASSERT(d->arch.shadow_dirty_bitmap != NULL);
    7.19  
    7.20 -    if ( !VALID_MFN(mfn) )
    7.21 -        return rc;
    7.22 -
    7.23 -    // N.B. This doesn't use __mfn_to_gpfn().
    7.24 -    // This wants the nice compact set of PFNs from 0..domain's max,
    7.25 -    // which __mfn_to_gpfn() only returns for translated domains.
    7.26 -    //
    7.27 +    /* We /really/ mean PFN here, even for non-translated guests. */
    7.28      pfn = get_pfn_from_mfn(mfn);
    7.29  
    7.30      /*
    7.31 @@ -491,16 +488,13 @@ static inline int __mark_dirty(struct do
    7.32       * Nothing to do here...
    7.33       */
    7.34      if ( unlikely(IS_INVALID_M2P_ENTRY(pfn)) )
    7.35 -        return rc;
    7.36 +        return;
    7.37  
    7.38 -    if ( likely(pfn < d->arch.shadow_dirty_bitmap_size) )
    7.39 +    /* N.B. Can use non-atomic TAS because protected by shadow_lock. */
    7.40 +    if ( likely(pfn < d->arch.shadow_dirty_bitmap_size) &&
    7.41 +         !__test_and_set_bit(pfn, d->arch.shadow_dirty_bitmap) )
    7.42      {
    7.43 -        /* N.B. Can use non-atomic TAS because protected by shadow_lock. */
    7.44 -        if ( !__test_and_set_bit(pfn, d->arch.shadow_dirty_bitmap) )
    7.45 -        {
    7.46 -            d->arch.shadow_dirty_count++;
    7.47 -            rc = 1;
    7.48 -        }
    7.49 +        d->arch.shadow_dirty_count++;
    7.50      }
    7.51  #ifndef NDEBUG
    7.52      else if ( mfn < max_page )
    7.53 @@ -513,18 +507,17 @@ static inline int __mark_dirty(struct do
    7.54                 frame_table[mfn].u.inuse.type_info );
    7.55      }
    7.56  #endif
    7.57 -
    7.58 -    return rc;
    7.59  }
    7.60  
    7.61  
    7.62 -static inline int mark_dirty(struct domain *d, unsigned int mfn)
    7.63 +static inline void mark_dirty(struct domain *d, unsigned int mfn)
    7.64  {
    7.65 -    int rc;
    7.66 -    shadow_lock(d);
    7.67 -    rc = __mark_dirty(d, mfn);
    7.68 -    shadow_unlock(d);
    7.69 -    return rc;
    7.70 +    if ( unlikely(shadow_mode_log_dirty(d)) )
    7.71 +    {
    7.72 +        shadow_lock(d);
    7.73 +        __mark_dirty(d, mfn);
    7.74 +        shadow_unlock(d);
    7.75 +    }
    7.76  }
    7.77  
    7.78  
    7.79 @@ -566,8 +559,7 @@ static inline void
    7.80      if ( unlikely(shadow_mode_translate(d)) )
    7.81          update_hl2e(v, va);
    7.82  
    7.83 -    if ( unlikely(shadow_mode_log_dirty(d)) )
    7.84 -        __mark_dirty(d, pagetable_get_pfn(v->arch.guest_table));
    7.85 +    __mark_dirty(d, pagetable_get_pfn(v->arch.guest_table));
    7.86  }
    7.87  
    7.88  static inline void
    7.89 @@ -787,8 +779,7 @@ static inline int l1pte_write_fault(
    7.90      SH_VVLOG("l1pte_write_fault: updating spte=0x%" PRIpte " gpte=0x%" PRIpte,
    7.91               l1e_get_intpte(spte), l1e_get_intpte(gpte));
    7.92  
    7.93 -    if ( shadow_mode_log_dirty(d) )
    7.94 -        __mark_dirty(d, gmfn);
    7.95 +    __mark_dirty(d, gmfn);
    7.96  
    7.97      if ( mfn_is_page_table(gmfn) )
    7.98          shadow_mark_va_out_of_sync(v, gpfn, gmfn, va);
    7.99 @@ -1325,46 +1316,6 @@ shadow_max_pgtable_type(struct domain *d
   7.100      return pttype;
   7.101  }
   7.102  
   7.103 -/*
   7.104 - * N.B. We can make this locking more fine grained (e.g., per shadow page) if
   7.105 - * it ever becomes a problem, but since we need a spin lock on the hash table 
   7.106 - * anyway it's probably not worth being too clever.
   7.107 - */
   7.108 -static inline unsigned long get_shadow_status(
   7.109 -    struct domain *d, unsigned long gpfn, unsigned long stype)
   7.110 -{
   7.111 -    unsigned long res;
   7.112 -
   7.113 -    ASSERT(shadow_mode_enabled(d));
   7.114 -
   7.115 -    /*
   7.116 -     * If we get here we know that some sort of update has happened to the
   7.117 -     * underlying page table page: either a PTE has been updated, or the page
   7.118 -     * has changed type. If we're in log dirty mode, we should set the
   7.119 -     * appropriate bit in the dirty bitmap.
   7.120 -     * N.B. The VA update path doesn't use this and is handled independently. 
   7.121 -     *
   7.122 -     * XXX need to think this through for vmx guests, but probably OK
   7.123 -     */
   7.124 -
   7.125 -    shadow_lock(d);
   7.126 -
   7.127 -    if ( shadow_mode_log_dirty(d) )
   7.128 -        __mark_dirty(d, __gpfn_to_mfn(d, gpfn));
   7.129 -
   7.130 -    if ( !(res = __shadow_status(d, gpfn, stype)) )
   7.131 -        shadow_unlock(d);
   7.132 -
   7.133 -    return res;
   7.134 -}
   7.135 -
   7.136 -
   7.137 -static inline void put_shadow_status(struct domain *d)
   7.138 -{
   7.139 -    shadow_unlock(d);
   7.140 -}
   7.141 -
   7.142 -
   7.143  static inline void delete_shadow_status(
   7.144      struct domain *d, unsigned long gpfn, unsigned long gmfn, unsigned int stype)
   7.145  {