ia64/xen-unstable

changeset 4292:13032fd25c06

bitkeeper revision 1.1261 (4242ef64dXDbGRaZN94_Vg02rxL1tg)

Snapshots of L1 page table pages now only snapshot the active portion of
the page.

Improved the tlb flushing of shadow mode somewhat...

Fixed a bug in the shadow_min_max encoding stuff.

Signed-off-by: michael.fetterman@cl.cam.ac.uk
author mafetter@fleming.research
date Thu Mar 24 16:48:36 2005 +0000 (2005-03-24)
parents b40fc0992e25
children 96924dd0a482 016ca3ca12ab
files xen/arch/x86/shadow.c xen/include/asm-x86/shadow.h xen/include/xen/perfc_defn.h
line diff
     1.1 --- a/xen/arch/x86/shadow.c	Thu Mar 24 12:33:04 2005 +0000
     1.2 +++ b/xen/arch/x86/shadow.c	Thu Mar 24 16:48:36 2005 +0000
     1.3 @@ -124,7 +124,7 @@ shadow_demote(struct domain *d, unsigned
     1.4  {
     1.5      ASSERT(frame_table[gmfn].count_info & PGC_page_table);
     1.6  
     1.7 -    if ( shadow_max_pgtable_type(d, gpfn) == PGT_none )
     1.8 +    if ( shadow_max_pgtable_type(d, gpfn, NULL) == PGT_none )
     1.9      {
    1.10          clear_bit(_PGC_page_table, &frame_table[gmfn].count_info);
    1.11  
    1.12 @@ -274,6 +274,10 @@ alloc_shadow_page(struct domain *d,
    1.13          break;
    1.14      }
    1.15  
    1.16 +    // Don't add a new shadow of something that already has a snapshot.
    1.17 +    //
    1.18 +    ASSERT( (psh_type == PGT_snapshot) || !mfn_out_of_sync(gmfn) );
    1.19 +
    1.20      set_shadow_status(d, gpfn, gmfn, smfn, psh_type);
    1.21  
    1.22      if ( pin )
    1.23 @@ -1566,8 +1570,10 @@ static inline unsigned long
    1.24  shadow_make_snapshot(
    1.25      struct domain *d, unsigned long gpfn, unsigned long gmfn)
    1.26  {
    1.27 -    unsigned long smfn;
    1.28 +    unsigned long smfn, sl1mfn;
    1.29      void *original, *snapshot;
    1.30 +    u32 min_max = 0;
    1.31 +    int min, max, length;
    1.32  
    1.33      if ( test_and_set_bit(_PGC_out_of_sync, &frame_table[gmfn].count_info) )
    1.34      {
    1.35 @@ -1588,9 +1594,21 @@ shadow_make_snapshot(
    1.36      if ( !get_shadow_ref(smfn) )
    1.37          BUG();
    1.38  
    1.39 +    if ( shadow_max_pgtable_type(d, gpfn, &sl1mfn) == PGT_l1_shadow )
    1.40 +        min_max = pfn_to_page(sl1mfn)->tlbflush_timestamp;
    1.41 +    pfn_to_page(smfn)->tlbflush_timestamp = min_max;
    1.42 +
    1.43 +    min = SHADOW_MIN(min_max);
    1.44 +    max = SHADOW_MAX(min_max);
    1.45 +    length = max - min + 1;
    1.46 +    perfc_incr_histo(snapshot_copies, length, PT_UPDATES);
    1.47 +
    1.48 +    min *= sizeof(l1_pgentry_t);
    1.49 +    length *= sizeof(l1_pgentry_t);
    1.50 +
    1.51      original = map_domain_mem(gmfn << PAGE_SHIFT);
    1.52      snapshot = map_domain_mem(smfn << PAGE_SHIFT);
    1.53 -    memcpy(snapshot, original, PAGE_SIZE);
    1.54 +    memcpy(snapshot + min, original + min, length);
    1.55      unmap_domain_mem(original);
    1.56      unmap_domain_mem(snapshot);
    1.57  
    1.58 @@ -2037,8 +2055,7 @@ static int resync_all(struct domain *d, 
    1.59      unsigned long *guest, *shadow, *snapshot;
    1.60      int need_flush = 0, external = shadow_mode_external(d);
    1.61      int unshadow;
    1.62 -    u32 min_max;
    1.63 -    int min, max;
    1.64 +    int changed;
    1.65  
    1.66      ASSERT(spin_is_locked(&d->arch.shadow_lock));
    1.67  
    1.68 @@ -2063,31 +2080,49 @@ static int resync_all(struct domain *d, 
    1.69  
    1.70          switch ( stype ) {
    1.71          case PGT_l1_shadow:
    1.72 -            min_max = pfn_to_page(smfn)->tlbflush_timestamp;
    1.73 -            min = SHADOW_MIN(min_max);
    1.74 -            max = SHADOW_MAX(min_max);
    1.75 -            for ( i = min; i <= max; i++ )
    1.76 +        {
    1.77 +            u32 min_max_shadow = pfn_to_page(smfn)->tlbflush_timestamp;
    1.78 +            int min_shadow = SHADOW_MIN(min_max_shadow);
    1.79 +            int max_shadow = SHADOW_MAX(min_max_shadow);
    1.80 +
    1.81 +            u32 min_max_snapshot =
    1.82 +                pfn_to_page(entry->snapshot_mfn)->tlbflush_timestamp;
    1.83 +            int min_snapshot = SHADOW_MIN(min_max_snapshot);
    1.84 +            int max_snapshot = SHADOW_MAX(min_max_snapshot);
    1.85 +
    1.86 +            changed = 0;
    1.87 +
    1.88 +            for ( i = min_shadow; i <= max_shadow; i++ )
    1.89              {
    1.90 -                unsigned new_pte = guest[i];
    1.91 -                if ( new_pte != snapshot[i] )
    1.92 +                if ( (i < min_snapshot) || (i > max_snapshot) ||
    1.93 +                     (guest[i] != snapshot[i]) )
    1.94                  {
    1.95 -                    need_flush |= validate_pte_change(d, new_pte, &shadow[i]);
    1.96 +                    need_flush |= validate_pte_change(d, guest[i], &shadow[i]);
    1.97  
    1.98                      // can't update snapshots of linear page tables -- they
    1.99                      // are used multiple times...
   1.100                      //
   1.101                      // snapshot[i] = new_pte;
   1.102 +
   1.103 +                    changed++;
   1.104                  }
   1.105              }
   1.106 +            perfc_incrc(resync_l1);
   1.107 +            perfc_incr_histo(wpt_updates, changed, PT_UPDATES);
   1.108 +            perfc_incr_histo(l1_entries_checked, max_shadow - min_shadow + 1, PT_UPDATES);
   1.109              break;
   1.110 +        }
   1.111          case PGT_l2_shadow:
   1.112 -            max = -1;
   1.113 +        {
   1.114 +            int max = -1;
   1.115 +
   1.116 +            changed = 0;
   1.117              for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
   1.118              {
   1.119                  if ( !is_guest_l2_slot(i) && !external )
   1.120                      continue;
   1.121  
   1.122 -                unsigned new_pde = guest[i];
   1.123 +                unsigned long new_pde = guest[i];
   1.124                  if ( new_pde != snapshot[i] )
   1.125                  {
   1.126                      need_flush |= validate_pde_change(d, new_pde, &shadow[i]);
   1.127 @@ -2096,6 +2131,8 @@ static int resync_all(struct domain *d, 
   1.128                      // are used multiple times...
   1.129                      //
   1.130                      // snapshot[i] = new_pde;
   1.131 +
   1.132 +                    changed++;
   1.133                  }
   1.134                  if ( new_pde != 0 )
   1.135                      max = i;
   1.136 @@ -2109,14 +2146,18 @@ static int resync_all(struct domain *d, 
   1.137              }
   1.138              if ( max == -1 )
   1.139                  unshadow = 1;
   1.140 +            perfc_incrc(resync_l2);
   1.141 +            perfc_incr_histo(shm_l2_updates, changed, PT_UPDATES);
   1.142              break;
   1.143 -        default:
   1.144 +        }
   1.145 +        case PGT_hl2_shadow:
   1.146 +            changed = 0;
   1.147              for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
   1.148              {
   1.149                  if ( !is_guest_l2_slot(i) && !external )
   1.150                      continue;
   1.151  
   1.152 -                unsigned new_pde = guest[i];
   1.153 +                unsigned long new_pde = guest[i];
   1.154                  if ( new_pde != snapshot[i] )
   1.155                  {
   1.156                      need_flush |= validate_hl2e_change(d, new_pde, &shadow[i]);
   1.157 @@ -2125,9 +2166,15 @@ static int resync_all(struct domain *d, 
   1.158                      // are used multiple times...
   1.159                      //
   1.160                      // snapshot[i] = new_pde;
   1.161 +
   1.162 +                    changed++;
   1.163                  }
   1.164              }
   1.165 +            perfc_incrc(resync_hl2);
   1.166 +            perfc_incr_histo(shm_hl2_updates, changed, PT_UPDATES);
   1.167              break;
   1.168 +        default:
   1.169 +            BUG();
   1.170          }
   1.171  
   1.172          unmap_domain_mem(shadow);
   1.173 @@ -2176,7 +2223,7 @@ void __shadow_sync_all(struct domain *d)
   1.174          unmap_domain_mem(ppte);
   1.175      }
   1.176  
   1.177 -    // XXX mafetter: SMP perf bug.
   1.178 +    // XXX mafetter: SMP
   1.179      //
   1.180      // With the current algorithm, we've gotta flush all the TLBs
   1.181      // before we can safely continue.  I don't think we want to
   1.182 @@ -2186,7 +2233,10 @@ void __shadow_sync_all(struct domain *d)
   1.183      // (any path from a PTE that grants write access to an out-of-sync
   1.184      // page table page needs to be vcpu private).
   1.185      //
   1.186 -    flush_tlb_all();
   1.187 +#if 0 // this should be enabled for SMP guests...
   1.188 +    flush_tlb_mask(((1 << smp_num_cpus) - 1) & ~(1 << smp_processor_id()));
   1.189 +#endif
   1.190 +    need_flush = 1;
   1.191  
   1.192      // Second, resync all L1 pages, then L2 pages, etc...
   1.193      //
   1.194 @@ -2195,7 +2245,7 @@ void __shadow_sync_all(struct domain *d)
   1.195          need_flush |= resync_all(d, PGT_hl2_shadow);
   1.196      need_flush |= resync_all(d, PGT_l2_shadow);
   1.197  
   1.198 -    if ( need_flush )
   1.199 +    if ( need_flush && !unlikely(shadow_mode_external(d)) )
   1.200          local_flush_tlb();
   1.201  
   1.202      free_out_of_sync_state(d);
     2.1 --- a/xen/include/asm-x86/shadow.h	Thu Mar 24 12:33:04 2005 +0000
     2.2 +++ b/xen/include/asm-x86/shadow.h	Thu Mar 24 16:48:36 2005 +0000
     2.3 @@ -52,9 +52,9 @@
     2.4  #define shadow_lock(_d)      do { ASSERT(!spin_is_locked(&(_d)->arch.shadow_lock)); spin_lock(&(_d)->arch.shadow_lock); } while (0)
     2.5  #define shadow_unlock(_d)    spin_unlock(&(_d)->arch.shadow_lock)
     2.6  
     2.7 -#define SHADOW_ENCODE_MIN_MAX(_min, _max) (((L1_PAGETABLE_ENTRIES - (_max)) << 16) | (_min))
     2.8 +#define SHADOW_ENCODE_MIN_MAX(_min, _max) ((((L1_PAGETABLE_ENTRIES - 1) - (_max)) << 16) | (_min))
     2.9  #define SHADOW_MIN(_encoded) ((_encoded) & ((1u<<16) - 1))
    2.10 -#define SHADOW_MAX(_encoded) (L1_PAGETABLE_ENTRIES - ((_encoded) >> 16))
    2.11 +#define SHADOW_MAX(_encoded) ((L1_PAGETABLE_ENTRIES - 1) - ((_encoded) >> 16))
    2.12  
    2.13  extern void shadow_mode_init(void);
    2.14  extern int shadow_mode_control(struct domain *p, dom0_shadow_control_t *sc);
    2.15 @@ -122,6 +122,8 @@ static void inline
    2.16  
    2.17      if ( d->arch.out_of_sync && __shadow_out_of_sync(ed, va) )
    2.18      {
    2.19 +        perfc_incrc(shadow_sync_va);
    2.20 +
    2.21          // XXX - could be smarter
    2.22          //
    2.23          __shadow_sync_all(ed->domain);
    2.24 @@ -1006,7 +1008,8 @@ static inline unsigned long __shadow_sta
    2.25   * Either returns PGT_none, or PGT_l{1,2,3,4}_page_table.
    2.26   */
    2.27  static inline unsigned long
    2.28 -shadow_max_pgtable_type(struct domain *d, unsigned long gpfn)
    2.29 +shadow_max_pgtable_type(struct domain *d, unsigned long gpfn,
    2.30 +                        unsigned long *smfn)
    2.31  {
    2.32      struct shadow_status *x;
    2.33      unsigned long pttype = PGT_none, type;
    2.34 @@ -1014,6 +1017,8 @@ shadow_max_pgtable_type(struct domain *d
    2.35      ASSERT(spin_is_locked(&d->arch.shadow_lock));
    2.36      ASSERT(gpfn == (gpfn & PGT_mfn_mask));
    2.37  
    2.38 +    perfc_incrc(shadow_max_type);
    2.39 +
    2.40      x = hash_bucket(d, gpfn);
    2.41  
    2.42      while ( x && x->gpfn_and_flags )
    2.43 @@ -1044,7 +1049,11 @@ shadow_max_pgtable_type(struct domain *d
    2.44              }
    2.45  
    2.46              if ( type > pttype )
    2.47 +            {
    2.48                  pttype = type;
    2.49 +                if ( smfn )
    2.50 +                    *smfn = x->smfn;
    2.51 +            }
    2.52          }
    2.53      next:
    2.54          x = x->next;
     3.1 --- a/xen/include/xen/perfc_defn.h	Thu Mar 24 12:33:04 2005 +0000
     3.2 +++ b/xen/include/xen/perfc_defn.h	Thu Mar 24 16:48:36 2005 +0000
     3.3 @@ -2,6 +2,10 @@
     3.4  #define PERFC_PT_UPDATES_BUCKET_SIZE 3
     3.5  PERFCOUNTER_ARRAY( wpt_updates, "writable pt updates", PERFC_MAX_PT_UPDATES )
     3.6  PERFCOUNTER_ARRAY( bpt_updates, "batched pt updates", PERFC_MAX_PT_UPDATES )
     3.7 +PERFCOUNTER_ARRAY( l1_entries_checked, "l1 entries checked", PERFC_MAX_PT_UPDATES )
     3.8 +PERFCOUNTER_ARRAY( shm_l2_updates, "shadow mode L2 pt updates", PERFC_MAX_PT_UPDATES )
     3.9 +PERFCOUNTER_ARRAY( shm_hl2_updates, "shadow mode HL2 pt updates", PERFC_MAX_PT_UPDATES )
    3.10 +PERFCOUNTER_ARRAY( snapshot_copies, "entries copied per snapshot", PERFC_MAX_PT_UPDATES )
    3.11  
    3.12  PERFCOUNTER_ARRAY( hypercalls, "hypercalls", NR_hypercalls )
    3.13  PERFCOUNTER_ARRAY( exceptions, "exceptions", 32 )
    3.14 @@ -31,12 +35,21 @@ PERFCOUNTER_CPU( num_page_updates, "num_
    3.15  PERFCOUNTER_CPU( calls_to_update_va, "calls_to_update_va_map" )
    3.16  PERFCOUNTER_CPU( page_faults, "page faults" )
    3.17  PERFCOUNTER_CPU( copy_user_faults, "copy_user faults" )
    3.18 +
    3.19 +PERFCOUNTER_CPU(shadow_fault_calls,                "calls to shadow_fault")
    3.20 +PERFCOUNTER_CPU(shadow_fault_bail_pde_not_present, "sf bailed due to pde not present")
    3.21 +PERFCOUNTER_CPU(shadow_fault_bail_pte_not_present, "sf bailed due to pte not present")
    3.22 +PERFCOUNTER_CPU(shadow_fault_bail_ro_mapping,      "sf bailed due to a ro mapping")
    3.23 +PERFCOUNTER_CPU(shadow_fault_fixed,                "sf fixed the pgfault")
    3.24 +PERFCOUNTER_CPU(write_fault_bail,                  "sf bailed due to write_fault")
    3.25 +PERFCOUNTER_CPU(read_fault_bail,                   "sf bailed due to read_fault")
    3.26 +
    3.27  PERFCOUNTER_CPU( map_domain_mem_count, "map_domain_mem count" )
    3.28  
    3.29 -PERFCOUNTER_CPU( shadow_l2_table_count, "shadow_l2_table count" )
    3.30 -PERFCOUNTER_CPU( shadow_l1_table_count, "shadow_l1_table count" )
    3.31 -PERFCOUNTER_CPU( unshadow_table_count, "unshadow_table count" )
    3.32 -PERFCOUNTER_CPU( shadow_fixup_count, "shadow_fixup count" )
    3.33 +PERFCOUNTER_CPU( shadow_l2_table_count,  "shadow_l2_table count" )
    3.34 +PERFCOUNTER_CPU( shadow_l1_table_count,  "shadow_l1_table count" )
    3.35 +PERFCOUNTER_CPU( unshadow_table_count,   "unshadow_table count" )
    3.36 +PERFCOUNTER_CPU( shadow_fixup_count,     "shadow_fixup count" )
    3.37  PERFCOUNTER_CPU( shadow_update_va_fail1, "shadow_update_va_fail1" )
    3.38  PERFCOUNTER_CPU( shadow_update_va_fail2, "shadow_update_va_fail2" )
    3.39  
    3.40 @@ -62,8 +75,13 @@ PERFCOUNTER_CPU(shadow_status_shortcut, 
    3.41  PERFCOUNTER_CPU(shadow_status_calls,    "calls to ___shadow_status" )
    3.42  PERFCOUNTER_CPU(shadow_status_miss,     "missed shadow cache" )
    3.43  PERFCOUNTER_CPU(shadow_status_hit_head, "hits on head of bucket" )
    3.44 +PERFCOUNTER_CPU(shadow_max_type,        "calls to shadow_max_type" )
    3.45  
    3.46  PERFCOUNTER_CPU(shadow_sync_all,                   "calls to shadow_sync_all")
    3.47 +PERFCOUNTER_CPU(shadow_sync_va,                    "calls to shadow_sync_va")
    3.48 +PERFCOUNTER_CPU(resync_l1,                         "resync L1 page")
    3.49 +PERFCOUNTER_CPU(resync_l2,                         "resync L2 page")
    3.50 +PERFCOUNTER_CPU(resync_hl2,                        "resync HL2 page")
    3.51  PERFCOUNTER_CPU(shadow_make_snapshot,              "snapshots created")
    3.52  PERFCOUNTER_CPU(shadow_mark_mfn_out_of_sync_calls, "calls to shadow_mk_out_of_sync")
    3.53  PERFCOUNTER_CPU(shadow_out_of_sync_calls,          "calls to shadow_out_of_sync")
    3.54 @@ -71,11 +89,6 @@ PERFCOUNTER_CPU(extra_va_update_sync,   
    3.55  PERFCOUNTER_CPU(snapshot_entry_matches_calls,      "calls to ss_entry_matches")
    3.56  PERFCOUNTER_CPU(snapshot_entry_matches_true,       "ss_entry_matches returns true")
    3.57  
    3.58 -PERFCOUNTER_CPU(shadow_fault_calls,                "calls to shadow_fault")
    3.59 -PERFCOUNTER_CPU(shadow_fault_bail_pde_not_present, "sf bailed due to pde not present")
    3.60 -PERFCOUNTER_CPU(shadow_fault_bail_pte_not_present, "sf bailed due to pte not present")
    3.61 -PERFCOUNTER_CPU(shadow_fault_bail_ro_mapping,      "sf bailed due to a ro mapping")
    3.62 -PERFCOUNTER_CPU(shadow_fault_fixed,                "sf fixed the pgfault")
    3.63  PERFCOUNTER_CPU(validate_pte_calls,                "calls to validate_pte_change")
    3.64  PERFCOUNTER_CPU(validate_pte_changes,              "validate_pte makes changes")
    3.65  PERFCOUNTER_CPU(validate_pde_calls,                "calls to validate_pde_change")
    3.66 @@ -92,6 +105,4 @@ PERFCOUNTER_CPU(remove_write_not_writabl
    3.67  PERFCOUNTER_CPU(remove_write_fast_exit,            "remove_write hit predicted entry")
    3.68  PERFCOUNTER_CPU(remove_write_predicted,            "remove_write predict hit&exit")
    3.69  PERFCOUNTER_CPU(remove_write_bad_prediction,       "remove_write bad prediction")
    3.70 -PERFCOUNTER_CPU(write_fault_bail,                  "sf bailed due to write_fault")
    3.71 -PERFCOUNTER_CPU(read_fault_bail,                   "sf bailed due to read_fault")
    3.72 -PERFCOUNTER_CPU(update_hl2e_invlpg,                 "update_hl2e calls invlpg")
    3.73 +PERFCOUNTER_CPU(update_hl2e_invlpg,                "update_hl2e calls invlpg")