direct-io.hg

changeset 834:d44118178696

bitkeeper revision 1.516 (3f8c15d525UM6tJE0aWBdRe-3P9cVw)

memory.c, traps.c:
Preload the first page of the guest LDT when the shadow mapping is invalidated for any reason.
author kaf24@scramble.cl.cam.ac.uk
date Tue Oct 14 15:27:17 2003 +0000 (2003-10-14)
parents 6d0f37cc9107
children 6dfc4a422bbd
files xen/arch/i386/traps.c xen/common/memory.c
line diff
     1.1 --- a/xen/arch/i386/traps.c	Tue Oct 14 14:18:49 2003 +0000
     1.2 +++ b/xen/arch/i386/traps.c	Tue Oct 14 15:27:17 2003 +0000
     1.3 @@ -286,10 +286,9 @@ asmlinkage void do_page_fault(struct pt_
     1.4  {
     1.5      struct guest_trap_bounce *gtb = guest_trap_bounce+smp_processor_id();
     1.6      trap_info_t *ti;
     1.7 -    unsigned long addr, off, fixup, l1e, *ldt_page;
     1.8 +    unsigned long addr, fixup;
     1.9      struct task_struct *p = current;
    1.10 -    struct pfn_info *page;
    1.11 -    int i;
    1.12 +    extern int map_ldt_shadow_page(unsigned int);
    1.13  
    1.14      __asm__ __volatile__ ("movl %%cr2,%0" : "=r" (addr) : );
    1.15  
    1.16 @@ -320,57 +319,11 @@ asmlinkage void do_page_fault(struct pt_
    1.17   fault_in_xen_space:
    1.18  
    1.19      if ( (addr < LDT_VIRT_START) || 
    1.20 -         (addr >= (LDT_VIRT_START + (p->mm.ldt_ents*LDT_ENTRY_SIZE))) )
    1.21 +         (addr >= (LDT_VIRT_START + (p->mm.ldt_ents*LDT_ENTRY_SIZE))) ||
    1.22 +         map_ldt_shadow_page((addr - LDT_VIRT_START) >> PAGE_SHIFT) )
    1.23          goto propagate_fault;
    1.24 -
    1.25 -    off  = addr - LDT_VIRT_START;
    1.26 -    addr = p->mm.ldt_base + off;
    1.27 -
    1.28 -    spin_lock(&p->page_lock);
    1.29 -
    1.30 -    __get_user(l1e, (unsigned long *)(linear_pg_table+(addr>>PAGE_SHIFT)));
    1.31 -    if ( !(l1e & _PAGE_PRESENT) )
    1.32 -        goto unlock_and_propagate_fault;
    1.33 -
    1.34 -    page = frame_table + (l1e >> PAGE_SHIFT);
    1.35 -    if ( (page->flags & PG_type_mask) != PGT_ldt_page )
    1.36 -    {
    1.37 -        if ( page->type_count != 0 )
    1.38 -            goto unlock_and_propagate_fault;
    1.39 -
    1.40 -        /* Check all potential LDT entries in the page. */
    1.41 -        ldt_page = (unsigned long *)(addr & PAGE_MASK);
    1.42 -        for ( i = 0; i < 512; i++ )
    1.43 -            if ( !check_descriptor(ldt_page[i*2], ldt_page[i*2+1]) )
    1.44 -                goto unlock_and_propagate_fault;
    1.45 -
    1.46 -        if ( page->flags & PG_need_flush )
    1.47 -        {
    1.48 -            perfc_incrc(need_flush_tlb_flush);
    1.49 -            local_flush_tlb();
    1.50 -            page->flags &= ~PG_need_flush;
    1.51 -        }
    1.52 -
    1.53 -        page->flags &= ~PG_type_mask;
    1.54 -        page->flags |= PGT_ldt_page;
    1.55 -    }
    1.56 -
    1.57 -    /* Success! */
    1.58 -    get_page_type(page);
    1.59 -    get_page_tot(page);
    1.60 -    p->mm.perdomain_pt[l1_table_offset(off)+16] = mk_l1_pgentry(l1e|_PAGE_RW);
    1.61 -    p->mm.shadow_ldt_mapcnt++;
    1.62 -
    1.63 -    spin_unlock(&p->page_lock);
    1.64      return;
    1.65  
    1.66 -
    1.67 - unlock_and_propagate_fault:
    1.68 -
    1.69 -    spin_unlock(&p->page_lock);
    1.70 -    goto propagate_fault;
    1.71 -
    1.72 -
    1.73   fault_in_hypervisor:
    1.74  
    1.75      if ( likely((fixup = search_exception_table(regs->eip)) != 0) )
     2.1 --- a/xen/common/memory.c	Tue Oct 14 14:18:49 2003 +0000
     2.2 +++ b/xen/common/memory.c	Tue Oct 14 15:27:17 2003 +0000
     2.3 @@ -172,8 +172,10 @@ spinlock_t free_list_lock = SPIN_LOCK_UN
     2.4  unsigned int free_pfns;
     2.5  
     2.6  /* Used to defer flushing of memory structures. */
     2.7 -static int flush_tlb[NR_CPUS] __cacheline_aligned;
     2.8 -
     2.9 +static struct {
    2.10 +    int flush_tlb;
    2.11 +    int refresh_ldt;
    2.12 +} deferred_op[NR_CPUS] __cacheline_aligned;
    2.13  
    2.14  /*
    2.15   * init_frametable:
    2.16 @@ -186,7 +188,7 @@ void __init init_frametable(unsigned lon
    2.17      unsigned long page_index;
    2.18      unsigned long flags;
    2.19  
    2.20 -    memset(flush_tlb, 0, sizeof(flush_tlb));
    2.21 +    memset(deferred_op, 0, sizeof(deferred_op));
    2.22  
    2.23      max_page = nr_pages;
    2.24      frame_table_size = nr_pages * sizeof(struct pfn_info);
    2.25 @@ -213,7 +215,7 @@ void __init init_frametable(unsigned lon
    2.26  
    2.27  static void __invalidate_shadow_ldt(void)
    2.28  {
    2.29 -    int i;
    2.30 +    int i, cpu = smp_processor_id();
    2.31      unsigned long pfn;
    2.32      struct pfn_info *page;
    2.33      
    2.34 @@ -233,7 +235,8 @@ static void __invalidate_shadow_ldt(void
    2.35      }
    2.36  
    2.37      /* Dispose of the (now possibly invalid) mappings from the TLB.  */
    2.38 -    flush_tlb[smp_processor_id()] = 1;
    2.39 +    deferred_op[cpu].flush_tlb   = 1;
    2.40 +    deferred_op[cpu].refresh_ldt = 1;
    2.41  }
    2.42  
    2.43  
    2.44 @@ -244,6 +247,58 @@ static inline void invalidate_shadow_ldt
    2.45  }
    2.46  
    2.47  
    2.48 +/* Map shadow page at offset @off. Returns 0 on success. */
    2.49 +int map_ldt_shadow_page(unsigned int off)
    2.50 +{
    2.51 +    struct task_struct *p = current;
    2.52 +    unsigned long addr = p->mm.ldt_base + (off << PAGE_SHIFT);
    2.53 +    unsigned long l1e, *ldt_page;
    2.54 +    struct pfn_info *page;
    2.55 +    int i, ret = -1;
    2.56 +
    2.57 +    spin_lock(&p->page_lock);
    2.58 +
    2.59 +    __get_user(l1e, (unsigned long *)(linear_pg_table+(addr>>PAGE_SHIFT)));
    2.60 +    if ( unlikely(!(l1e & _PAGE_PRESENT)) )
    2.61 +        goto out;
    2.62 +
    2.63 +    page = frame_table + (l1e >> PAGE_SHIFT);
    2.64 +    if ( unlikely((page->flags & PG_type_mask) != PGT_ldt_page) )
    2.65 +    {
    2.66 +        if ( unlikely(page->type_count != 0) )
    2.67 +            goto out;
    2.68 +
    2.69 +        /* Check all potential LDT entries in the page. */
    2.70 +        ldt_page = (unsigned long *)addr;
    2.71 +        for ( i = 0; i < 512; i++ )
    2.72 +            if ( unlikely(!check_descriptor(ldt_page[i*2], ldt_page[i*2+1])) )
    2.73 +                goto out;
    2.74 +
    2.75 +        if ( unlikely(page->flags & PG_need_flush) )
    2.76 +        {
    2.77 +            perfc_incrc(need_flush_tlb_flush);
    2.78 +            __write_cr3_counted(pagetable_val(p->mm.pagetable));
    2.79 +            page->flags &= ~PG_need_flush;
    2.80 +        }
    2.81 +
    2.82 +        page->flags &= ~PG_type_mask;
    2.83 +        page->flags |= PGT_ldt_page;
    2.84 +    }
    2.85 +
    2.86 +    /* Success! */
    2.87 +    get_page_type(page);
    2.88 +    get_page_tot(page);
    2.89 +    p->mm.perdomain_pt[l1_table_offset(off)+16] = mk_l1_pgentry(l1e|_PAGE_RW);
    2.90 +    p->mm.shadow_ldt_mapcnt++;
    2.91 +
    2.92 +    ret = 0;
    2.93 +
    2.94 + out:
    2.95 +    spin_unlock(&p->page_lock);
    2.96 +    return ret;
    2.97 +}
    2.98 +
    2.99 +
   2.100  /* Return original refcnt, or -1 on error. */
   2.101  static int inc_page_refcnt(unsigned long page_nr, unsigned int type)
   2.102  {
   2.103 @@ -274,7 +329,7 @@ static int inc_page_refcnt(unsigned long
   2.104  
   2.105          if ( unlikely(flags & PG_need_flush) )
   2.106          {
   2.107 -            flush_tlb[smp_processor_id()] = 1;
   2.108 +            deferred_op[smp_processor_id()].flush_tlb = 1;
   2.109              page->flags &= ~PG_need_flush;
   2.110              perfc_incrc(need_flush_tlb_flush);
   2.111          }
   2.112 @@ -628,7 +683,7 @@ static int mod_l1_entry(l1_pgentry_t *p_
   2.113  
   2.114  static int do_extended_command(unsigned long ptr, unsigned long val)
   2.115  {
   2.116 -    int err = 0;
   2.117 +    int err = 0, cpu = smp_processor_id();
   2.118      unsigned int cmd = val & PGEXT_CMD_MASK;
   2.119      unsigned long pfn = ptr >> PAGE_SHIFT;
   2.120      struct pfn_info *page = frame_table + pfn;
   2.121 @@ -694,7 +749,7 @@ static int do_extended_command(unsigned 
   2.122              put_l2_table(pagetable_val(current->mm.pagetable) >> PAGE_SHIFT);
   2.123              current->mm.pagetable = mk_pagetable(pfn << PAGE_SHIFT);
   2.124              invalidate_shadow_ldt();
   2.125 -            flush_tlb[smp_processor_id()] = 1;
   2.126 +            deferred_op[cpu].flush_tlb = 1;
   2.127          }
   2.128          else
   2.129          {
   2.130 @@ -703,7 +758,7 @@ static int do_extended_command(unsigned 
   2.131          break;
   2.132          
   2.133      case PGEXT_TLB_FLUSH:
   2.134 -        flush_tlb[smp_processor_id()] = 1;
   2.135 +        deferred_op[cpu].flush_tlb = 1;
   2.136          break;
   2.137      
   2.138      case PGEXT_INVLPG:
   2.139 @@ -729,6 +784,7 @@ static int do_extended_command(unsigned 
   2.140              current->mm.ldt_base = ptr;
   2.141              current->mm.ldt_ents = ents;
   2.142              load_LDT(current);
   2.143 +            deferred_op[cpu].refresh_ldt = (ents != 0);
   2.144          }
   2.145          break;
   2.146      }
   2.147 @@ -748,7 +804,7 @@ int do_process_page_updates(page_update_
   2.148      page_update_request_t req;
   2.149      unsigned long flags, pfn, l1e;
   2.150      struct pfn_info *page;
   2.151 -    int err = 0, i;
   2.152 +    int err = 0, i, cpu = smp_processor_id();
   2.153      unsigned int cmd;
   2.154      unsigned long cr0 = 0;
   2.155  
   2.156 @@ -884,11 +940,16 @@ int do_process_page_updates(page_update_
   2.157          ureqs++;
   2.158      }
   2.159  
   2.160 -    if ( flush_tlb[smp_processor_id()] )
   2.161 +    if ( deferred_op[cpu].flush_tlb )
   2.162      {
   2.163 -        flush_tlb[smp_processor_id()] = 0;
   2.164 +        deferred_op[cpu].flush_tlb = 0;
   2.165          __write_cr3_counted(pagetable_val(current->mm.pagetable));
   2.166 +    }
   2.167  
   2.168 +    if ( deferred_op[cpu].refresh_ldt )
   2.169 +    {
   2.170 +        deferred_op[cpu].refresh_ldt = 0;
   2.171 +        (void)map_ldt_shadow_page(0);
   2.172      }
   2.173  
   2.174      if ( cr0 != 0 )