direct-io.hg

changeset 799:3d5ab61162dc

bitkeeper revision 1.488 (3f831546g57eNx40G9Qp-XWxciMP9Q)

Many files:
Remove CPU-dependent page-directory entries. A singl epage table can now be used simultaneously by several CPUs.
author kaf24@scramble.cl.cam.ac.uk
date Tue Oct 07 19:34:30 2003 +0000 (2003-10-07)
parents 101f79f13c44
children b33875f82bbb
files xen/arch/i386/boot/boot.S xen/arch/i386/ioremap.c xen/arch/i386/mm.c xen/arch/i386/setup.c xen/arch/i386/smpboot.c xen/arch/i386/traps.c xen/common/domain.c xen/common/domain_page.c xen/common/memory.c xen/include/asm-i386/domain_page.h xen/include/asm-i386/page.h xen/include/xeno/sched.h
line diff
     1.1 --- a/xen/arch/i386/boot/boot.S	Tue Oct 07 16:18:34 2003 +0000
     1.2 +++ b/xen/arch/i386/boot/boot.S	Tue Oct 07 19:34:30 2003 +0000
     1.3 @@ -110,7 +110,7 @@ 1:      sub     $4,%eax                 
     1.4  skip_dom0_copy:              
     1.5  
     1.6          /* Initialize low and high mappings of all memory with 4MB pages */
     1.7 -        mov     $idle0_pg_table-__PAGE_OFFSET,%edi
     1.8 +        mov     $idle_pg_table-__PAGE_OFFSET,%edi
     1.9          mov     $0x1e3,%eax                  /* PRESENT+RW+A+D+4MB+GLOBAL */
    1.10  1:      mov     %eax,__PAGE_OFFSET>>20(%edi) /* high mapping */
    1.11          stosl                                /* low mapping */
    1.12 @@ -127,7 +127,7 @@ 1:      mov     %eax,__PAGE_OFFSET>>20(%
    1.13          ud2     /* Force a panic (invalid opcode). */
    1.14  
    1.15  start_paging:
    1.16 -        mov     $idle0_pg_table-__PAGE_OFFSET,%eax
    1.17 +        mov     $idle_pg_table-__PAGE_OFFSET,%eax
    1.18          mov     %eax,%cr3
    1.19          mov     %cr0,%eax
    1.20          or      $0x80010000,%eax /* set PG and WP bits */
    1.21 @@ -254,9 +254,9 @@ ENTRY(gdt_table)
    1.22          .quad 0x0000000000000000     /* unused                           */
    1.23          .fill 2*NR_CPUS,8,0          /* space for TSS and LDT per CPU    */
    1.24  
    1.25 -# The following adds 12kB to the kernel file size.
    1.26 +# The following adds 8-12kB to the kernel file size.
    1.27          .org 0x1000
    1.28 -ENTRY(idle0_pg_table)
    1.29 +ENTRY(idle_pg_table)
    1.30          .org 0x2000
    1.31  ENTRY(cpu0_stack)
    1.32          .org 0x3000
     2.1 --- a/xen/arch/i386/ioremap.c	Tue Oct 07 16:18:34 2003 +0000
     2.2 +++ b/xen/arch/i386/ioremap.c	Tue Oct 07 19:34:30 2003 +0000
     2.3 @@ -82,7 +82,7 @@ void * __ioremap(unsigned long phys_addr
     2.4       */
     2.5      vaddr = remap_base;
     2.6      remap_base += size;
     2.7 -    pl2e = idle0_pg_table + l2_table_offset(vaddr);
     2.8 +    pl2e = &idle_pg_table[l2_table_offset(vaddr)];
     2.9      if ( l2_pgentry_empty(*pl2e) ) new_l2e(pl2e);
    2.10      pl1e = l2_pgentry_to_l1(*pl2e++) + l1_table_offset(vaddr);
    2.11      for ( ; ; ) 
     3.1 --- a/xen/arch/i386/mm.c	Tue Oct 07 16:18:34 2003 +0000
     3.2 +++ b/xen/arch/i386/mm.c	Tue Oct 07 19:34:30 2003 +0000
     3.3 @@ -33,7 +33,7 @@ static inline void set_pte_phys (unsigne
     3.4      l2_pgentry_t *l2ent;
     3.5      l1_pgentry_t *l1ent;
     3.6  
     3.7 -    l2ent = idle0_pg_table + l2_table_offset(vaddr);
     3.8 +    l2ent = &idle_pg_table[l2_table_offset(vaddr)];
     3.9      l1ent = l2_pgentry_to_l1(*l2ent) + l1_table_offset(vaddr);
    3.10      *l1ent = entry;
    3.11  
    3.12 @@ -89,31 +89,37 @@ void __init paging_init(void)
    3.13       * created - mappings will be set by set_fixmap():
    3.14       */
    3.15      addr = FIXADDR_START & ~((1<<L2_PAGETABLE_SHIFT)-1);
    3.16 -    fixrange_init(addr, 0, idle0_pg_table);
    3.17 +    fixrange_init(addr, 0, idle_pg_table);
    3.18  
    3.19      /* Create page table for ioremap(). */
    3.20      ioremap_pt = (void *)get_free_page(GFP_KERNEL);
    3.21      clear_page(ioremap_pt);
    3.22 -    idle0_pg_table[IOREMAP_VIRT_START >> L2_PAGETABLE_SHIFT] = 
    3.23 +    idle_pg_table[IOREMAP_VIRT_START >> L2_PAGETABLE_SHIFT] = 
    3.24          mk_l2_pgentry(__pa(ioremap_pt) | __PAGE_HYPERVISOR);
    3.25  
    3.26      /* Create read-only mapping of MPT for guest-OS use. */
    3.27 -    idle0_pg_table[READONLY_MPT_VIRT_START >> L2_PAGETABLE_SHIFT] =
    3.28 -        idle0_pg_table[RDWR_MPT_VIRT_START >> L2_PAGETABLE_SHIFT];
    3.29 -    mk_l2_readonly(idle0_pg_table + 
    3.30 +    idle_pg_table[READONLY_MPT_VIRT_START >> L2_PAGETABLE_SHIFT] =
    3.31 +        idle_pg_table[RDWR_MPT_VIRT_START >> L2_PAGETABLE_SHIFT];
    3.32 +    mk_l2_readonly(idle_pg_table + 
    3.33                     (READONLY_MPT_VIRT_START >> L2_PAGETABLE_SHIFT));
    3.34 +
    3.35 +    /* Set up mapping cache for domain pages. */
    3.36 +    mapcache = (unsigned long *)get_free_page(GFP_KERNEL);
    3.37 +    clear_page(mapcache);
    3.38 +    idle_pg_table[MAPCACHE_VIRT_START >> L2_PAGETABLE_SHIFT] =
    3.39 +        mk_l2_pgentry(__pa(mapcache) | __PAGE_HYPERVISOR);
    3.40 +
    3.41 +    /* Set up linear page table mapping. */
    3.42 +    idle_pg_table[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] =
    3.43 +        mk_l2_pgentry(__pa(idle_pg_table) | __PAGE_HYPERVISOR);
    3.44 +
    3.45  }
    3.46  
    3.47  void __init zap_low_mappings (void)
    3.48  {
    3.49 -    int i, j;
    3.50 -    for ( i = 0; i < smp_num_cpus; i++ )
    3.51 -    {
    3.52 -        for ( j = 0; j < DOMAIN_ENTRIES_PER_L2_PAGETABLE; j++ )
    3.53 -        {
    3.54 -            idle_pg_table[i][j] = mk_l2_pgentry(0);
    3.55 -        }
    3.56 -    }
    3.57 +    int i;
    3.58 +    for ( i = 0; i < DOMAIN_ENTRIES_PER_L2_PAGETABLE; i++ )
    3.59 +        idle_pg_table[i] = mk_l2_pgentry(0);
    3.60      flush_tlb_all();
    3.61  }
    3.62  
     4.1 --- a/xen/arch/i386/setup.c	Tue Oct 07 16:18:34 2003 +0000
     4.2 +++ b/xen/arch/i386/setup.c	Tue Oct 07 19:34:30 2003 +0000
     4.3 @@ -18,13 +18,8 @@ struct cpuinfo_x86 boot_cpu_data = { 0 }
     4.4  unsigned long mmu_cr4_features = X86_CR4_PSE | X86_CR4_PGE;
     4.5  unsigned long wait_init_idle;
     4.6  
     4.7 -/* Basic page table for each CPU in the system. */
     4.8 -l2_pgentry_t *idle_pg_table[NR_CPUS] = { idle0_pg_table };
     4.9  struct task_struct *idle_task[NR_CPUS] = { &idle0_task };
    4.10  
    4.11 -/* for asm/domain_page.h, map_domain_page() */
    4.12 -unsigned long *mapcache[NR_CPUS];
    4.13 -
    4.14  int phys_proc_id[NR_CPUS];
    4.15  int logical_proc_id[NR_CPUS];
    4.16  
    4.17 @@ -234,7 +229,6 @@ void __init cpu_init(void)
    4.18  {
    4.19      int nr = smp_processor_id();
    4.20      struct tss_struct * t = &init_tss[nr];
    4.21 -    l2_pgentry_t *pl2e;
    4.22  
    4.23      if ( test_and_set_bit(nr, &cpu_initialized) )
    4.24          panic("CPU#%d already initialized!!!\n", nr);
    4.25 @@ -268,16 +262,6 @@ void __init cpu_init(void)
    4.26      __asm__ __volatile__ ("movl %%eax,%%cr3"
    4.27                            : : "a" (pagetable_val(current->mm.pagetable)));
    4.28  
    4.29 -    /* Set up mapping cache for domain pages. */
    4.30 -    pl2e = idle_pg_table[nr] + (MAPCACHE_VIRT_START >> L2_PAGETABLE_SHIFT);
    4.31 -    mapcache[nr] = (unsigned long *)get_free_page(GFP_KERNEL);
    4.32 -    clear_page(mapcache[nr]);
    4.33 -    *pl2e = mk_l2_pgentry(__pa(mapcache[nr]) | __PAGE_HYPERVISOR);
    4.34 -
    4.35 -    /* Set up linear page table mapping. */
    4.36 -    idle_pg_table[nr][LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] =
    4.37 -        mk_l2_pgentry(__pa(idle_pg_table[nr]) | __PAGE_HYPERVISOR);
    4.38 -
    4.39      init_idle_task();
    4.40  }
    4.41  
     5.1 --- a/xen/arch/i386/smpboot.c	Tue Oct 07 16:18:34 2003 +0000
     5.2 +++ b/xen/arch/i386/smpboot.c	Tue Oct 07 19:34:30 2003 +0000
     5.3 @@ -666,17 +666,13 @@ static void __init do_boot_cpu (int apic
     5.4      unsigned long boot_error = 0;
     5.5      int timeout, cpu;
     5.6      unsigned long start_eip;
     5.7 -    l2_pgentry_t *pagetable;
     5.8  
     5.9      cpu = ++cpucount;
    5.10  
    5.11      if ( (idle = do_newdomain(IDLE_DOMAIN_ID, cpu)) == NULL )
    5.12          panic("failed 'newdomain' for CPU %d", cpu);
    5.13   
    5.14 -    pagetable = (void *)get_free_page(GFP_KERNEL);
    5.15 -    memcpy(pagetable, idle0_pg_table, PAGE_SIZE);
    5.16 -    idle_pg_table[cpu] = pagetable;
    5.17 -    idle->mm.pagetable = mk_pagetable(__pa(pagetable));
    5.18 +    idle->mm.pagetable = mk_pagetable(__pa(idle_pg_table));
    5.19  
    5.20      map_cpu_to_boot_apicid(cpu, apicid);
    5.21  
    5.22 @@ -687,7 +683,7 @@ static void __init do_boot_cpu (int apic
    5.23      /* start_eip had better be page-aligned! */
    5.24      start_eip = setup_trampoline();
    5.25  
    5.26 -    /* So we see what's up   */
    5.27 +    /* So we see what's up. */
    5.28      printk("Booting processor %d/%d eip %lx\n", cpu, apicid, start_eip);
    5.29      stack_start.esp = __pa(get_free_page(GFP_KERNEL)) + 4000;
    5.30  
     6.1 --- a/xen/arch/i386/traps.c	Tue Oct 07 16:18:34 2003 +0000
     6.2 +++ b/xen/arch/i386/traps.c	Tue Oct 07 19:34:30 2003 +0000
     6.3 @@ -398,9 +398,7 @@ asmlinkage void do_page_fault(struct pt_
     6.4      if ( addr >= PAGE_OFFSET )
     6.5      {
     6.6          unsigned long page;
     6.7 -        unsigned long *pde;
     6.8 -        pde = (unsigned long *)idle_pg_table[smp_processor_id()];
     6.9 -        page = pde[addr >> L2_PAGETABLE_SHIFT];
    6.10 +        page = l2_pgentry_val(idle_pg_table[addr >> L2_PAGETABLE_SHIFT]);
    6.11          printk("*pde = %08lx\n", page);
    6.12          if ( page & _PAGE_PRESENT )
    6.13          {
    6.14 @@ -683,7 +681,7 @@ void __init trap_init(void)
    6.15      tss->ss     = __HYPERVISOR_DS;
    6.16      tss->esp    = (unsigned long)
    6.17          &doublefault_stack[DOUBLEFAULT_STACK_SIZE];
    6.18 -    tss->__cr3  = __pa(idle0_pg_table);
    6.19 +    tss->__cr3  = __pa(idle_pg_table);
    6.20      tss->cs     = __HYPERVISOR_CS;
    6.21      tss->eip    = (unsigned long)do_double_fault;
    6.22      tss->eflags = 2;
     7.1 --- a/xen/common/domain.c	Tue Oct 07 16:18:34 2003 +0000
     7.2 +++ b/xen/common/domain.c	Tue Oct 07 19:34:30 2003 +0000
     7.3 @@ -360,9 +360,8 @@ int final_setup_guestos(struct task_stru
     7.4       */
     7.5      phys_l2tab = meminfo->l2_pgt_addr;
     7.6      l2tab = map_domain_mem(phys_l2tab); 
     7.7 -    memcpy(l2tab + DOMAIN_ENTRIES_PER_L2_PAGETABLE, 
     7.8 -        ((l2_pgentry_t *)idle_pg_table[p->processor]) + 
     7.9 -        DOMAIN_ENTRIES_PER_L2_PAGETABLE, 
    7.10 +    memcpy(&l2tab[DOMAIN_ENTRIES_PER_L2_PAGETABLE], 
    7.11 +        &idle_pg_table[DOMAIN_ENTRIES_PER_L2_PAGETABLE],
    7.12          (ENTRIES_PER_L2_PAGETABLE - DOMAIN_ENTRIES_PER_L2_PAGETABLE) 
    7.13          * sizeof(l2_pgentry_t));
    7.14      l2tab[PERDOMAIN_VIRT_START >> L2_PAGETABLE_SHIFT] = 
    7.15 @@ -541,7 +540,7 @@ int setup_guestos(struct task_struct *p,
    7.16       */
    7.17      phys_l2tab = alloc_page_from_domain(&alloc_address, &alloc_index);
    7.18      l2start = l2tab = map_domain_mem(phys_l2tab);
    7.19 -    memcpy(l2tab, idle_pg_table[p->processor], PAGE_SIZE);
    7.20 +    memcpy(l2tab, &idle_pg_table[0], PAGE_SIZE);
    7.21      l2tab[PERDOMAIN_VIRT_START >> L2_PAGETABLE_SHIFT] =
    7.22          mk_l2_pgentry(__pa(p->mm.perdomain_pt) | __PAGE_HYPERVISOR);
    7.23      l2tab[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] =
     8.1 --- a/xen/common/domain_page.c	Tue Oct 07 16:18:34 2003 +0000
     8.2 +++ b/xen/common/domain_page.c	Tue Oct 07 19:34:30 2003 +0000
     8.3 @@ -18,14 +18,16 @@
     8.4  #include <asm/domain_page.h>
     8.5  #include <asm/pgalloc.h>
     8.6  
     8.7 -static unsigned int map_idx[NR_CPUS];
     8.8 +unsigned long *mapcache;
     8.9 +static unsigned int map_idx, shadow_map_idx[NR_CPUS];
    8.10 +static spinlock_t map_lock = SPIN_LOCK_UNLOCKED;
    8.11  
    8.12  /* Use a spare PTE bit to mark entries ready for recycling. */
    8.13  #define READY_FOR_TLB_FLUSH (1<<10)
    8.14  
    8.15  static void flush_all_ready_maps(void)
    8.16  {
    8.17 -    unsigned long *cache = mapcache[smp_processor_id()];
    8.18 +    unsigned long *cache = mapcache;
    8.19  
    8.20      /* A bit skanky -- depends on having an aligned PAGE_SIZE set of PTEs. */
    8.21      do { if ( (*cache & READY_FOR_TLB_FLUSH) ) *cache = 0; }
    8.22 @@ -39,23 +41,31 @@ static void flush_all_ready_maps(void)
    8.23  void *map_domain_mem(unsigned long pa)
    8.24  {
    8.25      unsigned long va;
    8.26 -    int cpu = smp_processor_id();
    8.27 -    unsigned int idx;
    8.28 -    unsigned long *cache = mapcache[cpu];
    8.29 +    unsigned int idx, cpu = smp_processor_id();
    8.30 +    unsigned long *cache = mapcache;
    8.31      unsigned long flags;
    8.32  
    8.33 -    local_irq_save(flags);
    8.34 +    spin_lock_irqsave(&map_lock, flags);
    8.35 +
    8.36 +    /* Has some other CPU caused a wrap? We must flush if so. */
    8.37 +    if ( map_idx < shadow_map_idx[cpu] )
    8.38 +    {
    8.39 +        perfc_incrc(domain_page_tlb_flush);
    8.40 +        local_flush_tlb();
    8.41 +    }
    8.42  
    8.43      for ( ; ; )
    8.44      {
    8.45 -        idx = map_idx[cpu] = (map_idx[cpu] + 1) & (MAPCACHE_ENTRIES - 1);
    8.46 +        idx = map_idx = (map_idx + 1) & (MAPCACHE_ENTRIES - 1);
    8.47          if ( idx == 0 ) flush_all_ready_maps();
    8.48          if ( cache[idx] == 0 ) break;
    8.49      }
    8.50  
    8.51      cache[idx] = (pa & PAGE_MASK) | __PAGE_HYPERVISOR;
    8.52  
    8.53 -    local_irq_restore(flags);
    8.54 +    spin_unlock_irqrestore(&map_lock, flags);
    8.55 +
    8.56 +    shadow_map_idx[cpu] = idx;
    8.57  
    8.58      va = MAPCACHE_VIRT_START + (idx << PAGE_SHIFT) + (pa & ~PAGE_MASK);
    8.59      return (void *)va;
    8.60 @@ -65,5 +75,5 @@ void unmap_domain_mem(void *va)
    8.61  {
    8.62      unsigned int idx;
    8.63      idx = ((unsigned long)va - MAPCACHE_VIRT_START) >> PAGE_SHIFT;
    8.64 -    mapcache[smp_processor_id()][idx] |= READY_FOR_TLB_FLUSH;
    8.65 +    mapcache[idx] |= READY_FOR_TLB_FLUSH;
    8.66  }
     9.1 --- a/xen/common/memory.c	Tue Oct 07 16:18:34 2003 +0000
     9.2 +++ b/xen/common/memory.c	Tue Oct 07 19:34:30 2003 +0000
     9.3 @@ -362,7 +362,7 @@ static int get_l2_table(unsigned long pa
     9.4      
     9.5      /* Now we simply slap in our high mapping. */
     9.6      memcpy(p_l2_entry, 
     9.7 -           idle_pg_table[smp_processor_id()] + DOMAIN_ENTRIES_PER_L2_PAGETABLE,
     9.8 +           &idle_pg_table[DOMAIN_ENTRIES_PER_L2_PAGETABLE],
     9.9             HYPERVISOR_ENTRIES_PER_L2_PAGETABLE * sizeof(l2_pgentry_t));
    9.10      p_l2_entry[(PERDOMAIN_VIRT_START >> L2_PAGETABLE_SHIFT) -
    9.11                DOMAIN_ENTRIES_PER_L2_PAGETABLE] =
    10.1 --- a/xen/include/asm-i386/domain_page.h	Tue Oct 07 16:18:34 2003 +0000
    10.2 +++ b/xen/include/asm-i386/domain_page.h	Tue Oct 07 19:34:30 2003 +0000
    10.3 @@ -7,7 +7,7 @@
    10.4  #include <xeno/config.h>
    10.5  #include <xeno/sched.h>
    10.6  
    10.7 -extern unsigned long *mapcache[NR_CPUS];
    10.8 +extern unsigned long *mapcache;
    10.9  #define MAPCACHE_ENTRIES        1024
   10.10  
   10.11  /*
    11.1 --- a/xen/include/asm-i386/page.h	Tue Oct 07 16:18:34 2003 +0000
    11.2 +++ b/xen/include/asm-i386/page.h	Tue Oct 07 19:34:30 2003 +0000
    11.3 @@ -89,8 +89,7 @@ typedef struct { unsigned long pt_lo; } 
    11.4  
    11.5  #define linear_pg_table ((l1_pgentry_t *)LINEAR_PT_VIRT_START)
    11.6  
    11.7 -extern l2_pgentry_t idle0_pg_table[ENTRIES_PER_L2_PAGETABLE];
    11.8 -extern l2_pgentry_t *idle_pg_table[NR_CPUS];
    11.9 +extern l2_pgentry_t idle_pg_table[ENTRIES_PER_L2_PAGETABLE];
   11.10  extern void paging_init(void);
   11.11  
   11.12  #define __flush_tlb() __flush_tlb_counted()
    12.1 --- a/xen/include/xeno/sched.h	Tue Oct 07 16:18:34 2003 +0000
    12.2 +++ b/xen/include/xeno/sched.h	Tue Oct 07 19:34:30 2003 +0000
    12.3 @@ -50,7 +50,7 @@ extern struct mm_struct init_mm;
    12.4  {                                                                   \
    12.5      cpu_vm_mask: 0,                                                 \
    12.6      perdomain_pt: 0,                                                \
    12.7 -    pagetable:   mk_pagetable(__pa(idle0_pg_table))                 \
    12.8 +    pagetable:   mk_pagetable(__pa(idle_pg_table))                  \
    12.9  }
   12.10  
   12.11  #define _HYP_EVENT_NEED_RESCHED 0