ia64/xen-unstable

changeset 277:7ba7a40d2083

bitkeeper revision 1.122 (3e6d2fc8CULc6D1QN5JO9MAu4FbLdw)

system.h, hypervisor.c:
Reverted to locking for page table updat esynchronisation. Shouldn't be noticeably slower, and the stronger guarantees may be useful in future.
author kaf24@labyrinth.cl.cam.ac.uk
date Tue Mar 11 00:37:28 2003 +0000 (2003-03-11)
parents 99f5d61af2be
children 314d67fab116 91fde8282f37
files xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c xenolinux-2.4.21-pre4-sparse/include/asm-xeno/system.h
line diff
     1.1 --- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c	Mon Mar 10 20:25:12 2003 +0000
     1.2 +++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c	Tue Mar 11 00:37:28 2003 +0000
     1.3 @@ -13,26 +13,17 @@
     1.4  #include <asm/pgtable.h>
     1.5  
     1.6  /*
     1.7 - * A note on atomicity of these operations. We assume that queue_xxx
     1.8 - * operations never occur in an asynchronous (eg. interrupt) context.
     1.9 - * Therefore they do not need to be synchronised w.r.t. each other.
    1.10 - * However, flush_update_queue may be called from an interrupt context
    1.11 - * (eg. this is done in the network driver).
    1.12 - * 
    1.13 - * We use lock-free techniques to synchronise on the queue index. If a
    1.14 - * queue_xxx operation finds this index changes while it runs, it will
    1.15 - * fail and retry.
    1.16 - * 
    1.17 - * Flush operations must synchronize with themselves. They do this by
    1.18 - * atomically updating the index to zero on entry. This effectively locks
    1.19 - * out any other asynchronous calls to a flush operation.
    1.20 - * 
    1.21 - * Debug routines synchronise by disabling interrupts. It's easier that way.
    1.22 + * This suffices to protect us if we ever move to SMP domains.
    1.23 + * Further, it protects us against interrupts. At the very least, this is
    1.24 + * required for the network driver which flushes the update queue before
    1.25 + * pushing new receive buffers.
    1.26   */
    1.27 +static spinlock_t update_lock = SPIN_LOCK_UNLOCKED;
    1.28  
    1.29  #define QUEUE_SIZE 2048
    1.30  static page_update_request_t update_queue[QUEUE_SIZE];
    1.31 -volatile unsigned int pt_update_queue_idx = 0;
    1.32 +unsigned int pt_update_queue_idx = 0;
    1.33 +#define idx pt_update_queue_idx
    1.34  
    1.35  #if PT_UPDATE_DEBUG > 0
    1.36  page_update_debug_t update_debug_queue[QUEUE_SIZE] = {{0}};
    1.37 @@ -42,11 +33,7 @@ static void DEBUG_allow_pt_reads(void)
    1.38  {
    1.39      pte_t *pte;
    1.40      page_update_request_t update;
    1.41 -    unsigned int idx;
    1.42 -    unsigned long flags;
    1.43      int i;
    1.44 -    local_irq_save(flags);
    1.45 -    idx = pt_update_queue_idx;
    1.46      for ( i = idx-1; i >= 0; i-- )
    1.47      {
    1.48          pte = update_debug_queue[i].ptep;
    1.49 @@ -56,17 +43,13 @@ static void DEBUG_allow_pt_reads(void)
    1.50          update.val = update_debug_queue[i].pteval;
    1.51          HYPERVISOR_pt_update(&update, 1);
    1.52      }
    1.53 -    local_irq_restore(flags);
    1.54  }
    1.55  static void DEBUG_disallow_pt_read(unsigned long pa)
    1.56  {
    1.57      pte_t *pte;
    1.58      pmd_t *pmd;
    1.59      pgd_t *pgd;
    1.60 -    unsigned long pteval, flags;
    1.61 -    unsigned int idx;
    1.62 -    local_irq_save(flags);
    1.63 -    idx = pt_update_queue_idx;
    1.64 +    unsigned long pteval;
    1.65      /*
    1.66       * We may fault because of an already outstanding update.
    1.67       * That's okay -- it'll get fixed up in the fault handler.
    1.68 @@ -82,7 +65,6 @@ static void DEBUG_disallow_pt_read(unsig
    1.69      HYPERVISOR_pt_update(&update, 1);
    1.70      update_debug_queue[idx].ptep = pte;
    1.71      update_debug_queue[idx].pteval = pteval;
    1.72 -    local_irq_restore(flags);
    1.73  }
    1.74  #endif
    1.75  
    1.76 @@ -103,10 +85,8 @@ static void DEBUG_disallow_pt_read(unsig
    1.77   */
    1.78  unsigned long pt_baseptr;
    1.79  
    1.80 -void _flush_page_update_queue(void)
    1.81 +static inline void __flush_page_update_queue(void)
    1.82  {
    1.83 -    unsigned int idx = xchg(&pt_update_queue_idx, 0);
    1.84 -    if ( idx == 0 ) return;
    1.85  #if PT_UPDATE_DEBUG > 1
    1.86      printk("Flushing %d entries from pt update queue\n", idx);
    1.87  #endif
    1.88 @@ -114,112 +94,118 @@ void _flush_page_update_queue(void)
    1.89      DEBUG_allow_pt_reads();
    1.90  #endif
    1.91      HYPERVISOR_pt_update(update_queue, idx);
    1.92 +    idx = 0;
    1.93 +}
    1.94 +
    1.95 +void _flush_page_update_queue(void)
    1.96 +{
    1.97 +    unsigned long flags;
    1.98 +    spin_lock_irqsave(&update_lock, flags);
    1.99 +    if ( idx != 0 ) __flush_page_update_queue();
   1.100 +    spin_unlock_irqrestore(&update_lock, flags);
   1.101 +}
   1.102 +
   1.103 +static inline void increment_index(void)
   1.104 +{
   1.105 +    idx++;
   1.106 +    if ( unlikely(idx == QUEUE_SIZE) ) __flush_page_update_queue();
   1.107  }
   1.108  
   1.109  void queue_l1_entry_update(unsigned long ptr, unsigned long val)
   1.110  {
   1.111 -    unsigned int idx;
   1.112 +    unsigned long flags;
   1.113 +    spin_lock_irqsave(&update_lock, flags);
   1.114  #if PT_UPDATE_DEBUG > 0
   1.115      DEBUG_disallow_pt_read(ptr);
   1.116  #endif
   1.117 -    do {
   1.118 -        idx = pt_update_queue_idx;
   1.119 -        update_queue[idx].ptr = phys_to_machine(ptr);
   1.120 -        update_queue[idx].val = val;
   1.121 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );    
   1.122 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.123 +    update_queue[idx].ptr = phys_to_machine(ptr);
   1.124 +    update_queue[idx].val = val;
   1.125 +    increment_index();
   1.126 +    spin_unlock_irqrestore(&update_lock, flags);
   1.127  }
   1.128  
   1.129  void queue_l2_entry_update(unsigned long ptr, unsigned long val)
   1.130  {
   1.131 -    unsigned int idx;
   1.132 -    do {
   1.133 -        idx = pt_update_queue_idx;
   1.134 -        update_queue[idx].ptr = phys_to_machine(ptr);
   1.135 -        update_queue[idx].val = val;
   1.136 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.137 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.138 +    unsigned long flags;
   1.139 +    spin_lock_irqsave(&update_lock, flags);
   1.140 +    update_queue[idx].ptr = phys_to_machine(ptr);
   1.141 +    update_queue[idx].val = val;
   1.142 +    increment_index();
   1.143 +    spin_unlock_irqrestore(&update_lock, flags);
   1.144  }
   1.145  
   1.146  void queue_pt_switch(unsigned long ptr)
   1.147  {
   1.148 -    unsigned int idx;
   1.149 -    do {
   1.150 -        idx = pt_update_queue_idx;
   1.151 -        update_queue[idx].ptr  = phys_to_machine(ptr);
   1.152 -        update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.153 -        update_queue[idx].val  = PGEXT_NEW_BASEPTR;
   1.154 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.155 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.156 +    unsigned long flags;
   1.157 +    spin_lock_irqsave(&update_lock, flags);
   1.158 +    update_queue[idx].ptr  = phys_to_machine(ptr);
   1.159 +    update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.160 +    update_queue[idx].val  = PGEXT_NEW_BASEPTR;
   1.161 +    increment_index();
   1.162 +    spin_unlock_irqrestore(&update_lock, flags);
   1.163  }
   1.164  
   1.165  void queue_tlb_flush(void)
   1.166  {
   1.167 -    unsigned int idx;
   1.168 -    do {
   1.169 -        idx = pt_update_queue_idx;
   1.170 -        update_queue[idx].ptr  = PGREQ_EXTENDED_COMMAND;
   1.171 -        update_queue[idx].val  = PGEXT_TLB_FLUSH;
   1.172 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.173 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.174 +    unsigned long flags;
   1.175 +    spin_lock_irqsave(&update_lock, flags);
   1.176 +    update_queue[idx].ptr  = PGREQ_EXTENDED_COMMAND;
   1.177 +    update_queue[idx].val  = PGEXT_TLB_FLUSH;
   1.178 +    increment_index();
   1.179 +    spin_unlock_irqrestore(&update_lock, flags);
   1.180  }
   1.181  
   1.182  void queue_invlpg(unsigned long ptr)
   1.183  {
   1.184 -    unsigned int idx;
   1.185 -    do {
   1.186 -        idx = pt_update_queue_idx;
   1.187 -        update_queue[idx].ptr  = PGREQ_EXTENDED_COMMAND;
   1.188 -        update_queue[idx].val  = ptr & PAGE_MASK;
   1.189 -        update_queue[idx].val |= PGEXT_INVLPG;
   1.190 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.191 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.192 +    unsigned long flags;
   1.193 +    spin_lock_irqsave(&update_lock, flags);
   1.194 +    update_queue[idx].ptr  = PGREQ_EXTENDED_COMMAND;
   1.195 +    update_queue[idx].val  = ptr & PAGE_MASK;
   1.196 +    update_queue[idx].val |= PGEXT_INVLPG;
   1.197 +    increment_index();
   1.198 +    spin_unlock_irqrestore(&update_lock, flags);
   1.199  }
   1.200  
   1.201  void queue_pgd_pin(unsigned long ptr)
   1.202  {
   1.203 -    unsigned int idx;
   1.204 -    do {
   1.205 -        idx = pt_update_queue_idx;
   1.206 -        update_queue[idx].ptr  = phys_to_machine(ptr);
   1.207 -        update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.208 -        update_queue[idx].val  = PGEXT_PIN_L2_TABLE;
   1.209 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.210 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.211 +    unsigned long flags;
   1.212 +    spin_lock_irqsave(&update_lock, flags);
   1.213 +    update_queue[idx].ptr  = phys_to_machine(ptr);
   1.214 +    update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.215 +    update_queue[idx].val  = PGEXT_PIN_L2_TABLE;
   1.216 +    increment_index();
   1.217 +    spin_unlock_irqrestore(&update_lock, flags);
   1.218  }
   1.219  
   1.220  void queue_pgd_unpin(unsigned long ptr)
   1.221  {
   1.222 -    unsigned int idx;
   1.223 -    do {
   1.224 -        idx = pt_update_queue_idx;
   1.225 -        update_queue[idx].ptr  = phys_to_machine(ptr);
   1.226 -        update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.227 -        update_queue[idx].val  = PGEXT_UNPIN_TABLE;
   1.228 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.229 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.230 +    unsigned long flags;
   1.231 +    spin_lock_irqsave(&update_lock, flags);
   1.232 +    update_queue[idx].ptr  = phys_to_machine(ptr);
   1.233 +    update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.234 +    update_queue[idx].val  = PGEXT_UNPIN_TABLE;
   1.235 +    increment_index();
   1.236 +    spin_unlock_irqrestore(&update_lock, flags);
   1.237  }
   1.238  
   1.239  void queue_pte_pin(unsigned long ptr)
   1.240  {
   1.241 -    unsigned int idx;
   1.242 -    do {
   1.243 -        idx = pt_update_queue_idx;
   1.244 -        update_queue[idx].ptr  = phys_to_machine(ptr);
   1.245 -        update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.246 -        update_queue[idx].val  = PGEXT_PIN_L1_TABLE;
   1.247 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.248 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.249 +    unsigned long flags;
   1.250 +    spin_lock_irqsave(&update_lock, flags);
   1.251 +    update_queue[idx].ptr  = phys_to_machine(ptr);
   1.252 +    update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.253 +    update_queue[idx].val  = PGEXT_PIN_L1_TABLE;
   1.254 +    increment_index();
   1.255 +    spin_unlock_irqrestore(&update_lock, flags);
   1.256  }
   1.257  
   1.258  void queue_pte_unpin(unsigned long ptr)
   1.259  {
   1.260 -    unsigned int idx;
   1.261 -    do {
   1.262 -        idx = pt_update_queue_idx;
   1.263 -        update_queue[idx].ptr  = phys_to_machine(ptr);
   1.264 -        update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.265 -        update_queue[idx].val  = PGEXT_UNPIN_TABLE;
   1.266 -    } while ( cmpxchg(&pt_update_queue_idx, idx, idx+1) != idx );
   1.267 -    if ( idx == (QUEUE_SIZE-1) ) _flush_page_update_queue();
   1.268 +    unsigned long flags;
   1.269 +    spin_lock_irqsave(&update_lock, flags);
   1.270 +    update_queue[idx].ptr  = phys_to_machine(ptr);
   1.271 +    update_queue[idx].ptr |= PGREQ_EXTENDED_COMMAND;
   1.272 +    update_queue[idx].val  = PGEXT_UNPIN_TABLE;
   1.273 +    increment_index();
   1.274 +    spin_unlock_irqrestore(&update_lock, flags);
   1.275  }
     2.1 --- a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/system.h	Mon Mar 10 20:25:12 2003 +0000
     2.2 +++ b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/system.h	Tue Mar 11 00:37:28 2003 +0000
     2.3 @@ -319,7 +319,7 @@ do {                                    
     2.4      shared_info_t *_shared = HYPERVISOR_shared_info;  \
     2.5      _shared->events_enable = (x);                     \
     2.6      barrier();                                        \
     2.7 -    if ( _shared->events && (x) ) do_hypervisor_callback(NULL);  \
     2.8 +    if ( unlikely(_shared->events) && (x) ) do_hypervisor_callback(NULL);  \
     2.9  } while (0)
    2.10  #define __cli()                 (HYPERVISOR_shared_info->events_enable = 0); barrier()
    2.11  #define __sti()                                       \
    2.12 @@ -327,7 +327,7 @@ do {                                    
    2.13      shared_info_t *_shared = HYPERVISOR_shared_info;  \
    2.14      _shared->events_enable = 1;                       \
    2.15      barrier();                                        \
    2.16 -    if ( _shared->events ) do_hypervisor_callback(NULL);  \
    2.17 +    if ( unlikely(_shared->events) ) do_hypervisor_callback(NULL);  \
    2.18  } while (0)
    2.19  #define safe_halt()             ((void)0)
    2.20