1.1 --- a/.rootkeys	Wed Oct 08 12:56:46 2003 +0000
     1.2 +++ b/.rootkeys	Wed Oct 08 13:20:22 2003 +0000
     1.3 @@ -475,6 +475,7 @@ 3ddb79c116WbJV8bwGZXFFJy_GNNvw xen/inclu
     1.4  3ddb79c1pwmlw8VXW8aaSKAVGVmjDA xen/include/xeno/byteorder/swabb.h
     1.5  3ddb79c0c0cX_DZE209-Bb-Rx1v-Aw xen/include/xeno/cache.h
     1.6  3e9c248aEG_nCngztiFmv5CfayNkcA xen/include/xeno/cdrom.h
     1.7 +3f840f12CkbYSlwMrY2S11Mpyxg7Nw xen/include/xeno/compiler.h
     1.8  3ddb79c259jh8hE7vre_8NuE7nwNSA xen/include/xeno/config.h
     1.9  3eb165e0eawr3R-p2ZQtSdLWtLRN_A xen/include/xeno/console.h
    1.10  3f0c428eIwGr7n9fj4FkBdX2YvA_Rw xen/include/xeno/crc32.h

     2.1 --- a/xen/common/domain.c	Wed Oct 08 12:56:46 2003 +0000
     2.2 +++ b/xen/common/domain.c	Wed Oct 08 13:20:22 2003 +0000
     2.3 @@ -163,8 +163,6 @@ void __kill_domain(struct task_struct *p
     2.4  
     2.5  void kill_domain(void)
     2.6  {
     2.7 -    /* May have been in middle of a p.t. update with WP bit cleared. */
     2.8 -    write_cr0(read_cr0()|X86_CR0_WP);
     2.9      __kill_domain(current);
    2.10  }
    2.11  

     3.1 --- a/xen/common/memory.c	Wed Oct 08 12:56:46 2003 +0000
     3.2 +++ b/xen/common/memory.c	Wed Oct 08 13:20:22 2003 +0000
     3.3 @@ -245,14 +245,14 @@ static int inc_page_refcnt(unsigned long
     3.4      struct pfn_info *page;
     3.5      unsigned long flags;
     3.6  
     3.7 -    if ( page_nr >= max_page )
     3.8 +    if ( unlikely(page_nr >= max_page) )
     3.9      {
    3.10          MEM_LOG("Page out of range (%08lx>%08lx)", page_nr, max_page);
    3.11          return -1;
    3.12      }
    3.13      page = frame_table + page_nr;
    3.14      flags = page->flags;
    3.15 -    if ( !DOMAIN_OKAY(flags) )
    3.16 +    if ( unlikely(!DOMAIN_OKAY(flags)) )
    3.17      {
    3.18          MEM_LOG("Bad page domain (%ld)", flags & PG_domain_mask);
    3.19          return -1;
    3.20 @@ -267,7 +267,7 @@ static int inc_page_refcnt(unsigned long
    3.21              return -1;
    3.22          }
    3.23  
    3.24 -        if ( flags & PG_need_flush )
    3.25 +        if ( unlikely(flags & PG_need_flush) )
    3.26          {
    3.27              flush_tlb[smp_processor_id()] = 1;
    3.28              page->flags &= ~PG_need_flush;
    3.29 @@ -288,14 +288,14 @@ static int dec_page_refcnt(unsigned long
    3.30  {
    3.31      struct pfn_info *page;
    3.32  
    3.33 -    if ( page_nr >= max_page )
    3.34 +    if ( unlikely(page_nr >= max_page) )
    3.35      {
    3.36          MEM_LOG("Page out of range (%08lx>%08lx)", page_nr, max_page);
    3.37          return -1;
    3.38      }
    3.39      page = frame_table + page_nr;
    3.40 -    if ( !DOMAIN_OKAY(page->flags) || 
    3.41 -         ((page->flags & PG_type_mask) != type) ) 
    3.42 +    if ( unlikely(!DOMAIN_OKAY(page->flags)) || 
    3.43 +         unlikely(((page->flags & PG_type_mask) != type)) ) 
    3.44      {
    3.45          MEM_LOG("Bad page type/domain (dom=%ld) (type %ld != expected %d)",
    3.46                  page->flags & PG_domain_mask, page->flags & PG_type_mask,
    3.47 @@ -339,7 +339,7 @@ static int get_l2_table(unsigned long pa
    3.48      int i, ret=0;
    3.49     
    3.50      ret = inc_page_refcnt(page_nr, PGT_l2_page_table);
    3.51 -    if ( ret != 0 ) return (ret < 0) ? ret : 0;
    3.52 +    if ( likely(ret != 0) ) return (ret < 0) ? ret : 0;
    3.53      
    3.54      /* NEW level-2 page table! Deal with every PDE in the table. */
    3.55      p_l2_entry = map_domain_mem(page_nr << PAGE_SHIFT);
    3.56 @@ -384,7 +384,7 @@ static int get_l1_table(unsigned long pa
    3.57  
    3.58      /* Update ref count for page pointed at by PDE. */
    3.59      ret = inc_page_refcnt(page_nr, PGT_l1_page_table);
    3.60 -    if ( ret != 0 ) return (ret < 0) ? ret : 0;
    3.61 +    if ( likely(ret != 0) ) return (ret < 0) ? ret : 0;
    3.62  
    3.63      /* NEW level-1 page table! Deal with every PTE in the table. */
    3.64      p_l1_entry = map_domain_mem(page_nr << PAGE_SHIFT);
    3.65 @@ -419,14 +419,14 @@ static int get_page(unsigned long page_n
    3.66      unsigned long flags;
    3.67  
    3.68      /* Update ref count for page pointed at by PTE. */
    3.69 -    if ( page_nr >= max_page )
    3.70 +    if ( unlikely(page_nr >= max_page) )
    3.71      {
    3.72          MEM_LOG("Page out of range (%08lx>%08lx)", page_nr, max_page);
    3.73          return(-1);
    3.74      }
    3.75      page = frame_table + page_nr;
    3.76      flags = page->flags;
    3.77 -    if ( !DOMAIN_OKAY(flags) )
    3.78 +    if ( unlikely(!DOMAIN_OKAY(flags)) )
    3.79      {
    3.80          MEM_LOG("Bad page domain (%ld)", flags & PG_domain_mask);
    3.81          return(-1);
    3.82 @@ -461,7 +461,7 @@ static void put_l2_table(unsigned long p
    3.83      l2_pgentry_t *p_l2_entry, l2_entry;
    3.84      int i;
    3.85  
    3.86 -    if ( dec_page_refcnt(page_nr, PGT_l2_page_table) ) return;
    3.87 +    if ( likely(dec_page_refcnt(page_nr, PGT_l2_page_table)) ) return;
    3.88  
    3.89      /* We had last reference to level-2 page table. Free the PDEs. */
    3.90      p_l2_entry = map_domain_mem(page_nr << PAGE_SHIFT);
    3.91 @@ -481,7 +481,7 @@ static void put_l1_table(unsigned long p
    3.92      l1_pgentry_t *p_l1_entry, l1_entry;
    3.93      int i;
    3.94  
    3.95 -    if ( dec_page_refcnt(page_nr, PGT_l1_page_table) ) return;
    3.96 +    if ( likely(dec_page_refcnt(page_nr, PGT_l1_page_table)) ) return;
    3.97  
    3.98      /* We had last reference to level-1 page table. Free the PTEs. */
    3.99      p_l1_entry = map_domain_mem(page_nr << PAGE_SHIFT);
   3.100 @@ -528,8 +528,8 @@ static int mod_l2_entry(l2_pgentry_t *p_
   3.101  {
   3.102      l2_pgentry_t old_l2_entry = *p_l2_entry;
   3.103  
   3.104 -    if ( (((unsigned long)p_l2_entry & (PAGE_SIZE-1)) >> 2) >=
   3.105 -         DOMAIN_ENTRIES_PER_L2_PAGETABLE )
   3.106 +    if ( unlikely((((unsigned long)p_l2_entry & (PAGE_SIZE-1)) >> 2) >=
   3.107 +                  DOMAIN_ENTRIES_PER_L2_PAGETABLE) )
   3.108      {
   3.109          MEM_LOG("Illegal L2 update attempt in hypervisor area %p",
   3.110                  p_l2_entry);
   3.111 @@ -538,7 +538,8 @@ static int mod_l2_entry(l2_pgentry_t *p_
   3.112  
   3.113      if ( (l2_pgentry_val(new_l2_entry) & _PAGE_PRESENT) )
   3.114      {
   3.115 -        if ( (l2_pgentry_val(new_l2_entry) & (_PAGE_GLOBAL|_PAGE_PSE)) )
   3.116 +        if ( unlikely((l2_pgentry_val(new_l2_entry) & 
   3.117 +                       (_PAGE_GLOBAL|_PAGE_PSE))) )
   3.118          {
   3.119              MEM_LOG("Bad L2 entry val %04lx",
   3.120                      l2_pgentry_val(new_l2_entry) & 
   3.121 @@ -582,10 +583,9 @@ static int mod_l1_entry(l1_pgentry_t *p_
   3.122  
   3.123      if ( (l1_pgentry_val(new_l1_entry) & _PAGE_PRESENT) )
   3.124      {
   3.125 -        if ( (l1_pgentry_val(new_l1_entry) &
   3.126 -              (_PAGE_GLOBAL|_PAGE_PAT)) ) 
   3.127 +        if ( unlikely((l1_pgentry_val(new_l1_entry) &
   3.128 +                       (_PAGE_GLOBAL|_PAGE_PAT))) ) 
   3.129          {
   3.130 -
   3.131              MEM_LOG("Bad L1 entry val %04lx",
   3.132                      l1_pgentry_val(new_l1_entry) & 
   3.133                      (_PAGE_GLOBAL|_PAGE_PAT));
   3.134 @@ -640,14 +640,14 @@ static int do_extended_command(unsigned 
   3.135      case PGEXT_PIN_L2_TABLE:
   3.136          err = get_l2_table(pfn);
   3.137      mark_as_pinned:
   3.138 -        if ( err )
   3.139 +        if ( unlikely(err) )
   3.140          {
   3.141              MEM_LOG("Error while pinning pfn %08lx", pfn);
   3.142              break;
   3.143          }
   3.144          put_page_type(page);
   3.145          put_page_tot(page);
   3.146 -        if ( !(page->type_count & REFCNT_PIN_BIT) )
   3.147 +        if ( likely(!(page->type_count & REFCNT_PIN_BIT)) )
   3.148          {
   3.149              page->type_count |= REFCNT_PIN_BIT;
   3.150              page->tot_count  |= REFCNT_PIN_BIT;
   3.151 @@ -752,8 +752,11 @@ int do_process_page_updates(page_update_
   3.152  
   3.153      for ( i = 0; i < count; i++ )
   3.154      {
   3.155 -        if ( copy_from_user(&req, ureqs, sizeof(req)) )
   3.156 +        if ( unlikely(copy_from_user(&req, ureqs, sizeof(req)) != 0) )
   3.157 +        {
   3.158 +            if ( cr0 != 0 ) write_cr0(cr0);
   3.159              kill_domain_with_errmsg("Cannot read page update request");
   3.160 +        }
   3.161  
   3.162          cmd = req.ptr & (sizeof(l1_pgentry_t)-1);
   3.163          pfn = req.ptr >> PAGE_SHIFT;
   3.164 @@ -773,7 +776,7 @@ int do_process_page_updates(page_update_
   3.165              /* Need to use 'get_user' since the VA's PGD may be absent. */
   3.166              __get_user(l1e, (unsigned long *)(linear_pg_table+pfn));
   3.167              /* Now check that the VA's PTE isn't absent. */
   3.168 -            if ( !(l1e & _PAGE_PRESENT) )
   3.169 +            if ( unlikely(!(l1e & _PAGE_PRESENT)) )
   3.170              {
   3.171                  MEM_LOG("L1E n.p. at VA %08lx (%08lx)", req.ptr&~3, l1e);
   3.172                  goto unlock;
   3.173 @@ -792,7 +795,7 @@ int do_process_page_updates(page_update_
   3.174              page  = frame_table + pfn;
   3.175              flags = page->flags;
   3.176  
   3.177 -            if ( DOMAIN_OKAY(flags) )
   3.178 +            if ( likely(DOMAIN_OKAY(flags)) )
   3.179              {
   3.180                  switch ( (flags & PG_type_mask) )
   3.181                  {
   3.182 @@ -824,7 +827,7 @@ int do_process_page_updates(page_update_
   3.183  
   3.184          case PGREQ_UNCHECKED_UPDATE:
   3.185              req.ptr &= ~(sizeof(l1_pgentry_t) - 1);
   3.186 -            if ( IS_PRIV(current) )
   3.187 +            if ( likely(IS_PRIV(current)) )
   3.188              {
   3.189                  *(unsigned long *)req.ptr = req.val;
   3.190                  err = 0;
   3.191 @@ -837,11 +840,11 @@ int do_process_page_updates(page_update_
   3.192              
   3.193          case PGREQ_MPT_UPDATE:
   3.194              page = frame_table + pfn;
   3.195 -            if ( pfn >= max_page )
   3.196 +            if ( unlikely(pfn >= max_page) )
   3.197              {
   3.198                  MEM_LOG("Page out of range (%08lx > %08lx)", pfn, max_page);
   3.199              }
   3.200 -            else if ( DOMAIN_OKAY(page->flags) )
   3.201 +            else if ( likely(DOMAIN_OKAY(page->flags)) )
   3.202              {
   3.203                  machine_to_phys_mapping[pfn] = req.val;
   3.204                  err = 0;
   3.205 @@ -870,8 +873,11 @@ int do_process_page_updates(page_update_
   3.206      unlock:
   3.207          spin_unlock_irq(&current->page_lock);
   3.208  
   3.209 -        if ( err )
   3.210 +        if ( unlikely(err) )
   3.211 +        {
   3.212 +            if ( cr0 != 0 ) write_cr0(cr0);
   3.213              kill_domain_with_errmsg("Illegal page update request");
   3.214 +        }
   3.215  
   3.216          ureqs++;
   3.217      }
   3.218 @@ -890,11 +896,6 @@ int do_process_page_updates(page_update_
   3.219  }
   3.220  
   3.221  
   3.222 -/*
   3.223 - * Note: This function is structured this way so that the common path is very 
   3.224 - * fast. Tests that are unlikely to be TRUE branch to out-of-line code. 
   3.225 - * Unfortunately GCC's 'unlikely()' macro doesn't do the right thing :-(
   3.226 - */
   3.227  int do_update_va_mapping(unsigned long page_nr, 
   3.228                           unsigned long val, 
   3.229                           unsigned long flags)
   3.230 @@ -903,57 +904,42 @@ int do_update_va_mapping(unsigned long p
   3.231      struct task_struct *p = current;
   3.232      int err = -EINVAL;
   3.233  
   3.234 -    if ( page_nr >= (HYPERVISOR_VIRT_START >> PAGE_SHIFT) )
   3.235 +    if ( unlikely(page_nr >= (HYPERVISOR_VIRT_START >> PAGE_SHIFT)) )
   3.236          goto out;
   3.237  
   3.238      spin_lock_irq(&p->page_lock);
   3.239  
   3.240      /* Check that the VA's page-directory entry is present.. */
   3.241 -    if ( (err = __get_user(_x, (unsigned long *)
   3.242 -                           (&linear_pg_table[page_nr]))) != 0 )
   3.243 +    if ( unlikely((err = __get_user(_x, (unsigned long *)
   3.244 +                                    (&linear_pg_table[page_nr]))) != 0) )
   3.245          goto unlock_and_out;
   3.246  
   3.247      /* If the VA's page-directory entry is read-only, we frob the WP bit. */
   3.248 -    if ( __put_user(_x, (unsigned long *)(&linear_pg_table[page_nr])) )
   3.249 -        goto clear_wp; return_from_clear_wp:
   3.250 -
   3.251 -    if ( (err = mod_l1_entry(&linear_pg_table[page_nr], 
   3.252 -                             mk_l1_pgentry(val))) != 0 )
   3.253 -        goto bad;
   3.254 +    if ( unlikely(__put_user(_x, (unsigned long *)
   3.255 +                             (&linear_pg_table[page_nr]))) )
   3.256 +    {
   3.257 +        cr0 = read_cr0();
   3.258 +        write_cr0(cr0 & ~X86_CR0_WP);        
   3.259 +    }
   3.260  
   3.261 -    if ( (flags & UVMF_INVLPG) )
   3.262 -        goto invlpg; return_from_invlpg:
   3.263 +    if ( unlikely((err = mod_l1_entry(&linear_pg_table[page_nr], 
   3.264 +                                      mk_l1_pgentry(val))) != 0) )
   3.265 +    {
   3.266 +        spin_unlock_irq(&p->page_lock);
   3.267 +        kill_domain_with_errmsg("Illegal VA-mapping update request");
   3.268 +    }
   3.269  
   3.270 -    if ( (flags & UVMF_FLUSH_TLB) )
   3.271 -        goto flush; return_from_flush:
   3.272 +    if ( unlikely(flags & UVMF_INVLPG) )
   3.273 +        __flush_tlb_one(page_nr << PAGE_SHIFT);
   3.274  
   3.275 -    if ( cr0 != 0 )
   3.276 -        goto write_cr0; return_from_write_cr0:
   3.277 +    if ( unlikely(flags & UVMF_FLUSH_TLB) )
   3.278 +        __write_cr3_counted(pagetable_val(p->mm.pagetable));
   3.279 +
   3.280 +    if ( unlikely(cr0 != 0) )
   3.281 +        write_cr0(cr0);
   3.282  
   3.283   unlock_and_out:
   3.284      spin_unlock_irq(&p->page_lock);
   3.285   out:
   3.286      return err;
   3.287 -
   3.288 - clear_wp:
   3.289 -    cr0 = read_cr0();
   3.290 -    write_cr0(cr0 & ~X86_CR0_WP);        
   3.291 -    goto return_from_clear_wp;
   3.292 -
   3.293 - bad:
   3.294 -    spin_unlock_irq(&p->page_lock);
   3.295 -    kill_domain_with_errmsg("Illegal VA-mapping update request");
   3.296 -    return 0;
   3.297 -
   3.298 - invlpg:
   3.299 -    flush_tlb[p->processor] = 1;
   3.300 -    goto return_from_invlpg;
   3.301 -    
   3.302 - flush:
   3.303 -    __write_cr3_counted(pagetable_val(p->mm.pagetable));
   3.304 -    goto return_from_flush;
   3.305 -
   3.306 - write_cr0:
   3.307 -    write_cr0(cr0);
   3.308 -    goto return_from_write_cr0;
   3.309  }

     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/xen/include/xeno/compiler.h	Wed Oct 08 13:20:22 2003 +0000
     4.3 @@ -0,0 +1,16 @@
     4.4 +#ifndef __LINUX_COMPILER_H
     4.5 +#define __LINUX_COMPILER_H
     4.6 +
     4.7 +/* Somewhere in the middle of the GCC 2.96 development cycle, we implemented
     4.8 +   a mechanism by which the user can annotate likely branch directions and
     4.9 +   expect the blocks to be reordered appropriately.  Define __builtin_expect
    4.10 +   to nothing for earlier compilers.  */
    4.11 +
    4.12 +#if __GNUC__ == 2 && __GNUC_MINOR__ < 96
    4.13 +#define __builtin_expect(x, expected_value) (x)
    4.14 +#endif
    4.15 +
    4.16 +#define likely(x)	__builtin_expect((x),1)
    4.17 +#define unlikely(x)	__builtin_expect((x),0)
    4.18 +
    4.19 +#endif /* __LINUX_COMPILER_H */

     5.1 --- a/xen/include/xeno/config.h	Wed Oct 08 12:56:46 2003 +0000
     5.2 +++ b/xen/include/xeno/config.h	Wed Oct 08 13:20:22 2003 +0000
     5.3 @@ -138,9 +138,6 @@
     5.4  #define offsetof(_p,_f) ((unsigned long)&(((_p *)0)->_f))
     5.5  #define struct_cpy(_x,_y) (memcpy((_x),(_y),sizeof(*(_x))))
     5.6  
     5.7 -#define likely(_x) (_x)
     5.8 -#define unlikely(_x) (_x)
     5.9 -
    5.10  #define dev_probe_lock() ((void)0)
    5.11  #define dev_probe_unlock() ((void)0)
    5.12  
    5.13 @@ -150,6 +147,8 @@
    5.14  
    5.15  #ifndef __ASSEMBLY__
    5.16  
    5.17 +#include <xeno/compiler.h>
    5.18 +
    5.19  extern unsigned long _end; /* standard ELF symbol */
    5.20  extern void __out_of_line_bug(int line) __attribute__((noreturn));
    5.21  #define out_of_line_bug() __out_of_line_bug(__LINE__)