1.1 --- a/extras/mini-os/arch/ia64/mm.c	Mon Mar 02 18:26:56 2009 +0900
     1.2 +++ b/extras/mini-os/arch/ia64/mm.c	Fri Mar 06 12:22:22 2009 +0900
     1.3 @@ -156,6 +156,12 @@ map_frames_ex(unsigned long* frames, uns
     1.4  	return (void*) __va(frames[0] << PAGE_SHIFT);
     1.5  }
     1.6  
     1.7 +int unmap_frames(unsigned long virt_addr, unsigned long num_frames)
     1.8 +{  
     1.9 +    /* TODO */
    1.10 +    ASSERT(0);
    1.11 +}
    1.12 +
    1.13  void arch_init_p2m(unsigned long max_pfn)
    1.14  {
    1.15      printk("Warn: p2m map not implemented.\n");

     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/extras/mini-os/arch/x86/ioremap.c	Fri Mar 06 12:22:22 2009 +0900
     2.3 @@ -0,0 +1,88 @@
     2.4 +/*
     2.5 + * Copyright (C) 2009,  Netronome Systems, Inc.
     2.6 + *                
     2.7 + * Permission is hereby granted, free of charge, to any person obtaining a copy
     2.8 + * of this software and associated documentation files (the "Software"), to
     2.9 + * deal in the Software without restriction, including without limitation the
    2.10 + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
    2.11 + * sell copies of the Software, and to permit persons to whom the Software is
    2.12 + * furnished to do so, subject to the following conditions:
    2.13 + * 
    2.14 + * The above copyright notice and this permission notice shall be included in
    2.15 + * all copies or substantial portions of the Software.
    2.16 + * 
    2.17 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
    2.18 + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
    2.19 + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
    2.20 + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
    2.21 + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
    2.22 + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
    2.23 + * DEALINGS IN THE SOFTWARE.
    2.24 + */
    2.25 +
    2.26 +
    2.27 +#include <types.h>
    2.28 +#include <lib.h>
    2.29 +#include <xmalloc.h>
    2.30 +#include <mm.h>
    2.31 +#include <ioremap.h>
    2.32 +
    2.33 +/* Map a physical address range into virtual address space with provided
    2.34 + * flags. Return a virtual address range it is mapped to. */
    2.35 +static void *__do_ioremap(unsigned long phys_addr, unsigned long size, 
    2.36 +                          unsigned long prot)
    2.37 +{
    2.38 +    unsigned long va;
    2.39 +    unsigned long mfns, mfn;
    2.40 +    unsigned long num_pages, offset;
    2.41 +    int i;
    2.42 +
    2.43 +    /* allow non page aligned addresses but for mapping we need to align them */
    2.44 +    offset = (phys_addr & ~PAGE_MASK);
    2.45 +    num_pages = (offset + size + PAGE_SIZE - 1) / PAGE_SIZE;
    2.46 +    phys_addr &= PAGE_MASK;
    2.47 +    mfns = mfn = phys_addr >> PAGE_SHIFT;
    2.48 +    
    2.49 +    /* sanity checks on list of MFNs */
    2.50 +    for ( i = 0; i < num_pages; i++, mfn++ )
    2.51 +    {
    2.52 +        if ( mfn_is_ram(mfn) )
    2.53 +        {
    2.54 +            printk("ioremap: mfn 0x%ulx is RAM\n", mfn);
    2.55 +            goto mfn_invalid;
    2.56 +        }
    2.57 +    }   
    2.58 +    va = (unsigned long)map_frames_ex(&mfns, num_pages, 0, 1, 1,
    2.59 +                                      DOMID_IO, 0, prot);
    2.60 +    return (void *)(va + offset);
    2.61 +    
    2.62 +mfn_invalid:
    2.63 +    return NULL;
    2.64 +}
    2.65 +
    2.66 +void *ioremap(unsigned long phys_addr, unsigned long size)
    2.67 +{
    2.68 +    return __do_ioremap(phys_addr, size, IO_PROT);
    2.69 +}
    2.70 +
    2.71 +void *ioremap_nocache(unsigned long phys_addr, unsigned long size)
    2.72 +{
    2.73 +    return __do_ioremap(phys_addr, size, IO_PROT_NOCACHE);
    2.74 +}
    2.75 +
    2.76 +/* Un-map the io-remapped region. Currently no list of existing mappings is
    2.77 + * maintained, so the caller has to supply the size */
    2.78 +void iounmap(void *virt_addr, unsigned long size)
    2.79 +{   
    2.80 +    unsigned long num_pages;
    2.81 +    unsigned long va = (unsigned long)virt_addr;
    2.82 +
    2.83 +    /* work out number of frames to unmap */
    2.84 +    num_pages = ((va & ~PAGE_MASK) + size + PAGE_SIZE - 1) / PAGE_SIZE;
    2.85 +
    2.86 +    unmap_frames(va & PAGE_MASK, num_pages);
    2.87 +}
    2.88 +
    2.89 +
    2.90 +
    2.91 +/* -*-  Mode:C; c-basic-offset:4; tab-width:4 indent-tabs-mode:nil -*- */

     3.1 --- a/extras/mini-os/arch/x86/mm.c	Mon Mar 02 18:26:56 2009 +0900
     3.2 +++ b/extras/mini-os/arch/x86/mm.c	Fri Mar 06 12:22:22 2009 +0900
     3.3 @@ -52,20 +52,26 @@
     3.4  unsigned long *phys_to_machine_mapping;
     3.5  unsigned long mfn_zero;
     3.6  extern char stack[];
     3.7 -extern void page_walk(unsigned long virt_addr);
     3.8 +extern void page_walk(unsigned long va);
     3.9  
    3.10 -void new_pt_frame(unsigned long *pt_pfn, unsigned long prev_l_mfn, 
    3.11 -                                unsigned long offset, unsigned long level)
    3.12 +/*
    3.13 + * Make pt_pfn a new 'level' page table frame and hook it into the page
    3.14 + * table at offset in previous level MFN (pref_l_mfn). pt_pfn is a guest
    3.15 + * PFN.
    3.16 + */
    3.17 +static void new_pt_frame(unsigned long *pt_pfn, unsigned long prev_l_mfn, 
    3.18 +                         unsigned long offset, unsigned long level)
    3.19  {   
    3.20      pgentry_t *tab = (pgentry_t *)start_info.pt_base;
    3.21      unsigned long pt_page = (unsigned long)pfn_to_virt(*pt_pfn); 
    3.22      pgentry_t prot_e, prot_t;
    3.23      mmu_update_t mmu_updates[1];
    3.24 +    int rc;
    3.25      
    3.26      prot_e = prot_t = 0;
    3.27 -    DEBUG("Allocating new L%d pt frame for pt_pfn=%lx, "
    3.28 -           "prev_l_mfn=%lx, offset=%lx", 
    3.29 -           level, *pt_pfn, prev_l_mfn, offset);
    3.30 +    DEBUG("Allocating new L%d pt frame for pfn=%lx, "
    3.31 +          "prev_l_mfn=%lx, offset=%lx", 
    3.32 +          level, *pt_pfn, prev_l_mfn, offset);
    3.33  
    3.34      /* We need to clear the page, otherwise we might fail to map it
    3.35         as a page table page */
    3.36 @@ -74,56 +80,63 @@ void new_pt_frame(unsigned long *pt_pfn,
    3.37      switch ( level )
    3.38      {
    3.39      case L1_FRAME:
    3.40 -         prot_e = L1_PROT;
    3.41 -         prot_t = L2_PROT;
    3.42 -         break;
    3.43 +        prot_e = L1_PROT;
    3.44 +        prot_t = L2_PROT;
    3.45 +        break;
    3.46      case L2_FRAME:
    3.47 -         prot_e = L2_PROT;
    3.48 -         prot_t = L3_PROT;
    3.49 -         break;
    3.50 +        prot_e = L2_PROT;
    3.51 +        prot_t = L3_PROT;
    3.52 +        break;
    3.53  #if defined(__x86_64__)
    3.54      case L3_FRAME:
    3.55 -         prot_e = L3_PROT;
    3.56 -         prot_t = L4_PROT;
    3.57 -         break;
    3.58 +        prot_e = L3_PROT;
    3.59 +        prot_t = L4_PROT;
    3.60 +        break;
    3.61  #endif
    3.62      default:
    3.63 -         printk("new_pt_frame() called with invalid level number %d\n", level);
    3.64 -         do_exit();
    3.65 -         break;
    3.66 +        printk("new_pt_frame() called with invalid level number %d\n", level);
    3.67 +        do_exit();
    3.68 +        break;
    3.69      }
    3.70  
    3.71 -    /* Update the entry */
    3.72 +    /* Make PFN a page table page */
    3.73  #if defined(__x86_64__)
    3.74      tab = pte_to_virt(tab[l4_table_offset(pt_page)]);
    3.75  #endif
    3.76      tab = pte_to_virt(tab[l3_table_offset(pt_page)]);
    3.77  
    3.78      mmu_updates[0].ptr = (tab[l2_table_offset(pt_page)] & PAGE_MASK) + 
    3.79 -                         sizeof(pgentry_t) * l1_table_offset(pt_page);
    3.80 +        sizeof(pgentry_t) * l1_table_offset(pt_page);
    3.81      mmu_updates[0].val = (pgentry_t)pfn_to_mfn(*pt_pfn) << PAGE_SHIFT | 
    3.82 -                         (prot_e & ~_PAGE_RW);
    3.83 -    if(HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF) < 0)
    3.84 +        (prot_e & ~_PAGE_RW);
    3.85 +    
    3.86 +    if ( (rc = HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF)) < 0 )
    3.87      {
    3.88 -         printk("PTE for new page table page could not be updated\n");
    3.89 -         do_exit();
    3.90 +        printk("ERROR: PTE for new page table page could not be updated\n");
    3.91 +        printk("       mmu_update failed with rc=%d\n", rc);
    3.92 +        do_exit();
    3.93      }
    3.94 -                        
    3.95 -    /* Now fill the new page table page with entries.
    3.96 -       Update the page directory as well. */
    3.97 -    mmu_updates[0].ptr = ((pgentry_t)prev_l_mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
    3.98 +
    3.99 +    /* Hook the new page table page into the hierarchy */
   3.100 +    mmu_updates[0].ptr =
   3.101 +        ((pgentry_t)prev_l_mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
   3.102      mmu_updates[0].val = (pgentry_t)pfn_to_mfn(*pt_pfn) << PAGE_SHIFT | prot_t;
   3.103 -    if(HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF) < 0) 
   3.104 +
   3.105 +    if ( (rc = HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF)) < 0 ) 
   3.106      {
   3.107 -       printk("ERROR: mmu_update failed\n");
   3.108 -       do_exit();
   3.109 +        printk("ERROR: mmu_update failed with rc=%d\n", rc);
   3.110 +        do_exit();
   3.111      }
   3.112  
   3.113      *pt_pfn += 1;
   3.114  }
   3.115  
   3.116 -/* Checks if a pagetable frame is needed (if weren't allocated by Xen) */
   3.117 -static int need_pt_frame(unsigned long virt_address, int level)
   3.118 +/*
   3.119 + * Checks if a pagetable frame is needed at 'level' to map a given
   3.120 + * address. Note, this function is specific to the initial page table
   3.121 + * building.
   3.122 + */
   3.123 +static int need_pt_frame(unsigned long va, int level)
   3.124  {
   3.125      unsigned long hyp_virt_start = HYPERVISOR_VIRT_START;
   3.126  #if defined(__x86_64__)
   3.127 @@ -135,63 +148,71 @@ static int need_pt_frame(unsigned long v
   3.128      /* In general frames will _not_ be needed if they were already
   3.129         allocated to map the hypervisor into our VA space */
   3.130  #if defined(__x86_64__)
   3.131 -    if(level == L3_FRAME)
   3.132 +    if ( level == L3_FRAME )
   3.133      {
   3.134 -        if(l4_table_offset(virt_address) >= 
   3.135 -           l4_table_offset(hyp_virt_start) &&
   3.136 -           l4_table_offset(virt_address) <= 
   3.137 -           l4_table_offset(hyp_virt_end))
   3.138 +        if ( l4_table_offset(va) >= 
   3.139 +             l4_table_offset(hyp_virt_start) &&
   3.140 +             l4_table_offset(va) <= 
   3.141 +             l4_table_offset(hyp_virt_end))
   3.142              return 0;
   3.143          return 1;
   3.144 -    } else
   3.145 +    } 
   3.146 +    else
   3.147  #endif
   3.148  
   3.149 -    if(level == L2_FRAME)
   3.150 +    if ( level == L2_FRAME )
   3.151      {
   3.152  #if defined(__x86_64__)
   3.153 -        if(l4_table_offset(virt_address) >= 
   3.154 -           l4_table_offset(hyp_virt_start) &&
   3.155 -           l4_table_offset(virt_address) <= 
   3.156 -           l4_table_offset(hyp_virt_end))
   3.157 +        if ( l4_table_offset(va) >= 
   3.158 +             l4_table_offset(hyp_virt_start) &&
   3.159 +             l4_table_offset(va) <= 
   3.160 +             l4_table_offset(hyp_virt_end))
   3.161  #endif
   3.162 -            if(l3_table_offset(virt_address) >= 
   3.163 -               l3_table_offset(hyp_virt_start) &&
   3.164 -               l3_table_offset(virt_address) <= 
   3.165 -               l3_table_offset(hyp_virt_end))
   3.166 +            if ( l3_table_offset(va) >= 
   3.167 +                 l3_table_offset(hyp_virt_start) &&
   3.168 +                 l3_table_offset(va) <= 
   3.169 +                 l3_table_offset(hyp_virt_end))
   3.170                  return 0;
   3.171  
   3.172          return 1;
   3.173 -    } else 
   3.174 -
   3.175 -    /* Always need l1 frames */
   3.176 -    if(level == L1_FRAME)
   3.177 -        return 1;
   3.178 +    } 
   3.179 +    else 
   3.180 +        /* Always need l1 frames */
   3.181 +        if ( level == L1_FRAME )
   3.182 +            return 1;
   3.183  
   3.184      printk("ERROR: Unknown frame level %d, hypervisor %llx,%llx\n", 
   3.185 -        level, hyp_virt_start, hyp_virt_end);
   3.186 +           level, hyp_virt_start, hyp_virt_end);
   3.187      return -1;
   3.188  }
   3.189  
   3.190 -void build_pagetable(unsigned long *start_pfn, unsigned long *max_pfn)
   3.191 +/*
   3.192 + * Build the initial pagetable.
   3.193 + */
   3.194 +static void build_pagetable(unsigned long *start_pfn, unsigned long *max_pfn)
   3.195  {
   3.196      unsigned long start_address, end_address;
   3.197      unsigned long pfn_to_map, pt_pfn = *start_pfn;
   3.198      static mmu_update_t mmu_updates[L1_PAGETABLE_ENTRIES + 1];
   3.199      pgentry_t *tab = (pgentry_t *)start_info.pt_base, page;
   3.200 -    unsigned long mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
   3.201 +    unsigned long pt_mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
   3.202      unsigned long offset;
   3.203      int count = 0;
   3.204 +    int rc;
   3.205  
   3.206 -    pfn_to_map = (start_info.nr_pt_frames - NOT_L1_FRAMES) * L1_PAGETABLE_ENTRIES;
   3.207 +    pfn_to_map = 
   3.208 +        (start_info.nr_pt_frames - NOT_L1_FRAMES) * L1_PAGETABLE_ENTRIES;
   3.209  
   3.210 -    if (*max_pfn >= virt_to_pfn(HYPERVISOR_VIRT_START))
   3.211 +    if ( *max_pfn >= virt_to_pfn(HYPERVISOR_VIRT_START) )
   3.212      {
   3.213          printk("WARNING: Mini-OS trying to use Xen virtual space. "
   3.214                 "Truncating memory from %dMB to ",
   3.215 -               ((unsigned long)pfn_to_virt(*max_pfn) - (unsigned long)&_text)>>20);
   3.216 +               ((unsigned long)pfn_to_virt(*max_pfn) -
   3.217 +                (unsigned long)&_text)>>20);
   3.218          *max_pfn = virt_to_pfn(HYPERVISOR_VIRT_START - PAGE_SIZE);
   3.219          printk("%dMB\n",
   3.220 -               ((unsigned long)pfn_to_virt(*max_pfn) - (unsigned long)&_text)>>20);
   3.221 +               ((unsigned long)pfn_to_virt(*max_pfn) - 
   3.222 +                (unsigned long)&_text)>>20);
   3.223      }
   3.224  
   3.225      start_address = (unsigned long)pfn_to_virt(pfn_to_map);
   3.226 @@ -200,49 +221,53 @@ void build_pagetable(unsigned long *star
   3.227      /* We worked out the virtual memory range to map, now mapping loop */
   3.228      printk("Mapping memory range 0x%lx - 0x%lx\n", start_address, end_address);
   3.229  
   3.230 -    while(start_address < end_address)
   3.231 +    while ( start_address < end_address )
   3.232      {
   3.233          tab = (pgentry_t *)start_info.pt_base;
   3.234 -        mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
   3.235 +        pt_mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
   3.236  
   3.237  #if defined(__x86_64__)
   3.238          offset = l4_table_offset(start_address);
   3.239          /* Need new L3 pt frame */
   3.240 -        if(!(start_address & L3_MASK)) 
   3.241 -            if(need_pt_frame(start_address, L3_FRAME)) 
   3.242 -                new_pt_frame(&pt_pfn, mfn, offset, L3_FRAME);
   3.243 +        if ( !(start_address & L3_MASK) )
   3.244 +            if ( need_pt_frame(start_address, L3_FRAME) ) 
   3.245 +                new_pt_frame(&pt_pfn, pt_mfn, offset, L3_FRAME);
   3.246  
   3.247          page = tab[offset];
   3.248 -        mfn = pte_to_mfn(page);
   3.249 -        tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT);
   3.250 +        pt_mfn = pte_to_mfn(page);
   3.251 +        tab = to_virt(mfn_to_pfn(pt_mfn) << PAGE_SHIFT);
   3.252  #endif
   3.253          offset = l3_table_offset(start_address);
   3.254          /* Need new L2 pt frame */
   3.255 -        if(!(start_address & L2_MASK))
   3.256 -            if(need_pt_frame(start_address, L2_FRAME))
   3.257 -                new_pt_frame(&pt_pfn, mfn, offset, L2_FRAME);
   3.258 +        if ( !(start_address & L2_MASK) )
   3.259 +            if ( need_pt_frame(start_address, L2_FRAME) )
   3.260 +                new_pt_frame(&pt_pfn, pt_mfn, offset, L2_FRAME);
   3.261  
   3.262          page = tab[offset];
   3.263 -        mfn = pte_to_mfn(page);
   3.264 -        tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT);
   3.265 +        pt_mfn = pte_to_mfn(page);
   3.266 +        tab = to_virt(mfn_to_pfn(pt_mfn) << PAGE_SHIFT);
   3.267          offset = l2_table_offset(start_address);        
   3.268          /* Need new L1 pt frame */
   3.269 -        if(!(start_address & L1_MASK))
   3.270 -            if(need_pt_frame(start_address, L1_FRAME)) 
   3.271 -                new_pt_frame(&pt_pfn, mfn, offset, L1_FRAME);
   3.272 +        if ( !(start_address & L1_MASK) )
   3.273 +            if ( need_pt_frame(start_address, L1_FRAME) )
   3.274 +                new_pt_frame(&pt_pfn, pt_mfn, offset, L1_FRAME);
   3.275  
   3.276          page = tab[offset];
   3.277 -        mfn = pte_to_mfn(page);
   3.278 +        pt_mfn = pte_to_mfn(page);
   3.279          offset = l1_table_offset(start_address);
   3.280  
   3.281 -        mmu_updates[count].ptr = ((pgentry_t)mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
   3.282 -        mmu_updates[count].val = (pgentry_t)pfn_to_mfn(pfn_to_map++) << PAGE_SHIFT | L1_PROT;
   3.283 +        mmu_updates[count].ptr =
   3.284 +            ((pgentry_t)pt_mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
   3.285 +        mmu_updates[count].val = 
   3.286 +            (pgentry_t)pfn_to_mfn(pfn_to_map++) << PAGE_SHIFT | L1_PROT;
   3.287          count++;
   3.288 -        if (count == L1_PAGETABLE_ENTRIES || pfn_to_map == *max_pfn)
   3.289 +        if ( count == L1_PAGETABLE_ENTRIES || pfn_to_map == *max_pfn )
   3.290          {
   3.291 -            if(HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF) < 0)
   3.292 +            rc = HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF);
   3.293 +            if ( rc < 0 )
   3.294              {
   3.295 -                printk("PTE could not be updated\n");
   3.296 +                printk("ERROR: build_pagetable(): PTE could not be updated\n");
   3.297 +                printk("       mmu_update failed with rc=%d\n", rc);
   3.298                  do_exit();
   3.299              }
   3.300              count = 0;
   3.301 @@ -253,20 +278,26 @@ void build_pagetable(unsigned long *star
   3.302      *start_pfn = pt_pfn;
   3.303  }
   3.304  
   3.305 +/*
   3.306 + * Mark portion of the address space read only.
   3.307 + */
   3.308  extern void shared_info;
   3.309  static void set_readonly(void *text, void *etext)
   3.310  {
   3.311 -    unsigned long start_address = ((unsigned long) text + PAGE_SIZE - 1) & PAGE_MASK;
   3.312 +    unsigned long start_address =
   3.313 +        ((unsigned long) text + PAGE_SIZE - 1) & PAGE_MASK;
   3.314      unsigned long end_address = (unsigned long) etext;
   3.315      static mmu_update_t mmu_updates[L1_PAGETABLE_ENTRIES + 1];
   3.316      pgentry_t *tab = (pgentry_t *)start_info.pt_base, page;
   3.317      unsigned long mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
   3.318      unsigned long offset;
   3.319      int count = 0;
   3.320 +    int rc;
   3.321  
   3.322      printk("setting %p-%p readonly\n", text, etext);
   3.323  
   3.324 -    while (start_address + PAGE_SIZE <= end_address) {
   3.325 +    while ( start_address + PAGE_SIZE <= end_address )
   3.326 +    {
   3.327          tab = (pgentry_t *)start_info.pt_base;
   3.328          mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
   3.329  
   3.330 @@ -287,20 +318,25 @@ static void set_readonly(void *text, voi
   3.331  
   3.332          offset = l1_table_offset(start_address);
   3.333  
   3.334 -	if (start_address != (unsigned long)&shared_info) {
   3.335 -	    mmu_updates[count].ptr = ((pgentry_t)mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
   3.336 -	    mmu_updates[count].val = tab[offset] & ~_PAGE_RW;
   3.337 -	    count++;
   3.338 -	} else
   3.339 -	    printk("skipped %p\n", start_address);
   3.340 +        if ( start_address != (unsigned long)&shared_info )
   3.341 +        {
   3.342 +            mmu_updates[count].ptr = 
   3.343 +                ((pgentry_t)mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
   3.344 +            mmu_updates[count].val = tab[offset] & ~_PAGE_RW;
   3.345 +            count++;
   3.346 +        }
   3.347 +        else
   3.348 +            printk("skipped %p\n", start_address);
   3.349  
   3.350          start_address += PAGE_SIZE;
   3.351  
   3.352 -        if (count == L1_PAGETABLE_ENTRIES || start_address + PAGE_SIZE > end_address)
   3.353 +        if ( count == L1_PAGETABLE_ENTRIES || 
   3.354 +             start_address + PAGE_SIZE > end_address )
   3.355          {
   3.356 -            if(HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF) < 0)
   3.357 +            rc = HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF);
   3.358 +            if ( rc < 0 )
   3.359              {
   3.360 -                printk("PTE could not be updated\n");
   3.361 +                printk("ERROR: set_readonly(): PTE could not be updated\n");
   3.362                  do_exit();
   3.363              }
   3.364              count = 0;
   3.365 @@ -308,41 +344,73 @@ static void set_readonly(void *text, voi
   3.366      }
   3.367  
   3.368      {
   3.369 -	mmuext_op_t op = {
   3.370 -	    .cmd = MMUEXT_TLB_FLUSH_ALL,
   3.371 -	};
   3.372 -	int count;
   3.373 -	HYPERVISOR_mmuext_op(&op, 1, &count, DOMID_SELF);
   3.374 +        mmuext_op_t op = {
   3.375 +            .cmd = MMUEXT_TLB_FLUSH_ALL,
   3.376 +        };
   3.377 +        int count;
   3.378 +        HYPERVISOR_mmuext_op(&op, 1, &count, DOMID_SELF);
   3.379      }
   3.380  }
   3.381  
   3.382 -void mem_test(unsigned long *start_add, unsigned long *end_add)
   3.383 +/*
   3.384 + * A useful mem testing function. Write the address to every address in the
   3.385 + * range provided and read back the value. If verbose, print page walk to
   3.386 + * some VA
   3.387 + * 
   3.388 + * If we get MEM_TEST_MAX_ERRORS we might as well stop
   3.389 + */
   3.390 +#define MEM_TEST_MAX_ERRORS 10 
   3.391 +int mem_test(unsigned long *start_va, unsigned long *end_va, int verbose)
   3.392  {
   3.393      unsigned long mask = 0x10000;
   3.394      unsigned long *pointer;
   3.395 -
   3.396 -    for(pointer = start_add; pointer < end_add; pointer++)
   3.397 +    int error_count = 0;
   3.398 + 
   3.399 +    /* write values and print page walks */
   3.400 +    if ( verbose && (((unsigned long)start_va) & 0xfffff) )
   3.401      {
   3.402 -        if(!(((unsigned long)pointer) & 0xfffff))
   3.403 +        printk("MemTest Start: 0x%lx\n", start_va);
   3.404 +        page_walk((unsigned long)start_va);
   3.405 +    }
   3.406 +    for ( pointer = start_va; pointer < end_va; pointer++ )
   3.407 +    {
   3.408 +        if ( verbose && !(((unsigned long)pointer) & 0xfffff) )
   3.409          {
   3.410              printk("Writing to %lx\n", pointer);
   3.411              page_walk((unsigned long)pointer);
   3.412          }
   3.413          *pointer = (unsigned long)pointer & ~mask;
   3.414      }
   3.415 -
   3.416 -    for(pointer = start_add; pointer < end_add; pointer++)
   3.417 +    if ( verbose && (((unsigned long)end_va) & 0xfffff) )
   3.418      {
   3.419 -        if(((unsigned long)pointer & ~mask) != *pointer)
   3.420 +        printk("MemTest End: %lx\n", end_va-1);
   3.421 +        page_walk((unsigned long)end_va-1);
   3.422 +    }
   3.423 + 
   3.424 +    /* verify values */
   3.425 +    for ( pointer = start_va; pointer < end_va; pointer++ )
   3.426 +    {
   3.427 +        if ( ((unsigned long)pointer & ~mask) != *pointer )
   3.428 +        {
   3.429              printk("Read error at 0x%lx. Read: 0x%lx, should read 0x%lx\n",
   3.430 -                (unsigned long)pointer, 
   3.431 -                *pointer, 
   3.432 -                ((unsigned long)pointer & ~mask));
   3.433 +                   (unsigned long)pointer, *pointer, 
   3.434 +                   ((unsigned long)pointer & ~mask));
   3.435 +            error_count++;
   3.436 +            if ( error_count >= MEM_TEST_MAX_ERRORS )
   3.437 +            {
   3.438 +                printk("mem_test: too many errors\n");
   3.439 +                return -1;
   3.440 +            }
   3.441 +        }
   3.442      }
   3.443 -
   3.444 +    return 0;
   3.445  }
   3.446  
   3.447 -static pgentry_t *get_pgt(unsigned long addr)
   3.448 +
   3.449 +/*
   3.450 + * get the PTE for virtual address va if it exists. Otherwise NULL.
   3.451 + */
   3.452 +static pgentry_t *get_pgt(unsigned long va)
   3.453  {
   3.454      unsigned long mfn;
   3.455      pgentry_t *tab;
   3.456 @@ -352,67 +420,78 @@ static pgentry_t *get_pgt(unsigned long 
   3.457      mfn = virt_to_mfn(start_info.pt_base);
   3.458  
   3.459  #if defined(__x86_64__)
   3.460 -    offset = l4_table_offset(addr);
   3.461 -    if (!(tab[offset] & _PAGE_PRESENT))
   3.462 +    offset = l4_table_offset(va);
   3.463 +    if ( !(tab[offset] & _PAGE_PRESENT) )
   3.464          return NULL;
   3.465      mfn = pte_to_mfn(tab[offset]);
   3.466      tab = mfn_to_virt(mfn);
   3.467  #endif
   3.468 -    offset = l3_table_offset(addr);
   3.469 -    if (!(tab[offset] & _PAGE_PRESENT))
   3.470 +    offset = l3_table_offset(va);
   3.471 +    if ( !(tab[offset] & _PAGE_PRESENT) )
   3.472 +        return NULL;
   3.473 +    mfn = pte_to_mfn(tab[offset]);
   3.474 +    tab = mfn_to_virt(mfn);
   3.475 +    offset = l2_table_offset(va);
   3.476 +    if ( !(tab[offset] & _PAGE_PRESENT) )
   3.477          return NULL;
   3.478      mfn = pte_to_mfn(tab[offset]);
   3.479      tab = mfn_to_virt(mfn);
   3.480 -    offset = l2_table_offset(addr);
   3.481 -    if (!(tab[offset] & _PAGE_PRESENT))
   3.482 -        return NULL;
   3.483 -    mfn = pte_to_mfn(tab[offset]);
   3.484 -    tab = mfn_to_virt(mfn);
   3.485 -    offset = l1_table_offset(addr);
   3.486 +    offset = l1_table_offset(va);
   3.487      return &tab[offset];
   3.488  }
   3.489  
   3.490 -pgentry_t *need_pgt(unsigned long addr)
   3.491 +
   3.492 +/*
   3.493 + * return a valid PTE for a given virtual address. If PTE does not exist,
   3.494 + * allocate page-table pages.
   3.495 + */
   3.496 +pgentry_t *need_pgt(unsigned long va)
   3.497  {
   3.498 -    unsigned long mfn;
   3.499 +    unsigned long pt_mfn;
   3.500      pgentry_t *tab;
   3.501      unsigned long pt_pfn;
   3.502      unsigned offset;
   3.503  
   3.504      tab = (pgentry_t *)start_info.pt_base;
   3.505 -    mfn = virt_to_mfn(start_info.pt_base);
   3.506 +    pt_mfn = virt_to_mfn(start_info.pt_base);
   3.507  
   3.508  #if defined(__x86_64__)
   3.509 -    offset = l4_table_offset(addr);
   3.510 -    if (!(tab[offset] & _PAGE_PRESENT)) {
   3.511 +    offset = l4_table_offset(va);
   3.512 +    if ( !(tab[offset] & _PAGE_PRESENT) )
   3.513 +    {
   3.514          pt_pfn = virt_to_pfn(alloc_page());
   3.515 -        new_pt_frame(&pt_pfn, mfn, offset, L3_FRAME);
   3.516 +        new_pt_frame(&pt_pfn, pt_mfn, offset, L3_FRAME);
   3.517      }
   3.518      ASSERT(tab[offset] & _PAGE_PRESENT);
   3.519 -    mfn = pte_to_mfn(tab[offset]);
   3.520 -    tab = mfn_to_virt(mfn);
   3.521 +    pt_mfn = pte_to_mfn(tab[offset]);
   3.522 +    tab = mfn_to_virt(pt_mfn);
   3.523  #endif
   3.524 -    offset = l3_table_offset(addr);
   3.525 -    if (!(tab[offset] & _PAGE_PRESENT)) {
   3.526 +    offset = l3_table_offset(va);
   3.527 +    if ( !(tab[offset] & _PAGE_PRESENT) ) 
   3.528 +    {
   3.529          pt_pfn = virt_to_pfn(alloc_page());
   3.530 -        new_pt_frame(&pt_pfn, mfn, offset, L2_FRAME);
   3.531 +        new_pt_frame(&pt_pfn, pt_mfn, offset, L2_FRAME);
   3.532      }
   3.533      ASSERT(tab[offset] & _PAGE_PRESENT);
   3.534 -    mfn = pte_to_mfn(tab[offset]);
   3.535 -    tab = mfn_to_virt(mfn);
   3.536 -    offset = l2_table_offset(addr);
   3.537 -    if (!(tab[offset] & _PAGE_PRESENT)) {
   3.538 +    pt_mfn = pte_to_mfn(tab[offset]);
   3.539 +    tab = mfn_to_virt(pt_mfn);
   3.540 +    offset = l2_table_offset(va);
   3.541 +    if ( !(tab[offset] & _PAGE_PRESENT) )
   3.542 +    {
   3.543          pt_pfn = virt_to_pfn(alloc_page());
   3.544 -	new_pt_frame(&pt_pfn, mfn, offset, L1_FRAME);
   3.545 +        new_pt_frame(&pt_pfn, pt_mfn, offset, L1_FRAME);
   3.546      }
   3.547      ASSERT(tab[offset] & _PAGE_PRESENT);
   3.548 -    mfn = pte_to_mfn(tab[offset]);
   3.549 -    tab = mfn_to_virt(mfn);
   3.550 +    pt_mfn = pte_to_mfn(tab[offset]);
   3.551 +    tab = mfn_to_virt(pt_mfn);
   3.552  
   3.553 -    offset = l1_table_offset(addr);
   3.554 +    offset = l1_table_offset(va);
   3.555      return &tab[offset];
   3.556  }
   3.557  
   3.558 +/*
   3.559 + * Reserve an area of virtual address space for mappings and Heap
   3.560 + */
   3.561  static unsigned long demand_map_area_start;
   3.562  #ifdef __x86_64__
   3.563  #define DEMAND_MAP_PAGES ((128ULL << 30) / PAGE_SIZE)
   3.564 @@ -437,7 +516,8 @@ void arch_init_demand_mapping_area(unsig
   3.565  
   3.566      demand_map_area_start = (unsigned long) pfn_to_virt(cur_pfn);
   3.567      cur_pfn += DEMAND_MAP_PAGES;
   3.568 -    printk("Demand map pfns at %lx-%lx.\n", demand_map_area_start, pfn_to_virt(cur_pfn));
   3.569 +    printk("Demand map pfns at %lx-%lx.\n", 
   3.570 +           demand_map_area_start, pfn_to_virt(cur_pfn));
   3.571  
   3.572  #ifdef HAVE_LIBC
   3.573      cur_pfn++;
   3.574 @@ -448,104 +528,204 @@ void arch_init_demand_mapping_area(unsig
   3.575  #endif
   3.576  }
   3.577  
   3.578 +unsigned long allocate_ondemand(unsigned long n, unsigned long alignment)
   3.579 +{
   3.580 +    unsigned long x;
   3.581 +    unsigned long y = 0;
   3.582 +
   3.583 +    /* Find a properly aligned run of n contiguous frames */
   3.584 +    for ( x = 0;
   3.585 +          x <= DEMAND_MAP_PAGES - n; 
   3.586 +          x = (x + y + 1 + alignment - 1) & ~(alignment - 1) )
   3.587 +    {
   3.588 +        unsigned long addr = demand_map_area_start + x * PAGE_SIZE;
   3.589 +        pgentry_t *pgt = get_pgt(addr);
   3.590 +        for ( y = 0; y < n; y++, addr += PAGE_SIZE ) 
   3.591 +        {
   3.592 +            if ( !(addr & L1_MASK) )
   3.593 +                pgt = get_pgt(addr);
   3.594 +            if ( pgt )
   3.595 +            {
   3.596 +                if ( *pgt & _PAGE_PRESENT )
   3.597 +                    break;
   3.598 +                pgt++;
   3.599 +            }
   3.600 +        }
   3.601 +        if ( y == n )
   3.602 +            break;
   3.603 +    }
   3.604 +    if ( y != n )
   3.605 +    {
   3.606 +        printk("Failed to find %ld frames!\n", n);
   3.607 +        return 0;
   3.608 +    }
   3.609 +    return demand_map_area_start + x * PAGE_SIZE;
   3.610 +}
   3.611 +
   3.612 +/*
   3.613 + * Map an array of MFNs contiguously into virtual address space starting at
   3.614 + * va. map f[i*stride]+i*increment for i in 0..n-1.
   3.615 + */
   3.616  #define MAP_BATCH ((STACK_SIZE / 2) / sizeof(mmu_update_t))
   3.617 -void do_map_frames(unsigned long addr,
   3.618 -        unsigned long *f, unsigned long n, unsigned long stride,
   3.619 -	unsigned long increment, domid_t id, int may_fail, unsigned long prot)
   3.620 +void do_map_frames(unsigned long va,
   3.621 +                   unsigned long *mfns, unsigned long n, 
   3.622 +                   unsigned long stride, unsigned long incr, 
   3.623 +                   domid_t id, int may_fail,
   3.624 +                   unsigned long prot)
   3.625  {
   3.626      pgentry_t *pgt = NULL;
   3.627      unsigned long done = 0;
   3.628      unsigned long i;
   3.629      int rc;
   3.630  
   3.631 -    while (done < n) {
   3.632 -	unsigned long todo;
   3.633 +    if ( !mfns ) 
   3.634 +    {
   3.635 +        printk("do_map_frames: no mfns supplied\n");
   3.636 +        return;
   3.637 +    }
   3.638 +    DEBUG("va=%p n=0x%lx, mfns[0]=0x%lx stride=0x%lx incr=0x%lx prot=0x%lx\n",
   3.639 +          va, n, mfns[0], stride, incr, prot);
   3.640 + 
   3.641 +    while ( done < n )
   3.642 +    {
   3.643 +        unsigned long todo;
   3.644  
   3.645 -	if (may_fail)
   3.646 -	    todo = 1;
   3.647 -	else
   3.648 -	    todo = n - done;
   3.649 +        if ( may_fail )
   3.650 +            todo = 1;
   3.651 +        else
   3.652 +            todo = n - done;
   3.653  
   3.654 -	if (todo > MAP_BATCH)
   3.655 -		todo = MAP_BATCH;
   3.656 +        if ( todo > MAP_BATCH )
   3.657 +            todo = MAP_BATCH;
   3.658  
   3.659 -	{
   3.660 -	    mmu_update_t mmu_updates[todo];
   3.661 +        {
   3.662 +            mmu_update_t mmu_updates[todo];
   3.663  
   3.664 -	    for (i = 0; i < todo; i++, addr += PAGE_SIZE, pgt++) {
   3.665 -                if (!pgt || !(addr & L1_MASK))
   3.666 -                    pgt = need_pgt(addr);
   3.667 -		mmu_updates[i].ptr = virt_to_mach(pgt);
   3.668 -		mmu_updates[i].val = ((pgentry_t)(f[(done + i) * stride] + (done + i) * increment) << PAGE_SHIFT) | prot;
   3.669 -	    }
   3.670 +            for ( i = 0; i < todo; i++, va += PAGE_SIZE, pgt++) 
   3.671 +            {
   3.672 +                if ( !pgt || !(va & L1_MASK) )
   3.673 +                    pgt = need_pgt(va);
   3.674 +                
   3.675 +                mmu_updates[i].ptr = virt_to_mach(pgt) | MMU_NORMAL_PT_UPDATE;
   3.676 +                mmu_updates[i].val = ((pgentry_t)(mfns[(done + i) * stride] +
   3.677 +                                                  (done + i) * incr)
   3.678 +                                      << PAGE_SHIFT) | prot;
   3.679 +            }
   3.680  
   3.681 -	    rc = HYPERVISOR_mmu_update(mmu_updates, todo, NULL, id);
   3.682 -	    if (rc < 0) {
   3.683 -		if (may_fail)
   3.684 -		    f[done * stride] |= 0xF0000000;
   3.685 -		else {
   3.686 -		    printk("Map %ld (%lx, ...) at %p failed: %d.\n", todo, f[done * stride] + done * increment, addr, rc);
   3.687 +            rc = HYPERVISOR_mmu_update(mmu_updates, todo, NULL, id);
   3.688 +            if ( rc < 0 )
   3.689 +            {
   3.690 +                if (may_fail)
   3.691 +                    mfns[done * stride] |= 0xF0000000;
   3.692 +                else {
   3.693 +                    printk("Map %ld (%lx, ...) at %p failed: %d.\n",
   3.694 +                           todo, mfns[done * stride] + done * incr, va, rc);
   3.695                      do_exit();
   3.696 -		}
   3.697 -	    }
   3.698 -	}
   3.699 -
   3.700 -	done += todo;
   3.701 +                }
   3.702 +            }
   3.703 +        }
   3.704 +        done += todo;
   3.705      }
   3.706  }
   3.707  
   3.708 -unsigned long allocate_ondemand(unsigned long n, unsigned long alignment)
   3.709 +/*
   3.710 + * Map an array of MFNs contiguous into virtual address space. Virtual
   3.711 + * addresses are allocated from the on demand area.
   3.712 + */
   3.713 +void *map_frames_ex(unsigned long *mfns, unsigned long n, 
   3.714 +                    unsigned long stride, unsigned long incr,
   3.715 +                    unsigned long alignment,
   3.716 +                    domid_t id, int may_fail, unsigned long prot)
   3.717  {
   3.718 -    unsigned long x;
   3.719 -    unsigned long y = 0;
   3.720 +    unsigned long va = allocate_ondemand(n, alignment);
   3.721 +
   3.722 +    if ( !va )
   3.723 +        return NULL;
   3.724 +
   3.725 +    do_map_frames(va, mfns, n, stride, incr, id, may_fail, prot);
   3.726 +
   3.727 +    return (void *)va;
   3.728 +}
   3.729 +
   3.730 +/*
   3.731 + * Unmap nun_frames frames mapped at virtual address va.
   3.732 + */
   3.733 +#define UNMAP_BATCH ((STACK_SIZE / 2) / sizeof(multicall_entry_t))
   3.734 +int unmap_frames(unsigned long va, unsigned long num_frames)
   3.735 +{
   3.736 +    int n = UNMAP_BATCH;
   3.737 +    multicall_entry_t call[n];
   3.738 +    int ret;
   3.739 +    int i;
   3.740 +
   3.741 +    ASSERT(!((unsigned long)va & ~PAGE_MASK));
   3.742  
   3.743 -    /* Find a properly aligned run of n contiguous frames */
   3.744 -    for (x = 0; x <= DEMAND_MAP_PAGES - n; x = (x + y + 1 + alignment - 1) & ~(alignment - 1)) {
   3.745 -        unsigned long addr = demand_map_area_start + x * PAGE_SIZE;
   3.746 -        pgentry_t *pgt = get_pgt(addr);
   3.747 -        for (y = 0; y < n; y++, addr += PAGE_SIZE) {
   3.748 -            if (!(addr & L1_MASK))
   3.749 -                pgt = get_pgt(addr);
   3.750 -            if (pgt) {
   3.751 -                if (*pgt & _PAGE_PRESENT)
   3.752 -                    break;
   3.753 -                pgt++;
   3.754 +    DEBUG("va=%p, num=0x%lx\n", va, num_frames);
   3.755 +
   3.756 +    while ( num_frames ) {
   3.757 +        if ( n > num_frames )
   3.758 +            n = num_frames;
   3.759 +
   3.760 +        for ( i = 0; i < n; i++ )
   3.761 +        {
   3.762 +            int arg = 0;
   3.763 +            /* simply update the PTE for the VA and invalidate TLB */
   3.764 +            call[i].op = __HYPERVISOR_update_va_mapping;
   3.765 +            call[i].args[arg++] = va;
   3.766 +            call[i].args[arg++] = 0;
   3.767 +#ifdef __i386__
   3.768 +            call[i].args[arg++] = 0;
   3.769 +#endif  
   3.770 +            call[i].args[arg++] = UVMF_INVLPG;
   3.771 +
   3.772 +            va += PAGE_SIZE;
   3.773 +        }
   3.774 +
   3.775 +        ret = HYPERVISOR_multicall(call, n);
   3.776 +        if ( ret )
   3.777 +        {
   3.778 +            printk("update_va_mapping hypercall failed with rc=%d.\n", ret);
   3.779 +            return -ret;
   3.780 +        }
   3.781 +
   3.782 +        for ( i = 0; i < n; i++ )
   3.783 +        {
   3.784 +            if ( call[i].result ) 
   3.785 +            {
   3.786 +                printk("update_va_mapping failed for with rc=%d.\n", ret);
   3.787 +                return -(call[i].result);
   3.788              }
   3.789          }
   3.790 -        if (y == n)
   3.791 -            break;
   3.792 +        num_frames -= n;
   3.793      }
   3.794 -    if (y != n) {
   3.795 -        printk("Failed to find %ld frames!\n", n);
   3.796 -        return 0;
   3.797 -    }
   3.798 -    return demand_map_area_start + x * PAGE_SIZE;
   3.799 +    return 0;
   3.800  }
   3.801  
   3.802 -void *map_frames_ex(unsigned long *f, unsigned long n, unsigned long stride,
   3.803 -	unsigned long increment, unsigned long alignment, domid_t id,
   3.804 -	int may_fail, unsigned long prot)
   3.805 +/*
   3.806 + * Check if a given MFN refers to real memory
   3.807 + */
   3.808 +static long system_ram_end_mfn;
   3.809 +int mfn_is_ram(unsigned long mfn)
   3.810  {
   3.811 -    unsigned long addr = allocate_ondemand(n, alignment);
   3.812 -
   3.813 -    if (!addr)
   3.814 -        return NULL;
   3.815 -
   3.816 -    /* Found it at x.  Map it in. */
   3.817 -    do_map_frames(addr, f, n, stride, increment, id, may_fail, prot);
   3.818 -
   3.819 -    return (void *)addr;
   3.820 +    /* very crude check if a given MFN is memory or not. Probably should
   3.821 +     * make this a little more sophisticated ;) */
   3.822 +    return (mfn <= system_ram_end_mfn) ? 1 : 0;
   3.823  }
   3.824  
   3.825 +
   3.826 +/*
   3.827 + * Clear some of the bootstrap memory
   3.828 + */
   3.829  static void clear_bootstrap(void)
   3.830  {
   3.831      pte_t nullpte = { };
   3.832 +    int rc;
   3.833  
   3.834      /* Use first page as the CoW zero page */
   3.835      memset(&_text, 0, PAGE_SIZE);
   3.836      mfn_zero = virt_to_mfn((unsigned long) &_text);
   3.837 -    if (HYPERVISOR_update_va_mapping(0, nullpte, UVMF_INVLPG))
   3.838 -	printk("Unable to unmap NULL page\n");
   3.839 +    if ( (rc = HYPERVISOR_update_va_mapping(0, nullpte, UVMF_INVLPG)) )
   3.840 +        printk("Unable to unmap NULL page. rc=%d\n", rc);
   3.841  }
   3.842  
   3.843  void arch_init_p2m(unsigned long max_pfn)
   3.844 @@ -570,19 +750,19 @@ void arch_init_p2m(unsigned long max_pfn
   3.845      unsigned long pfn;
   3.846      
   3.847      l3_list = (unsigned long *)alloc_page(); 
   3.848 -    for(pfn=0; pfn<max_pfn; pfn++)
   3.849 +    for ( pfn=0; pfn<max_pfn; pfn++ )
   3.850      {
   3.851 -        if(!(pfn % (L1_P2M_ENTRIES * L2_P2M_ENTRIES)))
   3.852 +        if ( !(pfn % (L1_P2M_ENTRIES * L2_P2M_ENTRIES)) )
   3.853          {
   3.854              l2_list = (unsigned long*)alloc_page();
   3.855 -            if((pfn >> L3_P2M_SHIFT) > 0)
   3.856 +            if ( (pfn >> L3_P2M_SHIFT) > 0 )
   3.857              {
   3.858                  printk("Error: Too many pfns.\n");
   3.859                  do_exit();
   3.860              }
   3.861              l3_list[(pfn >> L2_P2M_SHIFT)] = virt_to_mfn(l2_list);  
   3.862          }
   3.863 -        if(!(pfn % (L1_P2M_ENTRIES)))
   3.864 +        if ( !(pfn % (L1_P2M_ENTRIES)) )
   3.865          {
   3.866              l1_list = (unsigned long*)alloc_page();
   3.867              l2_list[(pfn >> L1_P2M_SHIFT) & L2_P2M_MASK] = 
   3.868 @@ -601,30 +781,34 @@ void arch_init_mm(unsigned long* start_p
   3.869  
   3.870      unsigned long start_pfn, max_pfn, virt_pfns;
   3.871  
   3.872 -    printk("  _text:        %p\n", &_text);
   3.873 -    printk("  _etext:       %p\n", &_etext);
   3.874 -    printk("  _erodata:     %p\n", &_erodata);
   3.875 -    printk("  _edata:       %p\n", &_edata);
   3.876 -    printk("  stack start:  %p\n", stack);
   3.877 -    printk("  _end:         %p\n", &_end);
   3.878 +    printk("      _text: %p(VA)\n", &_text);
   3.879 +    printk("     _etext: %p(VA)\n", &_etext);
   3.880 +    printk("   _erodata: %p(VA)\n", &_erodata);
   3.881 +    printk("     _edata: %p(VA)\n", &_edata);
   3.882 +    printk("stack start: %p(VA)\n", stack);
   3.883 +    printk("       _end: %p(VA)\n", &_end);
   3.884  
   3.885      /* First page follows page table pages and 3 more pages (store page etc) */
   3.886      start_pfn = PFN_UP(to_phys(start_info.pt_base)) + 
   3.887 -                start_info.nr_pt_frames + 3;
   3.888 +        start_info.nr_pt_frames + 3;
   3.889      max_pfn = start_info.nr_pages;
   3.890  
   3.891      /* We need room for demand mapping and heap, clip available memory */
   3.892      virt_pfns = DEMAND_MAP_PAGES + HEAP_PAGES;
   3.893 -    if (max_pfn + virt_pfns + 1 < max_pfn)
   3.894 +    if ( max_pfn + virt_pfns + 1 < max_pfn )
   3.895          max_pfn = -(virt_pfns + 1);
   3.896  
   3.897 -    printk("  start_pfn:    %lx\n", start_pfn);
   3.898 -    printk("  max_pfn:      %lx\n", max_pfn);
   3.899 +    printk("  start_pfn: %lx\n", start_pfn);
   3.900 +    printk("    max_pfn: %lx\n", max_pfn);
   3.901  
   3.902      build_pagetable(&start_pfn, &max_pfn);
   3.903      clear_bootstrap();
   3.904      set_readonly(&_text, &_erodata);
   3.905  
   3.906 +    /* get the number of physical pages the system has. Used to check for
   3.907 +     * system memory. */
   3.908 +    system_ram_end_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
   3.909 +
   3.910      *start_pfn_p = start_pfn;
   3.911      *max_pfn_p = max_pfn;
   3.912  }

     4.1 --- a/extras/mini-os/arch/x86/setup.c	Mon Mar 02 18:26:56 2009 +0900
     4.2 +++ b/extras/mini-os/arch/x86/setup.c	Fri Mar 06 12:22:22 2009 +0900
     4.3 @@ -63,10 +63,12 @@ void failsafe_callback(void);
     4.4  static
     4.5  shared_info_t *map_shared_info(unsigned long pa)
     4.6  {
     4.7 -	if ( HYPERVISOR_update_va_mapping(
     4.8 -		(unsigned long)shared_info, __pte(pa | 7), UVMF_INVLPG) )
     4.9 +    int rc;
    4.10 +
    4.11 +	if ( (rc = HYPERVISOR_update_va_mapping(
    4.12 +              (unsigned long)shared_info, __pte(pa | 7), UVMF_INVLPG)) )
    4.13  	{
    4.14 -		printk("Failed to map shared_info!!\n");
    4.15 +		printk("Failed to map shared_info!! rc=%d\n", rc);
    4.16  		do_exit();
    4.17  	}
    4.18  	return (shared_info_t *)shared_info;

     5.1 --- a/extras/mini-os/events.c	Mon Mar 02 18:26:56 2009 +0900
     5.2 +++ b/extras/mini-os/events.c	Fri Mar 06 12:22:22 2009 +0900
     5.3 @@ -42,19 +42,23 @@ void unbind_all_ports(void)
     5.4      int cpu = 0;
     5.5      shared_info_t *s = HYPERVISOR_shared_info;
     5.6      vcpu_info_t   *vcpu_info = &s->vcpu_info[cpu];
     5.7 +    int rc;
     5.8  
     5.9 -    for (i = 0; i < NR_EVS; i++)
    5.10 +    for ( i = 0; i < NR_EVS; i++ )
    5.11      {
    5.12 -        if (i == start_info.console.domU.evtchn ||
    5.13 -            i == start_info.store_evtchn)
    5.14 +        if ( i == start_info.console.domU.evtchn ||
    5.15 +             i == start_info.store_evtchn)
    5.16              continue;
    5.17 -        if (test_and_clear_bit(i, bound_ports))
    5.18 +
    5.19 +        if ( test_and_clear_bit(i, bound_ports) )
    5.20          {
    5.21              struct evtchn_close close;
    5.22              printk("port %d still bound!\n", i);
    5.23              mask_evtchn(i);
    5.24              close.port = i;
    5.25 -            HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
    5.26 +            rc = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
    5.27 +            if ( rc )
    5.28 +                printk("WARN: close_port %s failed rc=%d. ignored\n", i, rc);
    5.29              clear_evtchn(i);
    5.30          }
    5.31      }
    5.32 @@ -71,8 +75,9 @@ int do_event(evtchn_port_t port, struct 
    5.33  
    5.34      clear_evtchn(port);
    5.35  
    5.36 -    if (port >= NR_EVS) {
    5.37 -        printk("Port number too large: %d\n", port);
    5.38 +    if ( port >= NR_EVS )
    5.39 +    {
    5.40 +        printk("WARN: do_event(): Port number too large: %d\n", port);
    5.41          return 1;
    5.42      }
    5.43  
    5.44 @@ -89,9 +94,9 @@ int do_event(evtchn_port_t port, struct 
    5.45  evtchn_port_t bind_evtchn(evtchn_port_t port, evtchn_handler_t handler,
    5.46  						  void *data)
    5.47  {
    5.48 - 	if(ev_actions[port].handler != default_handler)
    5.49 + 	if ( ev_actions[port].handler != default_handler )
    5.50          printk("WARN: Handler for port %d already registered, replacing\n",
    5.51 -				port);
    5.52 +               port);
    5.53  
    5.54  	ev_actions[port].data = data;
    5.55  	wmb();
    5.56 @@ -104,8 +109,9 @@ evtchn_port_t bind_evtchn(evtchn_port_t 
    5.57  void unbind_evtchn(evtchn_port_t port )
    5.58  {
    5.59  	struct evtchn_close close;
    5.60 +    int rc;
    5.61  
    5.62 -	if (ev_actions[port].handler == default_handler)
    5.63 +	if ( ev_actions[port].handler == default_handler )
    5.64  		printk("WARN: No handler for port %d when unbinding\n", port);
    5.65  	mask_evtchn(port);
    5.66  	clear_evtchn(port);
    5.67 @@ -116,37 +122,43 @@ void unbind_evtchn(evtchn_port_t port )
    5.68  	clear_bit(port, bound_ports);
    5.69  
    5.70  	close.port = port;
    5.71 -	HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
    5.72 +	rc = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
    5.73 +    if ( rc )
    5.74 +        printk("WARN: close_port %s failed rc=%d. ignored\n", port, rc);
    5.75 +        
    5.76  }
    5.77  
    5.78  evtchn_port_t bind_virq(uint32_t virq, evtchn_handler_t handler, void *data)
    5.79  {
    5.80  	evtchn_bind_virq_t op;
    5.81 +    int rc;
    5.82  
    5.83  	/* Try to bind the virq to a port */
    5.84  	op.virq = virq;
    5.85  	op.vcpu = smp_processor_id();
    5.86  
    5.87 -	if ( HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &op) != 0 )
    5.88 +	if ( (rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &op)) != 0 )
    5.89  	{
    5.90 -		printk("Failed to bind virtual IRQ %d\n", virq);
    5.91 +		printk("Failed to bind virtual IRQ %d with rc=%d\n", virq, rc);
    5.92  		return -1;
    5.93      }
    5.94      bind_evtchn(op.port, handler, data);
    5.95  	return op.port;
    5.96  }
    5.97  
    5.98 -evtchn_port_t bind_pirq(uint32_t pirq, int will_share, evtchn_handler_t handler, void *data)
    5.99 +evtchn_port_t bind_pirq(uint32_t pirq, int will_share,
   5.100 +                        evtchn_handler_t handler, void *data)
   5.101  {
   5.102  	evtchn_bind_pirq_t op;
   5.103 +    int rc;
   5.104  
   5.105  	/* Try to bind the pirq to a port */
   5.106  	op.pirq = pirq;
   5.107  	op.flags = will_share ? BIND_PIRQ__WILL_SHARE : 0;
   5.108  
   5.109 -	if ( HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &op) != 0 )
   5.110 +	if ( (rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &op)) != 0 )
   5.111  	{
   5.112 -		printk("Failed to bind physical IRQ %d\n", pirq);
   5.113 +		printk("Failed to bind physical IRQ %d with rc=%d\n", pirq, rc);
   5.114  		return -1;
   5.115  	}
   5.116  	bind_evtchn(op.port, handler, data);
   5.117 @@ -173,7 +185,8 @@ void init_events(void)
   5.118      asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0));
   5.119      wrmsrl(0xc0000101, &cpu0_pda); /* 0xc0000101 is MSR_GS_BASE */
   5.120      cpu0_pda.irqcount = -1;
   5.121 -    cpu0_pda.irqstackptr = (void*) (((unsigned long)irqstack + 2 * STACK_SIZE) & ~(STACK_SIZE - 1));
   5.122 +    cpu0_pda.irqstackptr = (void*) (((unsigned long)irqstack + 2 * STACK_SIZE)
   5.123 +                                    & ~(STACK_SIZE - 1));
   5.124  #endif
   5.125      /* initialize event handler */
   5.126      for ( i = 0; i < NR_EVS; i++ )
   5.127 @@ -207,15 +220,19 @@ void default_handler(evtchn_port_t port,
   5.128  int evtchn_alloc_unbound(domid_t pal, evtchn_handler_t handler,
   5.129  						 void *data, evtchn_port_t *port)
   5.130  {
   5.131 -    int err;
   5.132 +    int rc;
   5.133 +
   5.134      evtchn_alloc_unbound_t op;
   5.135      op.dom = DOMID_SELF;
   5.136      op.remote_dom = pal;
   5.137 -    err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, &op);
   5.138 -    if (err)
   5.139 -		return err;
   5.140 +    rc = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, &op);
   5.141 +    if ( rc )
   5.142 +    {
   5.143 +        printk("ERROR: alloc_unbound failed with rc=%d", rc);
   5.144 +		return rc;
   5.145 +    }
   5.146      *port = bind_evtchn(op.port, handler, data);
   5.147 -    return err;
   5.148 +    return rc;
   5.149  }
   5.150  
   5.151  /* Connect to a port so as to allow the exchange of notifications with
   5.152 @@ -225,15 +242,28 @@ int evtchn_bind_interdomain(domid_t pal,
   5.153  			    evtchn_handler_t handler, void *data,
   5.154  			    evtchn_port_t *local_port)
   5.155  {
   5.156 -    int err;
   5.157 +    int rc;
   5.158      evtchn_port_t port;
   5.159      evtchn_bind_interdomain_t op;
   5.160      op.remote_dom = pal;
   5.161      op.remote_port = remote_port;
   5.162 -    err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &op);
   5.163 -    if (err)
   5.164 -		return err;
   5.165 +    rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &op);
   5.166 +    if ( rc )
   5.167 +    {
   5.168 +        printk("ERROR: bind_interdomain failed with rc=%d", rc);
   5.169 +		return rc;
   5.170 +    }
   5.171      port = op.local_port;
   5.172      *local_port = bind_evtchn(port, handler, data);
   5.173 -    return err;
   5.174 +    return rc;
   5.175  }
   5.176 +
   5.177 +/*
   5.178 + * Local variables:
   5.179 + * mode: C
   5.180 + * c-set-style: "BSD"
   5.181 + * c-basic-offset: 4
   5.182 + * tab-width: 4
   5.183 + * indent-tabs-mode: nil
   5.184 + * End:
   5.185 + */

     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/extras/mini-os/include/ioremap.h	Fri Mar 06 12:22:22 2009 +0900
     6.3 @@ -0,0 +1,33 @@
     6.4 +/**
     6.5 + * Copyright (C) 2009 Netronome Systems, Inc.  All rights reserved.
     6.6 + *
     6.7 + * Permission is hereby granted, free of charge, to any person obtaining a copy
     6.8 + * of this software and associated documentation files (the "Software"), to
     6.9 + * deal in the Software without restriction, including without limitation the
    6.10 + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
    6.11 + * sell copies of the Software, and to permit persons to whom the Software is
    6.12 + * furnished to do so, subject to the following conditions:
    6.13 + * 
    6.14 + * The above copyright notice and this permission notice shall be included in
    6.15 + * all copies or substantial portions of the Software.
    6.16 + * 
    6.17 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
    6.18 + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
    6.19 + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
    6.20 + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
    6.21 + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
    6.22 + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
    6.23 + * DEALINGS IN THE SOFTWARE.
    6.24 + */
    6.25 +
    6.26 +
    6.27 +#ifndef _IOREMAP_H_
    6.28 +#define _IOREMAP_H_
    6.29 +
    6.30 +void *ioremap(unsigned long phys_addr, unsigned long size);
    6.31 +void *ioremap_nocache(unsigned long phys_addr, unsigned long size);
    6.32 +void iounmap(void *virt_addr, unsigned long size);
    6.33 +
    6.34 +#endif /* _IOREMAP_H_ */
    6.35 +
    6.36 +/* -*-  Mode:C; c-basic-offset:4; tab-width:4 indent-tabs-mode:nil -*- */

     7.1 --- a/extras/mini-os/include/mm.h	Mon Mar 02 18:26:56 2009 +0900
     7.2 +++ b/extras/mini-os/include/mm.h	Fri Mar 06 12:22:22 2009 +0900
     7.3 @@ -71,6 +71,7 @@ void *map_frames_ex(unsigned long *f, un
     7.4  void do_map_frames(unsigned long addr,
     7.5          unsigned long *f, unsigned long n, unsigned long stride,
     7.6  	unsigned long increment, domid_t id, int may_fail, unsigned long prot);
     7.7 +int unmap_frames(unsigned long va, unsigned long num_frames);
     7.8  #ifdef HAVE_LIBC
     7.9  extern unsigned long heap, brk, heap_mapped, heap_end;
    7.10  #endif

     8.1 --- a/extras/mini-os/include/x86/arch_mm.h	Mon Mar 02 18:26:56 2009 +0900
     8.2 +++ b/extras/mini-os/include/x86/arch_mm.h	Fri Mar 06 12:22:22 2009 +0900
     8.3 @@ -133,6 +133,10 @@ typedef unsigned long pgentry_t;
     8.4  #define L4_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_DIRTY|_PAGE_USER)
     8.5  #endif /* __i386__ || __x86_64__ */
     8.6  
     8.7 +/* flags for ioremap */
     8.8 +#define IO_PROT (L1_PROT)
     8.9 +#define IO_PROT_NOCACHE (L1_PROT | _PAGE_PCD)
    8.10 +
    8.11  #include "arch_limits.h"
    8.12  #define PAGE_SIZE       __PAGE_SIZE
    8.13  #define PAGE_SHIFT      __PAGE_SHIFT
    8.14 @@ -222,5 +226,6 @@ static __inline__ paddr_t machine_to_phy
    8.15  #define do_map_zero(start, n) do_map_frames(start, &mfn_zero, n, 0, 0, DOMID_SELF, 0, L1_PROT_RO)
    8.16  
    8.17  pgentry_t *need_pgt(unsigned long addr);
    8.18 +int mfn_is_ram(unsigned long mfn);
    8.19  
    8.20  #endif /* _ARCH_MM_H_ */

     9.1 --- a/extras/mini-os/kernel.c	Mon Mar 02 18:26:56 2009 +0900
     9.2 +++ b/extras/mini-os/kernel.c	Fri Mar 06 12:22:22 2009 +0900
     9.3 @@ -490,14 +490,16 @@ void start_kernel(start_info_t *si)
     9.4  
     9.5      /* print out some useful information  */
     9.6      printk("Xen Minimal OS!\n");
     9.7 -    printk("start_info:   %p\n",    si);
     9.8 -    printk("  nr_pages:   %lu",     si->nr_pages);
     9.9 -    printk("  shared_inf: %08lx\n", si->shared_info);
    9.10 -    printk("  pt_base:    %p",      (void *)si->pt_base); 
    9.11 -    printk("  mod_start:  0x%lx\n", si->mod_start);
    9.12 -    printk("  mod_len:    %lu\n",   si->mod_len); 
    9.13 -    printk("  flags:      0x%x\n",  (unsigned int)si->flags);
    9.14 -    printk("  cmd_line:   %s\n",  
    9.15 +    printk("  start_info: %p(VA)\n", si);
    9.16 +    printk("    nr_pages: 0x%lx\n", si->nr_pages);
    9.17 +    printk("  shared_inf: 0x%08lx(MA)\n", si->shared_info);
    9.18 +    printk("     pt_base: %p(VA)\n", (void *)si->pt_base); 
    9.19 +    printk("nr_pt_frames: 0x%lx\n", si->nr_pt_frames);
    9.20 +    printk("    mfn_list: %p(VA)\n", (void *)si->mfn_list); 
    9.21 +    printk("   mod_start: 0x%lx(VA)\n", si->mod_start);
    9.22 +    printk("     mod_len: %lu\n", si->mod_len); 
    9.23 +    printk("       flags: 0x%x\n", (unsigned int)si->flags);
    9.24 +    printk("    cmd_line: %s\n",  
    9.25             si->cmd_line ? (const char *)si->cmd_line : "NULL");
    9.26  
    9.27      /* Set up events. */

    10.1 --- a/extras/mini-os/lib/sys.c	Mon Mar 02 18:26:56 2009 +0900
    10.2 +++ b/extras/mini-os/lib/sys.c	Fri Mar 06 12:22:22 2009 +0900
    10.3 @@ -1206,47 +1206,15 @@ void *mmap(void *start, size_t length, i
    10.4      } else ASSERT(0);
    10.5  }
    10.6  
    10.7 -#define UNMAP_BATCH ((STACK_SIZE / 2) / sizeof(multicall_entry_t))
    10.8  int munmap(void *start, size_t length)
    10.9  {
   10.10      int total = length / PAGE_SIZE;
   10.11 -    ASSERT(!((unsigned long)start & ~PAGE_MASK));
   10.12 -    while (total) {
   10.13 -        int n = UNMAP_BATCH;
   10.14 -        if (n > total)
   10.15 -            n = total;
   10.16 -        {
   10.17 -            int i;
   10.18 -            multicall_entry_t call[n];
   10.19 -            unsigned char (*data)[PAGE_SIZE] = start;
   10.20 -            int ret;
   10.21 +    int ret;
   10.22  
   10.23 -            for (i = 0; i < n; i++) {
   10.24 -                int arg = 0;
   10.25 -                call[i].op = __HYPERVISOR_update_va_mapping;
   10.26 -                call[i].args[arg++] = (unsigned long) &data[i];
   10.27 -                call[i].args[arg++] = 0;
   10.28 -#ifdef __i386__
   10.29 -                call[i].args[arg++] = 0;
   10.30 -#endif
   10.31 -                call[i].args[arg++] = UVMF_INVLPG;
   10.32 -            }
   10.33 -
   10.34 -            ret = HYPERVISOR_multicall(call, n);
   10.35 -            if (ret) {
   10.36 -                errno = -ret;
   10.37 -                return -1;
   10.38 -            }
   10.39 -
   10.40 -            for (i = 0; i < n; i++) {
   10.41 -                if (call[i].result) {
   10.42 -                    errno = call[i].result;
   10.43 -                    return -1;
   10.44 -                }
   10.45 -            }
   10.46 -        }
   10.47 -        start = (char *)start + n * PAGE_SIZE;
   10.48 -        total -= n;
   10.49 +    ret = unmap_frames((unsigned long)start, (unsigned long)total);
   10.50 +    if (ret) {
   10.51 +        errno = ret;
   10.52 +        return -1;
   10.53      }
   10.54      return 0;
   10.55  }

    11.1 --- a/tools/blktap/drivers/block-qcow2.c	Mon Mar 02 18:26:56 2009 +0900
    11.2 +++ b/tools/blktap/drivers/block-qcow2.c	Fri Mar 06 12:22:22 2009 +0900
    11.3 @@ -1980,6 +1980,91 @@ static int qcow_validate_parent(struct d
    11.4  	return 0;
    11.5  }
    11.6  
    11.7 +int qcow2_create(const char *filename, uint64_t total_size,
    11.8 +                      const char *backing_file, int flags)
    11.9 +{
   11.10 +    int fd, header_size, backing_filename_len, l1_size, i, shift, l2_bits;
   11.11 +    QCowHeader header;
   11.12 +    uint64_t tmp, offset;
   11.13 +    QCowCreateState s1, *s = &s1;
   11.14 +
   11.15 +    memset(s, 0, sizeof(*s));
   11.16 +
   11.17 +    fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
   11.18 +    if (fd < 0)
   11.19 +        return -1;
   11.20 +    memset(&header, 0, sizeof(header));
   11.21 +    header.magic = cpu_to_be32(QCOW_MAGIC);
   11.22 +    header.version = cpu_to_be32(QCOW_VERSION);
   11.23 +    header.size = cpu_to_be64(total_size * 512);
   11.24 +    header_size = sizeof(header);
   11.25 +    backing_filename_len = 0;
   11.26 +    if (backing_file) {
   11.27 +        header.backing_file_offset = cpu_to_be64(header_size);
   11.28 +        backing_filename_len = strlen(backing_file);
   11.29 +        header.backing_file_size = cpu_to_be32(backing_filename_len);
   11.30 +        header_size += backing_filename_len;
   11.31 +    }
   11.32 +    s->cluster_bits = 12;  /* 4 KB clusters */
   11.33 +    s->cluster_size = 1 << s->cluster_bits;
   11.34 +    header.cluster_bits = cpu_to_be32(s->cluster_bits);
   11.35 +    header_size = (header_size + 7) & ~7;
   11.36 +    if (flags & BLOCK_FLAG_ENCRYPT) {
   11.37 +        header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES);
   11.38 +    } else {
   11.39 +        header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE);
   11.40 +    }
   11.41 +    l2_bits = s->cluster_bits - 3;
   11.42 +    shift = s->cluster_bits + l2_bits;
   11.43 +    l1_size = (((total_size * 512) + (1LL << shift) - 1) >> shift);
   11.44 +    offset = align_offset(header_size, s->cluster_size);
   11.45 +    s->l1_table_offset = offset;
   11.46 +    header.l1_table_offset = cpu_to_be64(s->l1_table_offset);
   11.47 +    header.l1_size = cpu_to_be32(l1_size);
   11.48 +    offset += align_offset(l1_size * sizeof(uint64_t), s->cluster_size);
   11.49 +
   11.50 +    s->refcount_table = qemu_mallocz(s->cluster_size);
   11.51 +    s->refcount_block = qemu_mallocz(s->cluster_size);
   11.52 +
   11.53 +    s->refcount_table_offset = offset;
   11.54 +    header.refcount_table_offset = cpu_to_be64(offset);
   11.55 +    header.refcount_table_clusters = cpu_to_be32(1);
   11.56 +    offset += s->cluster_size;
   11.57 +
   11.58 +    s->refcount_table[0] = cpu_to_be64(offset);
   11.59 +    s->refcount_block_offset = offset;
   11.60 +    offset += s->cluster_size;
   11.61 +
   11.62 +    /* update refcounts */
   11.63 +    create_refcount_update(s, 0, header_size);
   11.64 +    create_refcount_update(s, s->l1_table_offset, l1_size * sizeof(uint64_t));
   11.65 +    create_refcount_update(s, s->refcount_table_offset, s->cluster_size);
   11.66 +    create_refcount_update(s, s->refcount_block_offset, s->cluster_size);
   11.67 +
   11.68 +    /* write all the data */
   11.69 +    write(fd, &header, sizeof(header));
   11.70 +    if (backing_file) {
   11.71 +        write(fd, backing_file, backing_filename_len);
   11.72 +    }
   11.73 +    lseek(fd, s->l1_table_offset, SEEK_SET);
   11.74 +    tmp = 0;
   11.75 +    for(i = 0;i < l1_size; i++) {
   11.76 +        write(fd, &tmp, sizeof(tmp));
   11.77 +    }
   11.78 +    lseek(fd, s->refcount_table_offset, SEEK_SET);
   11.79 +    write(fd, s->refcount_table, s->cluster_size);
   11.80 +
   11.81 +    lseek(fd, s->refcount_block_offset, SEEK_SET);
   11.82 +    write(fd, s->refcount_block, s->cluster_size);
   11.83 +
   11.84 +    qemu_free(s->refcount_table);
   11.85 +    qemu_free(s->refcount_block);
   11.86 +    close(fd);
   11.87 +    return 0;
   11.88 +}
   11.89 +
   11.90 +
   11.91 +
   11.92  struct tap_disk tapdisk_qcow2 = {
   11.93  	"qcow2",
   11.94  	sizeof(BDRVQcowState),

    12.1 --- a/tools/blktap/drivers/qcow-create.c	Mon Mar 02 18:26:56 2009 +0900
    12.2 +++ b/tools/blktap/drivers/qcow-create.c	Fri Mar 06 12:22:22 2009 +0900
    12.3 @@ -52,7 +52,7 @@ static void help(void)
    12.4  {
    12.5  	fprintf(stderr, "Qcow-utils: v1.0.0\n");
    12.6  	fprintf(stderr, 
    12.7 -		"usage: qcow-create [-h help] [-r reserve] <SIZE(MB)> <FILENAME> "
    12.8 +		"usage: qcow-create [-h help] [-r reserve] [-f format] <SIZE(MB)> <FILENAME> "
    12.9  		"[<BACKING_FILENAME>]\n"); 
   12.10  	exit(-1);
   12.11  }
   12.12 @@ -61,11 +61,13 @@ int main(int argc, char *argv[])
   12.13  {
   12.14  	int ret = -1, c, backed = 0;
   12.15  	int sparse =  1;
   12.16 +	char *fmt = "qcow";
   12.17  	uint64_t size;
   12.18  	char filename[MAX_NAME_LEN], bfilename[MAX_NAME_LEN];
   12.19 +	char *tmpfile;
   12.20  
   12.21          for(;;) {
   12.22 -                c = getopt(argc, argv, "hr");
   12.23 +                c = getopt(argc, argv, "hrf");
   12.24                  if (c == -1)
   12.25                          break;
   12.26                  switch(c) {
   12.27 @@ -73,6 +75,9 @@ int main(int argc, char *argv[])
   12.28                          help();
   12.29                          exit(0);
   12.30                          break;
   12.31 +                case 'f':
   12.32 +                        fmt = argv[optind++];
   12.33 +                        break;
   12.34                  case 'r':
   12.35  			sparse = 0;
   12.36  			break;
   12.37 @@ -105,11 +110,16 @@ int main(int argc, char *argv[])
   12.38  		}
   12.39  	}
   12.40  
   12.41 -	DFPRINTF("Creating file size %llu, name %s\n",(long long unsigned)size, filename);
   12.42 -	if (!backed)
   12.43 -		ret = qcow_create(filename,size,NULL,sparse);
   12.44 -	else
   12.45 -		ret = qcow_create(filename,size,bfilename,sparse);
   12.46 +    tmpfile = backed ? bfilename: NULL; 
   12.47 +    if (!strcmp(fmt, "qcow")) {
   12.48 +        ret = qcow_create(filename, size, tmpfile, sparse);
   12.49 +    } else if(!strcmp(fmt, "qcow2")) {
   12.50 +        ret = qcow2_create(filename, size, tmpfile, sparse);
   12.51 +    } else {
   12.52 +        fprintf(stderr,"Unsupport format:%s\n", fmt);
   12.53 +        exit(-1);
   12.54 +    } 
   12.55 +    DFPRINTF("Creating file size %llu, name %s\n",(long long unsigned)size, filename);
   12.56  
   12.57  	if (ret < 0)
   12.58  		DPRINTF("Unable to create QCOW file\n");

    13.1 --- a/tools/blktap/drivers/tapdisk.h	Mon Mar 02 18:26:56 2009 +0900
    13.2 +++ b/tools/blktap/drivers/tapdisk.h	Fri Mar 06 12:22:22 2009 +0900
    13.3 @@ -266,4 +266,7 @@ typedef struct fd_list_entry {
    13.4  
    13.5  int qcow_create(const char *filename, uint64_t total_size,
    13.6  		const char *backing_file, int flags);
    13.7 +
    13.8 +int qcow2_create(const char *filename, uint64_t total_size,
    13.9 +		const char *backing_file, int flags);
   13.10  #endif /*TAPDISK_H_*/

    14.1 --- a/tools/examples/xmexample.hvm	Mon Mar 02 18:26:56 2009 +0900
    14.2 +++ b/tools/examples/xmexample.hvm	Fri Mar 06 12:22:22 2009 +0900
    14.3 @@ -7,11 +7,11 @@
    14.4  #============================================================================
    14.5  
    14.6  import os, re
    14.7 +
    14.8 +arch_libdir = 'lib'
    14.9  arch = os.uname()[4]
   14.10 -if re.search('64', arch):
   14.11 +if os.uname()[0] == 'Linux' and re.search('64', arch):
   14.12      arch_libdir = 'lib64'
   14.13 -else:
   14.14 -    arch_libdir = 'lib'
   14.15  
   14.16  #----------------------------------------------------------------------------
   14.17  # Kernel image file.

    15.1 --- a/tools/examples/xmexample.vti	Mon Mar 02 18:26:56 2009 +0900
    15.2 +++ b/tools/examples/xmexample.vti	Fri Mar 06 12:22:22 2009 +0900
    15.3 @@ -7,8 +7,10 @@
    15.4  #============================================================================
    15.5  
    15.6  import os, re
    15.7 +arch_libdir = 'lib'
    15.8  arch = os.uname()[4]
    15.9 -arch_libdir = 'lib'
   15.10 +if os.uname()[0] == 'Linux' and re.search('64', arch):
   15.11 +    arch_libdir = 'lib64'
   15.12  
   15.13  #----------------------------------------------------------------------------
   15.14  # Kernel image file.

    16.1 --- a/tools/firmware/Makefile	Mon Mar 02 18:26:56 2009 +0900
    16.2 +++ b/tools/firmware/Makefile	Fri Mar 06 12:22:22 2009 +0900
    16.3 @@ -15,13 +15,13 @@ SUBDIRS += hvmloader
    16.4  .PHONY: all
    16.5  all:
    16.6  	@set -e; if [ $$((`( bcc -v 2>&1 | grep version || echo 0.0.0 ) | cut -d' ' -f 3 | awk -F. '{ printf "0x%02x%02x%02x", $$1, $$2, $$3}'`)) -lt $$((0x00100e)) ] ; then \
    16.7 -	echo "==========================================================="; \
    16.8 -	echo "Require dev86 package version >= 0.16.14 to build firmware!"; \
    16.9 -	echo "(visit http://www.cix.co.uk/~mayday for more information)"; \
   16.10 -	echo "==========================================================="; \
   16.11 -	else \
   16.12 +	echo "==========================================================================="; \
   16.13 +	echo "Require dev86 rpm or bin86 & bcc debs version >= 0.16.14 to build firmware!"; \
   16.14 +	echo "(visit http://www.debath.co.uk/dev86/ for more information)"; \
   16.15 +	echo "==========================================================================="; \
   16.16 +	false ; \
   16.17 +	fi
   16.18  	$(MAKE) subdirs-$@; \
   16.19 -	fi
   16.20  
   16.21  
   16.22  .PHONY: install

    17.1 --- a/tools/hotplug/Linux/init.d/xend	Mon Mar 02 18:26:56 2009 +0900
    17.2 +++ b/tools/hotplug/Linux/init.d/xend	Fri Mar 06 12:22:22 2009 +0900
    17.3 @@ -39,11 +39,13 @@ function await_daemons_up
    17.4  
    17.5  case "$1" in
    17.6    start)
    17.7 +	touch /var/lock/subsys/xend
    17.8  	xend start
    17.9  	await_daemons_up
   17.10  	;;
   17.11    stop)
   17.12  	xend stop
   17.13 +	rm -f /var/lock/subsys/xend
   17.14  	;;
   17.15    status)
   17.16  	xend status

    18.1 --- a/tools/hotplug/Linux/xen-hotplug-cleanup	Mon Mar 02 18:26:56 2009 +0900
    18.2 +++ b/tools/hotplug/Linux/xen-hotplug-cleanup	Fri Mar 06 12:22:22 2009 +0900
    18.3 @@ -11,6 +11,13 @@ dir=$(dirname "$0")
    18.4  # This is pretty horrible, but there's not really a nicer way of solving this.
    18.5  claim_lock "block"
    18.6  
    18.7 +# split backend/DEVCLASS/VMID/DEVID on slashes
    18.8 +path_array=( ${XENBUS_PATH//\// } )
    18.9 +# get /vm/UUID path
   18.10 +vm=$(xenstore-read "/local/domain/${path_array[2]}/vm")
   18.11 +# construct /vm/UUID/device/DEVCLASS/DEVID
   18.12 +vm_dev="$vm/device/${path_array[1]}/${path_array[3]}"
   18.13 +
   18.14  # remove device frontend store entries
   18.15  xenstore-rm -t \
   18.16    $(xenstore-read "$XENBUS_PATH/frontend" 2>/dev/null) 2>/dev/null || true
   18.17 @@ -19,4 +26,7 @@ xenstore-rm -t \
   18.18  xenstore-rm -t "$XENBUS_PATH"        2>/dev/null || true
   18.19  xenstore-rm -t "error/$XENBUS_PATH"  2>/dev/null || true
   18.20  
   18.21 +# remove device path from /vm/UUID
   18.22 +xenstore-rm -t "$vm_dev" 2>/dev/null || true
   18.23 +
   18.24  release_lock "block"

    19.1 --- a/tools/libxc/xc_domain.c	Mon Mar 02 18:26:56 2009 +0900
    19.2 +++ b/tools/libxc/xc_domain.c	Fri Mar 06 12:22:22 2009 +0900
    19.3 @@ -920,7 +920,8 @@ int xc_domain_update_msi_irq(
    19.4      uint32_t domid,
    19.5      uint32_t gvec,
    19.6      uint32_t pirq,
    19.7 -    uint32_t gflags)
    19.8 +    uint32_t gflags,
    19.9 +    uint64_t gtable)
   19.10  {
   19.11      int rc;
   19.12      xen_domctl_bind_pt_irq_t *bind;
   19.13 @@ -936,6 +937,7 @@ int xc_domain_update_msi_irq(
   19.14      bind->machine_irq = pirq;
   19.15      bind->u.msi.gvec = gvec;
   19.16      bind->u.msi.gflags = gflags;
   19.17 +    bind->u.msi.gtable = gtable;
   19.18  
   19.19      rc = do_domctl(xc_handle, &domctl);
   19.20      return rc;

    20.1 --- a/tools/libxc/xenctrl.h	Mon Mar 02 18:26:56 2009 +0900
    20.2 +++ b/tools/libxc/xenctrl.h	Fri Mar 06 12:22:22 2009 +0900
    20.3 @@ -1092,7 +1092,8 @@ int xc_domain_update_msi_irq(
    20.4      uint32_t domid,
    20.5      uint32_t gvec,
    20.6      uint32_t pirq,
    20.7 -    uint32_t gflags);
    20.8 +    uint32_t gflags,
    20.9 +    uint64_t gtable);
   20.10  
   20.11  int xc_domain_unbind_msi_irq(int xc_handle,
   20.12                               uint32_t domid,

    21.1 --- a/tools/libxen/src/xen_common.c	Mon Mar 02 18:26:56 2009 +0900
    21.2 +++ b/tools/libxen/src/xen_common.c	Fri Mar 06 12:22:22 2009 +0900
    21.3 @@ -90,6 +90,8 @@ add_param(xmlNode *, const char *, const
    21.4  static xmlNode *
    21.5  add_param_struct(xmlNode *);
    21.6  static xmlNode *
    21.7 +add_param_array(xmlNode *);
    21.8 +static xmlNode *
    21.9  add_struct_array(xmlNode *, const char *);
   21.10  static xmlNode *
   21.11  add_nested_struct(xmlNode *, const char *);
   21.12 @@ -1292,7 +1294,7 @@ make_body_add_type(enum abstract_typenam
   21.13          const struct abstract_type *member_type = v->type->child;
   21.14          arbitrary_set *set_val = v->u.struct_val;
   21.15          abstract_value v;
   21.16 -        xmlNode *data_node = add_param_struct(params_node);
   21.17 +        xmlNode *data_node = add_param_array(params_node);
   21.18  
   21.19          for (size_t i = 0; i < set_val->size; i++)
   21.20          {
   21.21 @@ -1611,6 +1613,16 @@ add_param_struct(xmlNode *params_node)
   21.22  }
   21.23  
   21.24  
   21.25 +static xmlNode *
   21.26 +add_param_array(xmlNode *params_node)
   21.27 +{
   21.28 +    xmlNode *param_node = add_container(params_node, "param");
   21.29 +    xmlNode *value_node = add_container(param_node,  "value");
   21.30 +
   21.31 +    return xmlNewChild(value_node, NULL, BAD_CAST "array", NULL);
   21.32 +}
   21.33 +
   21.34 +
   21.35  static void
   21.36  add_struct_member(xmlNode *struct_node, const char *name, const char *type,
   21.37                    const char *value)

    22.1 --- a/tools/python/xen/lowlevel/acm/acm.c	Mon Mar 02 18:26:56 2009 +0900
    22.2 +++ b/tools/python/xen/lowlevel/acm/acm.c	Fri Mar 06 12:22:22 2009 +0900
    22.3 @@ -68,8 +68,6 @@ static void *__getssid(int domid, uint32
    22.4          goto out2;
    22.5      } else {
    22.6          *buflen = SSID_BUFFER_SIZE;
    22.7 -        free(buf);
    22.8 -        buf = NULL;
    22.9          goto out2;
   22.10      }
   22.11   out2:

    23.1 --- a/tools/python/xen/util/blkif.py	Mon Mar 02 18:26:56 2009 +0900
    23.2 +++ b/tools/python/xen/util/blkif.py	Fri Mar 06 12:22:22 2009 +0900
    23.3 @@ -33,7 +33,7 @@ def blkdev_name_to_number(name):
    23.4          major = scsi_major[((ord(n[7:8]) - ord('a') + 1) * 26 + (ord(n[8:9]) - ord('a'))) / 16 ]
    23.5          minor = (((ord(n[7:8]) - ord('a') + 1 ) * 26 + (ord(n[8:9]) - ord('a'))) % 16) * 16 + int(n[9:] or 0)
    23.6          devnum = major * 256 + minor
    23.7 -    elif re.match( '/dev/hd[a-t]([1-9]|[1-5][0-9]|6[0-3])?', n):
    23.8 +    elif re.match( '/dev/hd[a-t]([1-9]|[1-5][0-9]|6[0-3])?$', n):
    23.9          ide_majors = [ 3, 22, 33, 34, 56, 57, 88, 89, 90, 91 ]
   23.10          major = ide_majors[(ord(n[7:8]) - ord('a')) / 2]
   23.11          minor = ((ord(n[7:8]) - ord('a')) % 2) * 64 + int(n[8:] or 0)

    24.1 --- a/tools/python/xen/util/pci.py	Mon Mar 02 18:26:56 2009 +0900
    24.2 +++ b/tools/python/xen/util/pci.py	Fri Mar 06 12:22:22 2009 +0900
    24.3 @@ -66,9 +66,10 @@ PCI_EXP_DEVCTL_FLR = (0x1 << 15)
    24.4  
    24.5  PCI_CAP_ID_PM = 0x01
    24.6  PCI_PM_CTRL = 4
    24.7 -PCI_PM_CTRL_NO_SOFT_RESET = 0x0004
    24.8 +PCI_PM_CTRL_NO_SOFT_RESET = 0x0008
    24.9  PCI_PM_CTRL_STATE_MASK = 0x0003
   24.10  PCI_D3hot = 3
   24.11 +PCI_D0hot = 0
   24.12  
   24.13  VENDOR_INTEL  = 0x8086
   24.14  PCI_CAP_ID_VENDOR_SPECIFIC_CAP = 0x09
   24.15 @@ -234,7 +235,7 @@ def find_all_devices_owned_by_pciback():
   24.16      return dev_list
   24.17  
   24.18  def transform_list(target, src):
   24.19 -    ''' src: its element is pci string (Format: xxxx:xx:xx:x).
   24.20 +    ''' src: its element is pci string (Format: xxxx:xx:xx.x).
   24.21          target: its element is pci string, or a list of pci string.
   24.22  
   24.23          If all the elements in src are in target, we remove them from target
   24.24 @@ -467,12 +468,12 @@ class PciDevice:
   24.25          os.lseek(fd, PCI_CB_BRIDGE_CONTROL, 0)
   24.26          br_cntl |= PCI_BRIDGE_CTL_BUS_RESET
   24.27          os.write(fd, struct.pack('H', br_cntl))
   24.28 -        time.sleep(0.200)
   24.29 +        time.sleep(0.100)
   24.30          # De-assert Secondary Bus Reset
   24.31          os.lseek(fd, PCI_CB_BRIDGE_CONTROL, 0)
   24.32          br_cntl &= ~PCI_BRIDGE_CTL_BUS_RESET
   24.33          os.write(fd, struct.pack('H', br_cntl))
   24.34 -        time.sleep(0.200)
   24.35 +        time.sleep(0.100)
   24.36          os.close(fd)
   24.37  
   24.38          # Restore the config spaces
   24.39 @@ -483,18 +484,25 @@ class PciDevice:
   24.40          if pos == 0:
   24.41              return False
   24.42          
   24.43 +        # No_Soft_Reset - When set 1, this bit indicates that
   24.44 +        # devices transitioning from D3hot to D0 because of
   24.45 +        # PowerState commands do not perform an internal reset.
   24.46 +        pm_ctl = self.pci_conf_read32(pos + PCI_PM_CTRL)
   24.47 +        if (pm_ctl & PCI_PM_CTRL_NO_SOFT_RESET) == 1:
   24.48 +            return False
   24.49 +
   24.50          (pci_list, cfg_list) = save_pci_conf_space([self.name])
   24.51          
   24.52 -        # Enter D3hot without soft reset
   24.53 -        pm_ctl = self.pci_conf_read32(pos + PCI_PM_CTRL)
   24.54 -        pm_ctl |= PCI_PM_CTRL_NO_SOFT_RESET
   24.55 +        # Enter D3hot
   24.56          pm_ctl &= ~PCI_PM_CTRL_STATE_MASK
   24.57          pm_ctl |= PCI_D3hot
   24.58          self.pci_conf_write32(pos + PCI_PM_CTRL, pm_ctl)
   24.59          time.sleep(0.010)
   24.60  
   24.61          # From D3hot to D0
   24.62 -        self.pci_conf_write32(pos + PCI_PM_CTRL, 0)
   24.63 +        pm_ctl &= ~PCI_PM_CTRL_STATE_MASK
   24.64 +        pm_ctl |= PCI_D0hot
   24.65 +        self.pci_conf_write32(pos + PCI_PM_CTRL, pm_ctl)
   24.66          time.sleep(0.010)
   24.67  
   24.68          restore_pci_conf_space((pci_list, cfg_list))
   24.69 @@ -516,7 +524,7 @@ class PciDevice:
   24.70          (pci_list, cfg_list) = save_pci_conf_space([self.name])
   24.71  
   24.72          self.pci_conf_write8(pos + PCI_USB_FLRCTRL, 1)
   24.73 -        time.sleep(0.010)
   24.74 +        time.sleep(0.100)
   24.75  
   24.76          restore_pci_conf_space((pci_list, cfg_list))
   24.77  
   24.78 @@ -636,7 +644,7 @@ class PciDevice:
   24.79                  self.dev_type = DEV_TYPE_PCI_BRIDGE
   24.80              else:
   24.81                  creg = self.pci_conf_read16(pos + PCI_EXP_FLAGS)
   24.82 -                if ((creg & PCI_EXP_TYPE_PCI_BRIDGE) >> 4) == \
   24.83 +                if ((creg & PCI_EXP_FLAGS_TYPE) >> 4) == \
   24.84                      PCI_EXP_TYPE_PCI_BRIDGE:
   24.85                      self.dev_type = DEV_TYPE_PCI_BRIDGE
   24.86                  else:
   24.87 @@ -701,7 +709,7 @@ class PciDevice:
   24.88                  pos = self.find_cap_offset(PCI_CAP_ID_EXP)
   24.89                  self.pci_conf_write32(pos + PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_FLR)
   24.90                  # We must sleep at least 100ms for the completion of FLR
   24.91 -                time.sleep(0.200)
   24.92 +                time.sleep(0.100)
   24.93                  restore_pci_conf_space((pci_list, cfg_list))
   24.94              else:
   24.95                  if self.bus == 0:
   24.96 @@ -722,7 +730,7 @@ class PciDevice:
   24.97                  # We use Advanced Capability to do FLR.
   24.98                  pos = self.find_cap_offset(PCI_CAP_ID_AF)
   24.99                  self.pci_conf_write8(pos + PCI_AF_CTL, PCI_AF_CTL_FLR)
  24.100 -                time.sleep(0.200)
  24.101 +                time.sleep(0.100)
  24.102                  restore_pci_conf_space((pci_list, cfg_list))
  24.103              else:
  24.104                  if self.bus == 0:

    25.1 --- a/tools/python/xen/xend/XendCheckpoint.py	Mon Mar 02 18:26:56 2009 +0900
    25.2 +++ b/tools/python/xen/xend/XendCheckpoint.py	Fri Mar 06 12:22:22 2009 +0900
    25.3 @@ -66,6 +66,13 @@ def insert_after(list, pred, value):
    25.4  
    25.5  
    25.6  def save(fd, dominfo, network, live, dst, checkpoint=False, node=-1):
    25.7 +    try:
    25.8 +        if not os.path.isdir("/var/lib/xen"):
    25.9 +            os.makedirs("/var/lib/xen")
   25.10 +    except Exception, exn:
   25.11 +        log.exception("Can't create directory '/var/lib/xen'")
   25.12 +        raise XendError("Can't create directory '/var/lib/xen'")
   25.13 +
   25.14      write_exact(fd, SIGNATURE, "could not write guest state file: signature")
   25.15  
   25.16      sxprep = dominfo.sxpr()
   25.17 @@ -166,6 +173,13 @@ def save(fd, dominfo, network, live, dst
   25.18  
   25.19  
   25.20  def restore(xd, fd, dominfo = None, paused = False, relocating = False):
   25.21 +    try:
   25.22 +        if not os.path.isdir("/var/lib/xen"):
   25.23 +            os.makedirs("/var/lib/xen")
   25.24 +    except Exception, exn:
   25.25 +        log.exception("Can't create directory '/var/lib/xen'")
   25.26 +        raise XendError("Can't create directory '/var/lib/xen'")
   25.27 +
   25.28      signature = read_exact(fd, len(SIGNATURE),
   25.29          "not a valid guest state file: signature read")
   25.30      if signature != SIGNATURE:

    26.1 --- a/tools/python/xen/xend/XendConfig.py	Mon Mar 02 18:26:56 2009 +0900
    26.2 +++ b/tools/python/xen/xend/XendConfig.py	Fri Mar 06 12:22:22 2009 +0900
    26.3 @@ -216,6 +216,7 @@ XENAPI_CFG_TYPES = {
    26.4      'cpuid_check' : dict,
    26.5      'machine_address_size': int,
    26.6      'suppress_spurious_page_faults': bool0,
    26.7 +    's3_integrity' : int,
    26.8  }
    26.9  
   26.10  # List of legacy configuration keys that have no equivalent in the
   26.11 @@ -1275,6 +1276,7 @@ class XendConfig(dict):
   26.12                  vscsi_dict = self.vscsi_convert_sxp_to_dict(config)
   26.13                  vscsi_devs = vscsi_dict['devs']
   26.14                  vscsi_mode = vscsi_dict['feature-host']
   26.15 +                vscsi_be = vscsi_dict.get('backend', None)
   26.16  
   26.17                  # create XenAPI DSCSI objects.
   26.18                  for vscsi_dev in vscsi_devs:
   26.19 @@ -1294,6 +1296,8 @@ class XendConfig(dict):
   26.20                      'feature-host': vscsi_mode,
   26.21                      'uuid': vscsi_devs_uuid
   26.22                  }
   26.23 +                if vscsi_be is not None:
   26.24 +                    vscsi_info['backend'] = vscsi_be
   26.25                  target['devices'][vscsi_devs_uuid] = (dev_type, vscsi_info)
   26.26                  log.debug("XendConfig: reading device: %s,%s" % \
   26.27                            (vscsi_devs, vscsi_mode))
   26.28 @@ -1621,6 +1625,7 @@ class XendConfig(dict):
   26.29          # [device,
   26.30          #   [vscsi,
   26.31          #     [feature-host, 0],
   26.32 +        #     [backend, 0],
   26.33          #     [dev,
   26.34          #       [devid, 0], [p-devname, sdb], [p-dev, 1:0:0:1],
   26.35          #       [v-dev, 0:0:0:0], [state, 1]
   26.36 @@ -1632,6 +1637,7 @@ class XendConfig(dict):
   26.37          #   ],
   26.38          #   [vscsi,
   26.39          #     [feature-host, 1],
   26.40 +        #     [backend, 0],
   26.41          #     [dev,
   26.42          #       [devid, 1], [p-devname, sdg], [p-dev, 2:0:0:0],
   26.43          #       [v-dev, 1:0:0:0], [state, 1]
   26.44 @@ -1653,6 +1659,7 @@ class XendConfig(dict):
   26.45          # [device,
   26.46          #   [vscsi,
   26.47          #     [feature-host, 0],
   26.48 +        #     [backend, 0],
   26.49          #     [dev,
   26.50          #       [devid, 0], [p-devname, sdd], [p-dev, 1:0:0:3],
   26.51          #       [v-dev, 0:0:0:2], [state, 1]
   26.52 @@ -1668,7 +1675,7 @@ class XendConfig(dict):
   26.53          #
   26.54          # { devs: [ {devid: 0, p-devname: sdd, p-dev: 1:0:0:3,
   26.55          #            v-dev: 0:0:0:2, state: 1} ],
   26.56 -        #   feature-host: 1 }
   26.57 +        #   feature-host: 1 , backend: 0 }
   26.58  
   26.59          dev_config = {}
   26.60  
   26.61 @@ -1689,6 +1696,11 @@ class XendConfig(dict):
   26.62  
   26.63          vscsi_mode = sxp.children(dev_sxp, 'feature-host')[0]
   26.64          dev_config['feature-host'] = vscsi_mode[1]
   26.65 +        try:
   26.66 +            vscsi_be = sxp.children(dev_sxp, 'backend')[0]
   26.67 +            dev_config['backend'] = vscsi_be[1]
   26.68 +        except IndexError:
   26.69 +            pass
   26.70  
   26.71          return dev_config
   26.72  
   26.73 @@ -1803,6 +1815,7 @@ class XendConfig(dict):
   26.74                  vscsi_dict = self.vscsi_convert_sxp_to_dict(config)
   26.75                  vscsi_devs = vscsi_dict['devs']
   26.76                  vscsi_mode = vscsi_dict['feature-host']
   26.77 +                vscsi_be = vscsi_dict.get('backend', None)
   26.78  
   26.79                  # destroy existing XenAPI DSCSI objects
   26.80                  for dscsi_uuid in XendDSCSI.get_by_VM(self['uuid']):
   26.81 @@ -1826,6 +1839,8 @@ class XendConfig(dict):
   26.82                      'feature-host': vscsi_mode,
   26.83                      'uuid': dev_uuid
   26.84                  }
   26.85 +                if vscsi_be is not None:
   26.86 +                    vscsi_info['backend'] = vscsi_be
   26.87                  self['devices'][dev_uuid] = (dev_type, vscsi_info)
   26.88                  return True
   26.89                  
   26.90 @@ -1919,6 +1934,8 @@ class XendConfig(dict):
   26.91                  elif dev_type == 'vscsi':
   26.92                      sxpr = ['vscsi', ['uuid', dev_info['uuid']],
   26.93                                       ['feature-host', dev_info['feature-host']]]
   26.94 +                    if dev_info.has_key('backend'):
   26.95 +                        sxpr.append(['backend', dev_info['backend']])
   26.96                  for pci_dev_info in dev_info['devs']:
   26.97                      pci_dev_sxpr = ['dev']
   26.98                      for opt, val in pci_dev_info.items():

    27.1 --- a/tools/python/xen/xend/XendDomainInfo.py	Mon Mar 02 18:26:56 2009 +0900
    27.2 +++ b/tools/python/xen/xend/XendDomainInfo.py	Fri Mar 06 12:22:22 2009 +0900
    27.3 @@ -900,6 +900,11 @@ class XendDomainInfo:
    27.4              new_dev_sxp = ['vscsi']
    27.5              cur_mode = sxp.children(cur_dev_sxp, 'feature-host')[0]
    27.6              new_dev_sxp.append(cur_mode)
    27.7 +            try:
    27.8 +                cur_be = sxp.children(cur_dev_sxp, 'backend')[0]
    27.9 +                new_dev_sxp.append(cur_be)
   27.10 +            except IndexError:
   27.11 +                pass
   27.12  
   27.13              for cur_dev in sxp.children(cur_dev_sxp, 'dev'):
   27.14                  if state == xenbusState['Closing']:
   27.15 @@ -2207,12 +2212,17 @@ class XendDomainInfo:
   27.16              if security.has_authorization(ssidref) == False:
   27.17                  raise VmError("VM is not authorized to run.")
   27.18  
   27.19 +        s3_integrity = 0
   27.20 +        if self.info.has_key('s3_integrity'):
   27.21 +            s3_integrity = self.info['s3_integrity']
   27.22 +        flags = (int(hvm) << 0) | (int(hap) << 1) | (int(s3_integrity) << 2)
   27.23 +
   27.24          try:
   27.25              self.domid = xc.domain_create(
   27.26                  domid = 0,
   27.27                  ssidref = ssidref,
   27.28                  handle = uuid.fromString(self.info['uuid']),
   27.29 -                flags = (int(hvm) << 0) | (int(hap) << 1),
   27.30 +                flags = flags,
   27.31                  target = self.info.target())
   27.32          except Exception, e:
   27.33              # may get here if due to ACM the operation is not permitted

    28.1 --- a/tools/python/xen/xend/server/DevController.py	Mon Mar 02 18:26:56 2009 +0900
    28.2 +++ b/tools/python/xen/xend/server/DevController.py	Fri Mar 06 12:22:22 2009 +0900
    28.3 @@ -235,8 +235,8 @@ class DevController:
    28.4                  xstransact.Remove(backpath)
    28.5              xstransact.Remove(frontpath)
    28.6  
    28.7 -        # xstransact.Remove(self.devicePath()) ?? Below is the same ?
    28.8 -        self.vm._removeVm("device/%s/%d" % (self.deviceClass, dev))
    28.9 +            # xstransact.Remove(self.devicePath()) ?? Below is the same ?
   28.10 +            self.vm._removeVm("device/%s/%d" % (self.deviceClass, dev))
   28.11  
   28.12      def configurations(self, transaction = None):
   28.13          return map(lambda x: self.configuration(x, transaction), self.deviceIDs(transaction))

    29.1 --- a/tools/python/xen/xm/create.py	Mon Mar 02 18:26:56 2009 +0900
    29.2 +++ b/tools/python/xen/xm/create.py	Fri Mar 06 12:22:22 2009 +0900
    29.3 @@ -579,6 +579,11 @@ gopts.var('hap', val='HAP',
    29.4            use="""Hap status (0=hap is disabled;
    29.5            1=hap is enabled.""")
    29.6  
    29.7 +gopts.var('s3_integrity', val='TBOOT_MEMORY_PROTECT',
    29.8 +          fn=set_int, default=1,
    29.9 +          use="""Should domain memory integrity be verified during S3?
   29.10 +          (0=protection is disabled; 1=protection is enabled.""")
   29.11 +
   29.12  gopts.var('cpuid', val="IN[,SIN]:eax=EAX,ebx=EBX,ecx=ECX,edx=EDX",
   29.13            fn=append_value, default=[],
   29.14            use="""Cpuid description.""")
   29.15 @@ -832,6 +837,10 @@ def configure_security(config, vals):
   29.16      elif num > 1:
   29.17          err("VM config error: Multiple access_control definitions!")
   29.18  
   29.19 +def configure_mem_prot(config_image, vals):
   29.20 +    """Create the config for S3 memory integrity verification under tboot.
   29.21 +    """
   29.22 +    config_image.append(['s3_integrity', vals.s3_integrity])
   29.23  
   29.24  def configure_vtpm(config_devs, vals):
   29.25      """Create the config for virtual TPM interfaces.
   29.26 @@ -964,6 +973,7 @@ def make_config(vals):
   29.27              else:
   29.28                  config.append(['bootloader_args', '-q'])
   29.29      config.append(['image', config_image])
   29.30 +    configure_mem_prot(config, vals);
   29.31  
   29.32      config_devs = []
   29.33      configure_disks(config_devs, vals)

    30.1 --- a/tools/python/xen/xm/xenapi_create.py	Mon Mar 02 18:26:56 2009 +0900
    30.2 +++ b/tools/python/xen/xm/xenapi_create.py	Fri Mar 06 12:22:22 2009 +0900
    30.3 @@ -269,6 +269,8 @@ class xenapi_create:
    30.4                  vm.attributes["is_a_template"].value == 'true',
    30.5              "auto_power_on":
    30.6                  vm.attributes["auto_power_on"].value == 'true',
    30.7 +            "s3_integrity":
    30.8 +                vm.attributes["s3_integrity"].value,
    30.9              "memory_static_max":
   30.10                  get_child_node_attribute(vm, "memory", "static_max"),
   30.11              "memory_static_min":
   30.12 @@ -650,6 +652,8 @@ class sxp2xml:
   30.13              = str(get_child_by_name(config, "vcpus", 1))
   30.14          vm.attributes["vcpus_at_startup"] \
   30.15              = str(get_child_by_name(config, "vcpus", 1))
   30.16 +        vm.attributes["s3_integrity"] \
   30.17 +            = str(get_child_by_name(config, "s3_integrity", 0))
   30.18  
   30.19          sec_data = get_child_by_name(config, "security")
   30.20          if sec_data:

    31.1 --- a/xen/Rules.mk	Mon Mar 02 18:26:56 2009 +0900
    31.2 +++ b/xen/Rules.mk	Fri Mar 06 12:22:22 2009 +0900
    31.3 @@ -38,6 +38,7 @@ ALL_OBJS-y               += $(BASEDIR)/c
    31.4  ALL_OBJS-y               += $(BASEDIR)/drivers/built_in.o
    31.5  ALL_OBJS-y               += $(BASEDIR)/xsm/built_in.o
    31.6  ALL_OBJS-y               += $(BASEDIR)/arch/$(TARGET_ARCH)/built_in.o
    31.7 +ALL_OBJS-$(x86)          += $(BASEDIR)/crypto/built_in.o
    31.8  
    31.9  CFLAGS-y                += -g -D__XEN__
   31.10  CFLAGS-$(XSM_ENABLE)    += -DXSM_ENABLE

    32.1 --- a/xen/arch/ia64/vmx/vmx_interrupt.c	Mon Mar 02 18:26:56 2009 +0900
    32.2 +++ b/xen/arch/ia64/vmx/vmx_interrupt.c	Fri Mar 06 12:22:22 2009 +0900
    32.3 @@ -154,3 +154,14 @@ void hvm_isa_irq_deassert(struct domain 
    32.4  {
    32.5      /* dummy */
    32.6  }
    32.7 +
    32.8 +int msixtbl_pt_register(struct domain *d, int pirq, uint64_t gtable)
    32.9 +{
   32.10 +    /* dummy */
   32.11 +    return -ENOSYS;
   32.12 +}
   32.13 +
   32.14 +void msixtbl_pt_unregister(struct domain *d, int pirq)
   32.15 +{
   32.16 +    /* dummy */
   32.17 +}

    33.1 --- a/xen/arch/ia64/xen/mm.c	Mon Mar 02 18:26:56 2009 +0900
    33.2 +++ b/xen/arch/ia64/xen/mm.c	Fri Mar 06 12:22:22 2009 +0900
    33.3 @@ -3236,9 +3236,56 @@ int get_page_type(struct page_info *page
    33.4      return 1;
    33.5  }
    33.6  
    33.7 -int page_is_conventional_ram(unsigned long mfn)
    33.8 +int page_is_ram_type(unsigned long mfn, unsigned long type)
    33.9  {
   33.10 -    return (efi_mem_type(pfn_to_paddr(mfn)) == EFI_CONVENTIONAL_MEMORY);
   33.11 +    u32 mem_type = efi_mem_type(pfn_to_paddr(mfn));
   33.12 +
   33.13 +    if (type & RAM_TYPE_CONVENTIONAL)
   33.14 +    {
   33.15 +        switch (mem_type)
   33.16 +        {
   33.17 +        case EFI_BOOT_SERVICES_CODE:
   33.18 +        case EFI_BOOT_SERVICES_DATA:
   33.19 +        case EFI_LOADER_CODE:
   33.20 +        case EFI_LOADER_DATA:
   33.21 +        case EFI_CONVENTIONAL_MEMORY:
   33.22 +            return 1;
   33.23 +        default:
   33.24 +            break;
   33.25 +        }       
   33.26 +    }
   33.27 +    if (type & RAM_TYPE_RESERVED)
   33.28 +    {
   33.29 +        switch (mem_type)
   33.30 +        {
   33.31 +        case EFI_RUNTIME_SERVICES_CODE:
   33.32 +        case EFI_RUNTIME_SERVICES_DATA:
   33.33 +        case EFI_RESERVED_TYPE:
   33.34 +        case EFI_MEMORY_MAPPED_IO:
   33.35 +        case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
   33.36 +        case EFI_PAL_CODE:
   33.37 +            return 1;
   33.38 +        default:
   33.39 +            break;
   33.40 +        }
   33.41 +    }
   33.42 +    if (type & RAM_TYPE_ACPI)
   33.43 +    {
   33.44 +        switch (mem_type)
   33.45 +        {
   33.46 +        case EFI_ACPI_RECLAIM_MEMORY:
   33.47 +        case EFI_ACPI_MEMORY_NVS:
   33.48 +            return 1;
   33.49 +        default:
   33.50 +            break;
   33.51 +        }
   33.52 +    }
   33.53 +    else if (type & RAM_TYPE_UNUSABLE)
   33.54 +    {
   33.55 +        return (mem_type == EFI_UNUSABLE_MEMORY);
   33.56 +    }
   33.57 +
   33.58 +    return 0;
   33.59  }
   33.60  
   33.61  

    34.1 --- a/xen/arch/x86/acpi/cpu_idle.c	Mon Mar 02 18:26:56 2009 +0900
    34.2 +++ b/xen/arch/x86/acpi/cpu_idle.c	Fri Mar 06 12:22:22 2009 +0900
    34.3 @@ -55,6 +55,7 @@ static void (*lapic_timer_on)(void);
    34.4  
    34.5  extern u32 pmtmr_ioport;
    34.6  extern void (*pm_idle) (void);
    34.7 +extern void (*dead_idle) (void);
    34.8  
    34.9  static void (*pm_idle_save) (void) __read_mostly;
   34.10  unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER - 1;
   34.11 @@ -369,6 +370,43 @@ static void acpi_processor_idle(void)
   34.12          cpuidle_current_governor->reflect(power);
   34.13  }
   34.14  
   34.15 +static void acpi_dead_idle(void)
   34.16 +{
   34.17 +    struct acpi_processor_power *power;
   34.18 +    struct acpi_processor_cx *cx;
   34.19 +    int unused;
   34.20 +
   34.21 +    if ( (power = processor_powers[smp_processor_id()]) == NULL )
   34.22 +        goto default_halt;
   34.23 +
   34.24 +    if ( (cx = &power->states[power->count-1]) == NULL )
   34.25 +        goto default_halt;
   34.26 +
   34.27 +    for ( ; ; )
   34.28 +    {
   34.29 +        if ( !power->flags.bm_check && cx->type == ACPI_STATE_C3 )
   34.30 +            ACPI_FLUSH_CPU_CACHE();
   34.31 +
   34.32 +        switch ( cx->entry_method )
   34.33 +        {
   34.34 +            case ACPI_CSTATE_EM_FFH:
   34.35 +                /* Not treat interrupt as break event */
   34.36 +                mwait_idle_with_hints(cx->address, 0);
   34.37 +                break;
   34.38 +            case ACPI_CSTATE_EM_SYSIO:
   34.39 +                inb(cx->address);
   34.40 +                unused = inl(pmtmr_ioport);
   34.41 +                break;
   34.42 +            default:
   34.43 +                goto default_halt;
   34.44 +        }
   34.45 +    }
   34.46 +
   34.47 +default_halt:
   34.48 +    for ( ; ; )
   34.49 +        halt();
   34.50 +}
   34.51 +
   34.52  static int init_cx_pminfo(struct acpi_processor_power *acpi_power)
   34.53  {
   34.54      int i;
   34.55 @@ -740,6 +778,11 @@ long set_cx_pminfo(uint32_t cpu, struct 
   34.56          pm_idle_save = pm_idle;
   34.57          pm_idle = acpi_processor_idle;
   34.58      }
   34.59 +
   34.60 +    if ( cpu_id == 0 )
   34.61 +    {
   34.62 +        dead_idle = acpi_dead_idle;
   34.63 +    }
   34.64          
   34.65      return 0;
   34.66  }

    35.1 --- a/xen/arch/x86/acpi/power.c	Mon Mar 02 18:26:56 2009 +0900
    35.2 +++ b/xen/arch/x86/acpi/power.c	Fri Mar 06 12:22:22 2009 +0900
    35.3 @@ -190,6 +190,7 @@ static int enter_state(u32 state)
    35.4      case ACPI_STATE_S3:
    35.5          do_suspend_lowlevel();
    35.6          system_reset_counter++;
    35.7 +        error = tboot_s3_resume();
    35.8          break;
    35.9      case ACPI_STATE_S5:
   35.10          acpi_enter_sleep_state(ACPI_STATE_S5);
   35.11 @@ -206,7 +207,10 @@ static int enter_state(u32 state)
   35.12  
   35.13      device_power_up();
   35.14  
   35.15 -    printk(XENLOG_INFO "Finishing wakeup from ACPI S%d state.", state);
   35.16 +    printk(XENLOG_INFO "Finishing wakeup from ACPI S%d state.\n", state);
   35.17 +
   35.18 +    if ( (state == ACPI_STATE_S3) && error )
   35.19 +        panic("Memory integrity was lost on resume (%d)\n", error);
   35.20  
   35.21   done:
   35.22      spin_debug_enable();
   35.23 @@ -318,7 +322,7 @@ static void tboot_sleep(u8 sleep_state)
   35.24  
   35.25      tboot_shutdown(shutdown_type);
   35.26  }
   35.27 -         
   35.28 +
   35.29  /* System is really put into sleep state by this stub */
   35.30  acpi_status asmlinkage acpi_enter_sleep_state(u8 sleep_state)
   35.31  {

    36.1 --- a/xen/arch/x86/cpu/mcheck/mce_intel.c	Mon Mar 02 18:26:56 2009 +0900
    36.2 +++ b/xen/arch/x86/cpu/mcheck/mce_intel.c	Fri Mar 06 12:22:22 2009 +0900
    36.3 @@ -359,12 +359,6 @@ static int do_cmci_discover(int i)
    36.4          return 0;
    36.5      }
    36.6      set_bit(i, __get_cpu_var(mce_banks_owned));
    36.7 -    /* Clear Corected Error Counter field, make sure CMCI could 
    36.8 -     * be triggered on the new owner
    36.9 -     */
   36.10 -    msr = MSR_IA32_MC0_STATUS + 4 * i;
   36.11 -    rdmsrl(msr, val);
   36.12 -    wrmsrl(msr, val & ~MCi_STATUS_ERRCOUNT);
   36.13  out:
   36.14      clear_bit(i, __get_cpu_var(no_cmci_banks));
   36.15      return 1;
   36.16 @@ -374,6 +368,7 @@ static void cmci_discover(void)
   36.17  {
   36.18      unsigned long flags;
   36.19      int i;
   36.20 +    struct mc_info *mi = NULL;
   36.21  
   36.22      printk(KERN_DEBUG "CMCI: find owner on CPU%d\n", smp_processor_id());
   36.23  
   36.24 @@ -385,6 +380,18 @@ static void cmci_discover(void)
   36.25  
   36.26      spin_unlock_irqrestore(&cmci_discover_lock, flags);
   36.27  
   36.28 +    /* In case CMCI happended when do owner change.
   36.29 +     * If CMCI happened yet not processed immediately,
   36.30 +     * MCi_status (error_count bit 38~52) is not cleared,
   36.31 +     * the CMCI interrupt will never be triggered again.
   36.32 +     */
   36.33 +    mi = machine_check_poll(MC_FLAG_CMCI);
   36.34 +    if (mi) {
   36.35 +        x86_mcinfo_dump(mi);
   36.36 +        if (dom0 && guest_enabled_event(dom0->vcpu[0], VIRQ_MCA))
   36.37 +            send_guest_global_virq(dom0, VIRQ_MCA);
   36.38 +    }
   36.39 +
   36.40      printk(KERN_DEBUG "CMCI: CPU%d owner_map[%lx], no_cmci_map[%lx]\n", 
   36.41             smp_processor_id(), 
   36.42             *((unsigned long *)__get_cpu_var(mce_banks_owned)), 

    37.1 --- a/xen/arch/x86/cpu/mcheck/x86_mca.h	Mon Mar 02 18:26:56 2009 +0900
    37.2 +++ b/xen/arch/x86/cpu/mcheck/x86_mca.h	Fri Mar 06 12:22:22 2009 +0900
    37.3 @@ -46,8 +46,6 @@
    37.4  #define MCi_STATUS_MSEC         0x00000000ffff0000ULL
    37.5  /* Other information */
    37.6  #define MCi_STATUS_OTHER        0x01ffffff00000000ULL
    37.7 -/*Corrected Error Count*/
    37.8 -#define MCi_STATUS_ERRCOUNT     0x001FFFC0000000000ULL 
    37.9  /* processor context corrupt */
   37.10  #define MCi_STATUS_PCC          0x0200000000000000ULL
   37.11  /* MSR_K8_MCi_ADDR register valid */

    38.1 --- a/xen/arch/x86/domain.c	Mon Mar 02 18:26:56 2009 +0900
    38.2 +++ b/xen/arch/x86/domain.c	Fri Mar 06 12:22:22 2009 +0900
    38.3 @@ -58,7 +58,9 @@ DEFINE_PER_CPU(u64, efer);
    38.4  DEFINE_PER_CPU(unsigned long, cr4);
    38.5  
    38.6  static void default_idle(void);
    38.7 +static void default_dead_idle(void);
    38.8  void (*pm_idle) (void) = default_idle;
    38.9 +void (*dead_idle) (void) = default_dead_idle;
   38.10  
   38.11  static void paravirt_ctxt_switch_from(struct vcpu *v);
   38.12  static void paravirt_ctxt_switch_to(struct vcpu *v);
   38.13 @@ -84,6 +86,12 @@ static void default_idle(void)
   38.14          local_irq_enable();
   38.15  }
   38.16  
   38.17 +static void default_dead_idle(void)
   38.18 +{
   38.19 +    for ( ; ; )
   38.20 +        halt();
   38.21 +}
   38.22 +
   38.23  static void play_dead(void)
   38.24  {
   38.25      /*
   38.26 @@ -102,8 +110,7 @@ static void play_dead(void)
   38.27  
   38.28      /* With physical CPU hotplug, we should halt the cpu. */
   38.29      local_irq_disable();
   38.30 -    for ( ; ; )
   38.31 -        halt();
   38.32 +    (*dead_idle)();
   38.33  }
   38.34  
   38.35  void idle_loop(void)
   38.36 @@ -379,6 +386,8 @@ int arch_domain_create(struct domain *d,
   38.37          hvm_funcs.hap_supported &&
   38.38          (domcr_flags & DOMCRF_hap);
   38.39  
   38.40 +    d->arch.s3_integrity = !!(domcr_flags & DOMCRF_s3_integrity);
   38.41 +
   38.42      INIT_LIST_HEAD(&d->arch.pdev_list);
   38.43  
   38.44      d->arch.relmem = RELMEM_not_started;
   38.45 @@ -831,7 +840,7 @@ map_vcpu_info(struct vcpu *v, unsigned l
   38.46       * lost.  The domain will get a spurious event, but it can cope.
   38.47       */
   38.48      vcpu_info(v, evtchn_upcall_pending) = 1;
   38.49 -    for ( i = 0; i < BITS_PER_GUEST_LONG(d); i++ )
   38.50 +    for ( i = 0; i < BITS_PER_EVTCHN_WORD(d); i++ )
   38.51          set_bit(i, &vcpu_info(v, evtchn_pending_sel));
   38.52  
   38.53      return 0;

    39.1 --- a/xen/arch/x86/domctl.c	Mon Mar 02 18:26:56 2009 +0900
    39.2 +++ b/xen/arch/x86/domctl.c	Fri Mar 06 12:22:22 2009 +0900
    39.3 @@ -494,7 +494,8 @@ long arch_do_domctl(
    39.4              break;
    39.5          }
    39.6  
    39.7 -        domctl->u.address_size.size = BITS_PER_GUEST_LONG(d);
    39.8 +        domctl->u.address_size.size =
    39.9 +            is_pv_32on64_domain(d) ? 32 : BITS_PER_LONG;
   39.10  
   39.11          ret = 0;
   39.12          rcu_unlock_domain(d);

    40.1 --- a/xen/arch/x86/hvm/hvm.c	Mon Mar 02 18:26:56 2009 +0900
    40.2 +++ b/xen/arch/x86/hvm/hvm.c	Fri Mar 06 12:22:22 2009 +0900
    40.3 @@ -308,6 +308,9 @@ int hvm_domain_initialise(struct domain 
    40.4      spin_lock_init(&d->arch.hvm_domain.irq_lock);
    40.5      spin_lock_init(&d->arch.hvm_domain.uc_lock);
    40.6  
    40.7 +    INIT_LIST_HEAD(&d->arch.hvm_domain.msixtbl_list);
    40.8 +    spin_lock_init(&d->arch.hvm_domain.msixtbl_list_lock);
    40.9 +
   40.10      hvm_init_guest_time(d);
   40.11  
   40.12      d->arch.hvm_domain.params[HVM_PARAM_HPET_ENABLED] = 1;
   40.13 @@ -348,11 +351,15 @@ int hvm_domain_initialise(struct domain 
   40.14      return rc;
   40.15  }
   40.16  
   40.17 +extern void msixtbl_pt_cleanup(struct domain *d);
   40.18 +
   40.19  void hvm_domain_relinquish_resources(struct domain *d)
   40.20  {
   40.21      hvm_destroy_ioreq_page(d, &d->arch.hvm_domain.ioreq);
   40.22      hvm_destroy_ioreq_page(d, &d->arch.hvm_domain.buf_ioreq);
   40.23  
   40.24 +    msixtbl_pt_cleanup(d);
   40.25 +
   40.26      /* Stop all asynchronous timer actions. */
   40.27      rtc_deinit(d);
   40.28      if ( d->vcpu[0] != NULL )

    41.1 --- a/xen/arch/x86/hvm/intercept.c	Mon Mar 02 18:26:56 2009 +0900
    41.2 +++ b/xen/arch/x86/hvm/intercept.c	Fri Mar 06 12:22:22 2009 +0900
    41.3 @@ -35,14 +35,16 @@
    41.4  extern struct hvm_mmio_handler hpet_mmio_handler;
    41.5  extern struct hvm_mmio_handler vlapic_mmio_handler;
    41.6  extern struct hvm_mmio_handler vioapic_mmio_handler;
    41.7 +extern struct hvm_mmio_handler msixtbl_mmio_handler;
    41.8  
    41.9 -#define HVM_MMIO_HANDLER_NR 3
   41.10 +#define HVM_MMIO_HANDLER_NR 4
   41.11  
   41.12  static struct hvm_mmio_handler *hvm_mmio_handlers[HVM_MMIO_HANDLER_NR] =
   41.13  {
   41.14      &hpet_mmio_handler,
   41.15      &vlapic_mmio_handler,
   41.16 -    &vioapic_mmio_handler
   41.17 +    &vioapic_mmio_handler,
   41.18 +    &msixtbl_mmio_handler
   41.19  };
   41.20  
   41.21  static int hvm_mmio_access(struct vcpu *v,

    42.1 --- a/xen/arch/x86/hvm/vmsi.c	Mon Mar 02 18:26:56 2009 +0900
    42.2 +++ b/xen/arch/x86/hvm/vmsi.c	Fri Mar 06 12:22:22 2009 +0900
    42.3 @@ -193,3 +193,286 @@ int vmsi_deliver(struct domain *d, int p
    42.4      return 1;
    42.5  }
    42.6  
    42.7 +/* MSI-X mask bit hypervisor interception */
    42.8 +struct msixtbl_entry
    42.9 +{
   42.10 +    struct list_head list;
   42.11 +    atomic_t refcnt;    /* how many bind_pt_irq called for the device */
   42.12 +
   42.13 +    /* TODO: resolve the potential race by destruction of pdev */
   42.14 +    struct pci_dev *pdev;
   42.15 +    unsigned long gtable;       /* gpa of msix table */
   42.16 +    unsigned long table_len;
   42.17 +    unsigned long table_flags[MAX_MSIX_TABLE_ENTRIES / BITS_PER_LONG + 1];
   42.18 +
   42.19 +    struct rcu_head rcu;
   42.20 +};
   42.21 +
   42.22 +static struct msixtbl_entry *msixtbl_find_entry(
   42.23 +    struct vcpu *v, unsigned long addr)
   42.24 +{
   42.25 +    struct msixtbl_entry *entry;
   42.26 +    struct domain *d = v->domain;
   42.27 +
   42.28 +    list_for_each_entry( entry, &d->arch.hvm_domain.msixtbl_list, list )
   42.29 +        if ( addr >= entry->gtable &&
   42.30 +             addr < entry->gtable + entry->table_len )
   42.31 +            return entry;
   42.32 +
   42.33 +    return NULL;
   42.34 +}
   42.35 +
   42.36 +static void __iomem *msixtbl_addr_to_virt(
   42.37 +    struct msixtbl_entry *entry, unsigned long addr)
   42.38 +{
   42.39 +    int idx, nr_page;
   42.40 +
   42.41 +    if ( !entry )
   42.42 +        return NULL;
   42.43 +
   42.44 +    nr_page = (addr >> PAGE_SHIFT) -
   42.45 +              (entry->gtable >> PAGE_SHIFT);
   42.46 +
   42.47 +    if ( !entry->pdev )
   42.48 +        return NULL;
   42.49 +
   42.50 +    idx = entry->pdev->msix_table_idx[nr_page];
   42.51 +    if ( !idx )
   42.52 +        return NULL;
   42.53 +
   42.54 +    return (void *)(fix_to_virt(idx) +
   42.55 +                    (addr & ((1UL << PAGE_SHIFT) - 1)));
   42.56 +}
   42.57 +
   42.58 +static int msixtbl_read(
   42.59 +    struct vcpu *v, unsigned long address,
   42.60 +    unsigned long len, unsigned long *pval)
   42.61 +{
   42.62 +    unsigned long offset;
   42.63 +    struct msixtbl_entry *entry;
   42.64 +    void *virt;
   42.65 +    int r = X86EMUL_UNHANDLEABLE;
   42.66 +
   42.67 +    rcu_read_lock();
   42.68 +
   42.69 +    if ( len != 4 )
   42.70 +        goto out;
   42.71 +
   42.72 +    offset = address & (PCI_MSIX_ENTRY_SIZE - 1);
   42.73 +    if ( offset != PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET)
   42.74 +        goto out;
   42.75 +
   42.76 +    entry = msixtbl_find_entry(v, address);
   42.77 +    virt = msixtbl_addr_to_virt(entry, address);
   42.78 +    if ( !virt )
   42.79 +        goto out;
   42.80 +
   42.81 +    *pval = readl(virt);
   42.82 +    r = X86EMUL_OKAY;
   42.83 +
   42.84 +out:
   42.85 +    rcu_read_unlock();
   42.86 +    return r;
   42.87 +}
   42.88 +
   42.89 +static int msixtbl_write(struct vcpu *v, unsigned long address,
   42.90 +                        unsigned long len, unsigned long val)
   42.91 +{
   42.92 +    unsigned long offset;
   42.93 +    struct msixtbl_entry *entry;
   42.94 +    void *virt;
   42.95 +    int nr_entry;
   42.96 +    int r = X86EMUL_UNHANDLEABLE;
   42.97 +
   42.98 +    rcu_read_lock();
   42.99 +
  42.100 +    if ( len != 4 )
  42.101 +        goto out;
  42.102 +
  42.103 +    entry = msixtbl_find_entry(v, address);
  42.104 +    nr_entry = (address - entry->gtable) % PCI_MSIX_ENTRY_SIZE;
  42.105 +
  42.106 +    offset = address & (PCI_MSIX_ENTRY_SIZE - 1);
  42.107 +    if ( offset != PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET)
  42.108 +    {
  42.109 +        set_bit(nr_entry, &entry->table_flags);
  42.110 +        goto out;
  42.111 +    }
  42.112 +
  42.113 +    /* exit to device model if address/data has been modified */
  42.114 +    if ( test_and_clear_bit(nr_entry, &entry->table_flags) )
  42.115 +        goto out;
  42.116 +
  42.117 +    virt = msixtbl_addr_to_virt(entry, address);
  42.118 +    if ( !virt )
  42.119 +        goto out;
  42.120 +
  42.121 +    writel(val, virt);
  42.122 +    r = X86EMUL_OKAY;
  42.123 +
  42.124 +out:
  42.125 +    rcu_read_unlock();
  42.126 +    return r;
  42.127 +}
  42.128 +
  42.129 +static int msixtbl_range(struct vcpu *v, unsigned long addr)
  42.130 +{
  42.131 +    struct msixtbl_entry *entry;
  42.132 +    void *virt;
  42.133 +
  42.134 +    rcu_read_lock();
  42.135 +
  42.136 +    entry = msixtbl_find_entry(v, addr);
  42.137 +    virt = msixtbl_addr_to_virt(entry, addr);
  42.138 +
  42.139 +    rcu_read_unlock();
  42.140 +
  42.141 +    return !!virt;
  42.142 +}
  42.143 +
  42.144 +struct hvm_mmio_handler msixtbl_mmio_handler = {
  42.145 +    .check_handler = msixtbl_range,
  42.146 +    .read_handler = msixtbl_read,
  42.147 +    .write_handler = msixtbl_write
  42.148 +};
  42.149 +
  42.150 +static struct msixtbl_entry *add_msixtbl_entry(struct domain *d,
  42.151 +                                               struct pci_dev *pdev,
  42.152 +                                               uint64_t gtable)
  42.153 +{
  42.154 +    struct msixtbl_entry *entry;
  42.155 +    u32 len;
  42.156 +
  42.157 +    entry = xmalloc(struct msixtbl_entry);
  42.158 +    if ( !entry )
  42.159 +        return NULL;
  42.160 +
  42.161 +    memset(entry, 0, sizeof(struct msixtbl_entry));
  42.162 +        
  42.163 +    INIT_LIST_HEAD(&entry->list);
  42.164 +    INIT_RCU_HEAD(&entry->rcu);
  42.165 +    atomic_set(&entry->refcnt, 0);
  42.166 +
  42.167 +    len = pci_msix_get_table_len(pdev);
  42.168 +    entry->table_len = len;
  42.169 +    entry->pdev = pdev;
  42.170 +    entry->gtable = (unsigned long) gtable;
  42.171 +
  42.172 +    list_add_rcu(&entry->list, &d->arch.hvm_domain.msixtbl_list);
  42.173 +
  42.174 +    return entry;
  42.175 +}
  42.176 +
  42.177 +static void free_msixtbl_entry(struct rcu_head *rcu)
  42.178 +{
  42.179 +    struct msixtbl_entry *entry;
  42.180 +
  42.181 +    entry = container_of (rcu, struct msixtbl_entry, rcu);
  42.182 +
  42.183 +    xfree(entry);
  42.184 +}
  42.185 +
  42.186 +static void del_msixtbl_entry(struct msixtbl_entry *entry)
  42.187 +{
  42.188 +    list_del_rcu(&entry->list);
  42.189 +    call_rcu(&entry->rcu, free_msixtbl_entry);
  42.190 +}
  42.191 +
  42.192 +int msixtbl_pt_register(struct domain *d, int pirq, uint64_t gtable)
  42.193 +{
  42.194 +    irq_desc_t *irq_desc;
  42.195 +    struct msi_desc *msi_desc;
  42.196 +    struct pci_dev *pdev;
  42.197 +    struct msixtbl_entry *entry;
  42.198 +    int r = -EINVAL;
  42.199 +
  42.200 +    ASSERT(spin_is_locked(&pcidevs_lock));
  42.201 +
  42.202 +    irq_desc = domain_spin_lock_irq_desc(d, pirq, NULL);
  42.203 +
  42.204 +    if ( irq_desc->handler != &pci_msi_type )
  42.205 +        goto out;
  42.206 +
  42.207 +    msi_desc = irq_desc->msi_desc;
  42.208 +    if ( !msi_desc )
  42.209 +        goto out;
  42.210 +
  42.211 +    pdev = msi_desc->dev;
  42.212 +
  42.213 +    spin_lock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.214 +
  42.215 +    list_for_each_entry( entry, &d->arch.hvm_domain.msixtbl_list, list )
  42.216 +        if ( pdev == entry->pdev )
  42.217 +            goto found;
  42.218 +
  42.219 +    entry = add_msixtbl_entry(d, pdev, gtable);
  42.220 +    if ( !entry )
  42.221 +    {
  42.222 +        spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.223 +        goto out;
  42.224 +    }
  42.225 +
  42.226 +found:
  42.227 +    atomic_inc(&entry->refcnt);
  42.228 +    spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.229 +    r = 0;
  42.230 +
  42.231 +out:
  42.232 +    spin_unlock_irq(&irq_desc->lock);
  42.233 +    return r;
  42.234 +
  42.235 +}
  42.236 +
  42.237 +void msixtbl_pt_unregister(struct domain *d, int pirq)
  42.238 +{
  42.239 +    irq_desc_t *irq_desc;
  42.240 +    struct msi_desc *msi_desc;
  42.241 +    struct pci_dev *pdev;
  42.242 +    struct msixtbl_entry *entry;
  42.243 +
  42.244 +    ASSERT(spin_is_locked(&pcidevs_lock));
  42.245 +
  42.246 +    irq_desc = domain_spin_lock_irq_desc(d, pirq, NULL);
  42.247 +
  42.248 +    if ( irq_desc->handler != &pci_msi_type )
  42.249 +        goto out;
  42.250 +
  42.251 +    msi_desc = irq_desc->msi_desc;
  42.252 +    if ( !msi_desc )
  42.253 +        goto out;
  42.254 +
  42.255 +    pdev = msi_desc->dev;
  42.256 +
  42.257 +    spin_lock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.258 +
  42.259 +    list_for_each_entry( entry, &d->arch.hvm_domain.msixtbl_list, list )
  42.260 +        if ( pdev == entry->pdev )
  42.261 +            goto found;
  42.262 +
  42.263 +    spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.264 +
  42.265 +
  42.266 +out:
  42.267 +    spin_unlock(&irq_desc->lock);
  42.268 +    return;
  42.269 +
  42.270 +found:
  42.271 +    if ( !atomic_dec_and_test(&entry->refcnt) )
  42.272 +        del_msixtbl_entry(entry);
  42.273 +
  42.274 +    spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.275 +    spin_unlock(&irq_desc->lock);
  42.276 +}
  42.277 +
  42.278 +void msixtbl_pt_cleanup(struct domain *d, int pirq)
  42.279 +{
  42.280 +    struct msixtbl_entry *entry, *temp;
  42.281 +
  42.282 +    spin_lock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.283 +
  42.284 +    list_for_each_entry_safe( entry, temp,
  42.285 +                              &d->arch.hvm_domain.msixtbl_list, list )
  42.286 +        del_msixtbl_entry(entry);
  42.287 +
  42.288 +    spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock);
  42.289 +}

    43.1 --- a/xen/arch/x86/irq.c	Mon Mar 02 18:26:56 2009 +0900
    43.2 +++ b/xen/arch/x86/irq.c	Fri Mar 06 12:22:22 2009 +0900
    43.3 @@ -1104,7 +1104,8 @@ static void dump_irqs(unsigned char key)
    43.4                         (test_bit(d->pirq_to_evtchn[irq],
    43.5                                   &shared_info(d, evtchn_pending)) ?
    43.6                          'P' : '-'),
    43.7 -                       (test_bit(d->pirq_to_evtchn[irq]/BITS_PER_GUEST_LONG(d),
    43.8 +                       (test_bit(d->pirq_to_evtchn[irq] /
    43.9 +                                 BITS_PER_EVTCHN_WORD(d),
   43.10                                   &vcpu_info(d->vcpu[0], evtchn_pending_sel)) ?
   43.11                          'S' : '-'),
   43.12                         (test_bit(d->pirq_to_evtchn[irq],

    44.1 --- a/xen/arch/x86/mm.c	Mon Mar 02 18:26:56 2009 +0900
    44.2 +++ b/xen/arch/x86/mm.c	Fri Mar 06 12:22:22 2009 +0900
    44.3 @@ -279,15 +279,39 @@ void __init arch_init_memory(void)
    44.4      subarch_init_memory();
    44.5  }
    44.6  
    44.7 -int page_is_conventional_ram(unsigned long mfn)
    44.8 +int page_is_ram_type(unsigned long mfn, unsigned long mem_type)
    44.9  {
   44.10      uint64_t maddr = pfn_to_paddr(mfn);
   44.11      int i;
   44.12  
   44.13      for ( i = 0; i < e820.nr_map; i++ )
   44.14      {
   44.15 -        if ( (e820.map[i].type == E820_RAM) &&
   44.16 -             (e820.map[i].addr <= maddr) &&
   44.17 +        switch ( e820.map[i].type )
   44.18 +        {
   44.19 +        case E820_RAM:
   44.20 +            if ( mem_type & RAM_TYPE_CONVENTIONAL )
   44.21 +                break;
   44.22 +            continue;
   44.23 +        case E820_RESERVED:
   44.24 +            if ( mem_type & RAM_TYPE_RESERVED )
   44.25 +                break;
   44.26 +            continue;
   44.27 +        case E820_UNUSABLE:
   44.28 +            if ( mem_type & RAM_TYPE_UNUSABLE )
   44.29 +                break;
   44.30 +            continue;
   44.31 +        case E820_ACPI:
   44.32 +        case E820_NVS:
   44.33 +            if ( mem_type & RAM_TYPE_ACPI )
   44.34 +                break;
   44.35 +            continue;
   44.36 +        default:
   44.37 +            /* unknown */
   44.38 +            continue;
   44.39 +        }
   44.40 +        
   44.41 +        /* Test the range. */
   44.42 +        if ( (e820.map[i].addr <= maddr) &&
   44.43               ((e820.map[i].addr + e820.map[i].size) >= (maddr + PAGE_SIZE)) )
   44.44              return 1;
   44.45      }
   44.46 @@ -347,14 +371,14 @@ void share_xen_page_with_privileged_gues
   44.47  #else
   44.48  /*
   44.49   * In debug builds we shadow a selection of <4GB PDPTs to exercise code paths.
   44.50 - * We cannot safely shadow the idle page table, nor shadow (v1) page tables
   44.51 - * (detected by lack of an owning domain). As required for correctness, we
   44.52 + * We cannot safely shadow the idle page table, nor shadow page tables
   44.53 + * (detected by zero reference count). As required for correctness, we
   44.54   * always shadow PDPTs above 4GB.
   44.55   */
   44.56 -#define l3tab_needs_shadow(mfn)                         \
   44.57 -    (((((mfn) << PAGE_SHIFT) != __pa(idle_pg_table)) && \
   44.58 -      (page_get_owner(mfn_to_page(mfn)) != NULL) &&     \
   44.59 -      ((mfn) & 1)) || /* odd MFNs are shadowed */       \
   44.60 +#define l3tab_needs_shadow(mfn)                          \
   44.61 +    (((((mfn) << PAGE_SHIFT) != __pa(idle_pg_table)) &&  \
   44.62 +      (mfn_to_page(mfn)->count_info & PGC_count_mask) && \
   44.63 +      ((mfn) & 1)) || /* odd MFNs are shadowed */        \
   44.64       ((mfn) >= 0x100000))
   44.65  #endif
   44.66  
   44.67 @@ -666,7 +690,16 @@ get_##level##_linear_pagetable(         
   44.68  
   44.69  int is_iomem_page(unsigned long mfn)
   44.70  {
   44.71 -    return (!mfn_valid(mfn) || (page_get_owner(mfn_to_page(mfn)) == dom_io));
   44.72 +    struct page_info *page;
   44.73 +
   44.74 +    if ( !mfn_valid(mfn) )
   44.75 +        return 1;
   44.76 +
   44.77 +    /* Caller must know that it is an iomem page, or a reference is held. */
   44.78 +    page = mfn_to_page(mfn);
   44.79 +    ASSERT((page->count_info & PGC_count_mask) != 0);
   44.80 +
   44.81 +    return (page_get_owner(page) == dom_io);
   44.82  }
   44.83  
   44.84  
   44.85 @@ -679,7 +712,6 @@ get_page_from_l1e(
   44.86      uint32_t l1f = l1e_get_flags(l1e);
   44.87      struct vcpu *curr = current;
   44.88      struct domain *owner;
   44.89 -    int okay;
   44.90  
   44.91      if ( !(l1f & _PAGE_PRESENT) )
   44.92          return 1;
   44.93 @@ -690,8 +722,13 @@ get_page_from_l1e(
   44.94          return 0;
   44.95      }
   44.96  
   44.97 -    if ( is_iomem_page(mfn) )
   44.98 +    if ( !mfn_valid(mfn) ||
   44.99 +         (owner = page_get_owner_and_reference(page)) == dom_io )
  44.100      {
  44.101 +        /* Only needed the reference to confirm dom_io ownership. */
  44.102 +        if ( mfn_valid(mfn) )
  44.103 +            put_page(page);
  44.104 +
  44.105          /* DOMID_IO reverts to caller for privilege checks. */
  44.106          if ( d == dom_io )
  44.107              d = curr->domain;
  44.108 @@ -707,33 +744,29 @@ get_page_from_l1e(
  44.109          return 1;
  44.110      }
  44.111  
  44.112 +    if ( owner == NULL )
  44.113 +        goto could_not_pin;
  44.114 +
  44.115      /*
  44.116       * Let privileged domains transfer the right to map their target
  44.117       * domain's pages. This is used to allow stub-domain pvfb export to dom0,
  44.118       * until pvfb supports granted mappings. At that time this minor hack
  44.119       * can go away.
  44.120       */
  44.121 -    owner = page_get_owner(page);
  44.122 -    if ( unlikely(d != owner) && (owner != NULL) &&
  44.123 -         (d != curr->domain) && IS_PRIV_FOR(d, owner) )
  44.124 +    if ( unlikely(d != owner) && (d != curr->domain) && IS_PRIV_FOR(d, owner) )
  44.125          d = owner;
  44.126  
  44.127      /* Foreign mappings into guests in shadow external mode don't
  44.128       * contribute to writeable mapping refcounts.  (This allows the
  44.129       * qemu-dm helper process in dom0 to map the domain's memory without
  44.130       * messing up the count of "real" writable mappings.) */
  44.131 -    okay = get_data_page(
  44.132 -        page, d,
  44.133 -        (l1f & _PAGE_RW) && !(paging_mode_external(d) && (d != curr->domain)));
  44.134 -    if ( !okay )
  44.135 -    {
  44.136 -        MEM_LOG("Error getting mfn %lx (pfn %lx) from L1 entry %" PRIpte
  44.137 -                " for dom%d",
  44.138 -                mfn, get_gpfn_from_mfn(mfn),
  44.139 -                l1e_get_intpte(l1e), d->domain_id);
  44.140 -    }
  44.141 -    else if ( pte_flags_to_cacheattr(l1f) !=
  44.142 -              ((page->count_info >> PGC_cacheattr_base) & 7) )
  44.143 +    if ( (l1f & _PAGE_RW) &&
  44.144 +         !(paging_mode_external(d) && (d != curr->domain)) &&
  44.145 +         !get_page_type(page, PGT_writable_page) )
  44.146 +        goto could_not_pin;
  44.147 +
  44.148 +    if ( pte_flags_to_cacheattr(l1f) !=
  44.149 +         ((page->count_info >> PGC_cacheattr_base) & 7) )
  44.150      {
  44.151          unsigned long x, nx, y = page->count_info;
  44.152          unsigned long cacheattr = pte_flags_to_cacheattr(l1f);
  44.153 @@ -762,7 +795,16 @@ get_page_from_l1e(
  44.154  #endif
  44.155      }
  44.156  
  44.157 -    return okay;
  44.158 +    return 1;
  44.159 +
  44.160 + could_not_pin:
  44.161 +    MEM_LOG("Error getting mfn %lx (pfn %lx) from L1 entry %" PRIpte
  44.162 +            " for dom%d",
  44.163 +            mfn, get_gpfn_from_mfn(mfn),
  44.164 +            l1e_get_intpte(l1e), d->domain_id);
  44.165 +    if ( owner != NULL )
  44.166 +        put_page(page);
  44.167 +    return 0;
  44.168  }
  44.169  
  44.170  
  44.171 @@ -1150,7 +1192,7 @@ static int create_pae_xen_mappings(struc
  44.172      for ( i = 0; i < PDPT_L2_ENTRIES; i++ )
  44.173      {
  44.174          l2e = l2e_from_page(
  44.175 -            virt_to_page(page_get_owner(page)->arch.mm_perdomain_pt) + i,
  44.176 +            virt_to_page(d->arch.mm_perdomain_pt) + i,
  44.177              __PAGE_HYPERVISOR);
  44.178          l2e_write(&pl2e[l2_table_offset(PERDOMAIN_VIRT_START) + i], l2e);
  44.179      }
  44.180 @@ -1900,7 +1942,7 @@ void put_page(struct page_info *page)
  44.181  }
  44.182  
  44.183  
  44.184 -int get_page(struct page_info *page, struct domain *domain)
  44.185 +struct domain *page_get_owner_and_reference(struct page_info *page)
  44.186  {
  44.187      unsigned long x, y = page->count_info;
  44.188  
  44.189 @@ -1909,22 +1951,29 @@ int get_page(struct page_info *page, str
  44.190          if ( unlikely((x & PGC_count_mask) == 0) ||  /* Not allocated? */
  44.191               /* Keep one spare reference to be acquired by get_page_light(). */
  44.192               unlikely(((x + 2) & PGC_count_mask) <= 1) ) /* Overflow? */
  44.193 -            goto fail;
  44.194 +            return NULL;
  44.195      }
  44.196      while ( (y = cmpxchg(&page->count_info, x, x + 1)) != x );
  44.197  
  44.198 -    if ( likely(page_get_owner(page) == domain) )
  44.199 +    return page_get_owner(page);
  44.200 +}
  44.201 +
  44.202 +
  44.203 +int get_page(struct page_info *page, struct domain *domain)
  44.204 +{
  44.205 +    struct domain *owner = page_get_owner_and_reference(page);
  44.206 +
  44.207 +    if ( likely(owner == domain) )
  44.208          return 1;
  44.209  
  44.210      put_page(page);
  44.211  
  44.212 - fail:
  44.213      if ( !_shadow_mode_refcounts(domain) && !domain->is_dying )
  44.214          gdprintk(XENLOG_INFO,
  44.215                   "Error pfn %lx: rd=%p, od=%p, caf=%08lx, taf=%"
  44.216                   PRtype_info "\n",
  44.217 -                 page_to_mfn(page), domain, page_get_owner(page),
  44.218 -                 y, page->u.inuse.type_info);
  44.219 +                 page_to_mfn(page), domain, owner,
  44.220 +                 page->count_info, page->u.inuse.type_info);
  44.221      return 0;
  44.222  }
  44.223  
  44.224 @@ -1950,7 +1999,6 @@ static void get_page_light(struct page_i
  44.225      while ( unlikely(y != x) );
  44.226  }
  44.227  
  44.228 -
  44.229  static int alloc_page_type(struct page_info *page, unsigned long type,
  44.230                             int preemptible)
  44.231  {

    45.1 --- a/xen/arch/x86/msi.c	Mon Mar 02 18:26:56 2009 +0900
    45.2 +++ b/xen/arch/x86/msi.c	Fri Mar 06 12:22:22 2009 +0900
    45.3 @@ -839,3 +839,23 @@ int pci_restore_msi_state(struct pci_dev
    45.4      return 0;
    45.5  }
    45.6  
    45.7 +unsigned int pci_msix_get_table_len(struct pci_dev *pdev)
    45.8 +{
    45.9 +    int pos;
   45.10 +    u16 control;
   45.11 +    u8 bus, slot, func;
   45.12 +    unsigned int len;
   45.13 +
   45.14 +    bus = pdev->bus;
   45.15 +    slot = PCI_SLOT(pdev->devfn);
   45.16 +    func = PCI_FUNC(pdev->devfn);
   45.17 +
   45.18 +    pos = pci_find_cap_offset(bus, slot, func, PCI_CAP_ID_MSIX);
   45.19 +    if ( !pos )
   45.20 +        return 0;
   45.21 +
   45.22 +    control = pci_conf_read16(bus, slot, func, msix_control_reg(pos));
   45.23 +    len = msix_table_size(control) * PCI_MSIX_ENTRY_SIZE;
   45.24 +
   45.25 +    return len;
   45.26 +}

    46.1 --- a/xen/arch/x86/setup.c	Mon Mar 02 18:26:56 2009 +0900
    46.2 +++ b/xen/arch/x86/setup.c	Fri Mar 06 12:22:22 2009 +0900
    46.3 @@ -97,6 +97,7 @@ int early_boot = 1;
    46.4  cpumask_t cpu_present_map;
    46.5  
    46.6  unsigned long xen_phys_start;
    46.7 +unsigned long allocator_bitmap_end;
    46.8  
    46.9  #ifdef CONFIG_X86_32
   46.10  /* Limits of Xen heap, used to initialise the allocator. */
   46.11 @@ -418,7 +419,6 @@ void __init __start_xen(unsigned long mb
   46.12      multiboot_info_t *mbi = __va(mbi_p);
   46.13      module_t *mod = (module_t *)__va(mbi->mods_addr);
   46.14      unsigned long nr_pages, modules_length, modules_headroom;
   46.15 -    unsigned long allocator_bitmap_end;
   46.16      int i, e820_warn = 0, bytes = 0;
   46.17      struct ns16550_defaults ns16550 = {
   46.18          .data_bits = 8,
   46.19 @@ -985,9 +985,12 @@ void __init __start_xen(unsigned long mb
   46.20  
   46.21      if ( opt_watchdog ) 
   46.22          watchdog_enable();
   46.23 +    
   46.24 +    if ( !tboot_protect_mem_regions() )
   46.25 +        panic("Could not protect TXT memory regions\n");
   46.26  
   46.27      /* Create initial domain 0. */
   46.28 -    dom0 = domain_create(0, 0, DOM0_SSIDREF);
   46.29 +    dom0 = domain_create(0, DOMCRF_s3_integrity, DOM0_SSIDREF);
   46.30      if ( (dom0 == NULL) || (alloc_vcpu(dom0, 0, 0) == NULL) )
   46.31          panic("Error creating domain 0\n");
   46.32  
   46.33 @@ -1037,9 +1040,6 @@ void __init __start_xen(unsigned long mb
   46.34      if ( xen_cpuidle )
   46.35          xen_processor_pmbits |= XEN_PROCESSOR_PM_CX;
   46.36  
   46.37 -    if ( !tboot_protect_mem_regions() )
   46.38 -        panic("Could not protect TXT memory regions\n");
   46.39 -
   46.40      /*
   46.41       * We're going to setup domain0 using the module(s) that we stashed safely
   46.42       * above our heap. The second module, if present, is an initrd ramdisk.

    47.1 --- a/xen/arch/x86/tboot.c	Mon Mar 02 18:26:56 2009 +0900
    47.2 +++ b/xen/arch/x86/tboot.c	Fri Mar 06 12:22:22 2009 +0900
    47.3 @@ -3,11 +3,14 @@
    47.4  #include <xen/types.h>
    47.5  #include <xen/lib.h>
    47.6  #include <xen/sched.h>
    47.7 +#include <xen/domain_page.h>
    47.8 +#include <xen/iommu.h>
    47.9  #include <asm/fixmap.h>
   47.10  #include <asm/page.h>
   47.11  #include <asm/processor.h>
   47.12  #include <asm/e820.h>
   47.13  #include <asm/tboot.h>
   47.14 +#include <crypto/vmac.h>
   47.15  
   47.16  /* tboot=<physical address of shared page> */
   47.17  static char opt_tboot[20] = "";
   47.18 @@ -16,6 +19,10 @@ string_param("tboot", opt_tboot);
   47.19  /* Global pointer to shared data; NULL means no measured launch. */
   47.20  tboot_shared_t *g_tboot_shared;
   47.21  
   47.22 +static vmac_t domain_mac;     /* MAC for all domains during S3 */
   47.23 +static vmac_t xenheap_mac;    /* MAC for xen heap during S3 */
   47.24 +static vmac_t frametable_mac; /* MAC for frame table during S3 */
   47.25 +
   47.26  static const uuid_t tboot_shared_uuid = TBOOT_SHARED_UUID;
   47.27  
   47.28  /* used by tboot_protect_mem_regions() and/or tboot_parse_dmar_table() */
   47.29 @@ -40,6 +47,7 @@ static uint64_t sinit_base, sinit_size;
   47.30  #define TXTCR_HEAP_SIZE             0x0308
   47.31  
   47.32  extern char __init_begin[], __per_cpu_start[], __per_cpu_end[], __bss_start[];
   47.33 +extern unsigned long allocator_bitmap_end;
   47.34  
   47.35  #define SHA1_SIZE      20
   47.36  typedef uint8_t   sha1_hash_t[SHA1_SIZE];
   47.37 @@ -82,8 +90,9 @@ void __init tboot_probe(void)
   47.38      if ( memcmp(&tboot_shared_uuid, (uuid_t *)tboot_shared, sizeof(uuid_t)) )
   47.39          return;
   47.40  
   47.41 -    /* new tboot_shared (w/ GAS support) is not backwards compatible */
   47.42 -    if ( tboot_shared->version < 3 ) {
   47.43 +    /* new tboot_shared (w/ GAS support, integrity, etc.) is not backwards
   47.44 +       compatible */
   47.45 +    if ( tboot_shared->version < 4 ) {
   47.46          printk("unsupported version of tboot (%u)\n", tboot_shared->version);
   47.47          return;
   47.48      }
   47.49 @@ -121,6 +130,143 @@ void __init tboot_probe(void)
   47.50                           (unsigned long)__va((map_base + map_size) << PAGE_SHIFT));
   47.51  }
   47.52  
   47.53 +/* definitions from xen/drivers/passthrough/vtd/iommu.h
   47.54 + * used to walk through vtd page tables */
   47.55 +#define LEVEL_STRIDE (9)
   47.56 +#define PTE_NUM (1<<LEVEL_STRIDE)
   47.57 +#define dma_pte_present(p) (((p).val & 3) != 0)
   47.58 +#define dma_pte_addr(p) ((p).val & PAGE_MASK_4K)
   47.59 +#define agaw_to_level(val) ((val)+2)
   47.60 +struct dma_pte {
   47.61 +    u64 val;
   47.62 +};
   47.63 +
   47.64 +static void update_iommu_mac(vmac_ctx_t *ctx, uint64_t pt_maddr, int level)
   47.65 +{
   47.66 +    int i;
   47.67 +    struct dma_pte *pt_vaddr, *pte;
   47.68 +    int next_level = level - 1;
   47.69 +
   47.70 +    if ( pt_maddr == 0 )
   47.71 +        return;
   47.72 +
   47.73 +    pt_vaddr = (struct dma_pte *)map_domain_page(pt_maddr >> PAGE_SHIFT_4K);
   47.74 +    vmac_update((void *)pt_vaddr, PAGE_SIZE, ctx);
   47.75 +
   47.76 +    for ( i = 0; i < PTE_NUM; i++ )
   47.77 +    {
   47.78 +        pte = &pt_vaddr[i];
   47.79 +        if ( !dma_pte_present(*pte) )
   47.80 +            continue;
   47.81 +
   47.82 +        if ( next_level >= 1 )
   47.83 +            update_iommu_mac(ctx, dma_pte_addr(*pte), next_level);
   47.84 +    }
   47.85 +
   47.86 +    unmap_domain_page(pt_vaddr);
   47.87 +}
   47.88 +
   47.89 +#define is_page_in_use(page) \
   47.90 +    ((page->count_info & PGC_count_mask) != 0 || page->count_info == 0)
   47.91 +
   47.92 +static void update_pagetable_mac(vmac_ctx_t *ctx)
   47.93 +{
   47.94 +    unsigned long mfn;
   47.95 +
   47.96 +    for ( mfn = 0; mfn < max_page; mfn++ )
   47.97 +    {
   47.98 +        struct page_info *page = mfn_to_page(mfn);
   47.99 +        if ( is_page_in_use(page) && !is_xen_heap_page(page) ) {
  47.100 +            if ( page->count_info & PGC_page_table ) {
  47.101 +                void *pg = map_domain_page(mfn);
  47.102 +                vmac_update(pg, PAGE_SIZE, ctx);
  47.103 +                unmap_domain_page(pg);
  47.104 +            }
  47.105 +        }
  47.106 +    }
  47.107 +}
  47.108 + 
  47.109 +static void tboot_gen_domain_integrity(const uint8_t key[TB_KEY_SIZE],
  47.110 +                                       vmac_t *mac)
  47.111 +{
  47.112 +    struct domain *d;
  47.113 +    struct page_info *page;
  47.114 +    uint8_t nonce[16] = {};
  47.115 +    vmac_ctx_t ctx;
  47.116 +
  47.117 +    vmac_set_key((uint8_t *)key, &ctx);
  47.118 +    for_each_domain( d )
  47.119 +    {
  47.120 +        if ( !d->arch.s3_integrity )
  47.121 +            continue;
  47.122 +        printk("MACing Domain %u\n", d->domain_id);
  47.123 +
  47.124 +        page_list_for_each(page, &d->page_list)
  47.125 +        {
  47.126 +            void *pg;
  47.127 +            pg = map_domain_page(page_to_mfn(page));
  47.128 +            vmac_update(pg, PAGE_SIZE, &ctx);
  47.129 +            unmap_domain_page(pg);
  47.130 +        }
  47.131 +
  47.132 +        if ( !is_idle_domain(d) )
  47.133 +        {
  47.134 +            struct hvm_iommu *hd = domain_hvm_iommu(d);
  47.135 +            update_iommu_mac(&ctx, hd->pgd_maddr, agaw_to_level(hd->agaw));
  47.136 +        }
  47.137 +    }
  47.138 +
  47.139 +    /* MAC all shadow page tables */
  47.140 +    update_pagetable_mac(&ctx);
  47.141 +
  47.142 +    *mac = vmac(NULL, 0, nonce, NULL, &ctx);
  47.143 +
  47.144 +    printk("MAC for domains is: 0x%08"PRIx64"\n", *mac);
  47.145 +
  47.146 +    /* wipe ctx to ensure key is not left in memory */
  47.147 +    memset(&ctx, 0, sizeof(ctx));
  47.148 +}
  47.149 +
  47.150 +static void tboot_gen_xenheap_integrity(const uint8_t key[TB_KEY_SIZE],
  47.151 +                                        vmac_t *mac)
  47.152 +{
  47.153 +    unsigned long mfn;
  47.154 +    uint8_t nonce[16] = {};
  47.155 +    vmac_ctx_t ctx;
  47.156 +
  47.157 +    vmac_set_key((uint8_t *)key, &ctx);
  47.158 +    for ( mfn = 0; mfn < max_page; mfn++ )
  47.159 +    {
  47.160 +        struct page_info *page = __mfn_to_page(mfn);
  47.161 +        if ( is_page_in_use(page) && is_xen_heap_page(page) ) {
  47.162 +            void *pg = mfn_to_virt(mfn);
  47.163 +            vmac_update((uint8_t *)pg, PAGE_SIZE, &ctx);
  47.164 +        }
  47.165 +    }
  47.166 +    *mac = vmac(NULL, 0, nonce, NULL, &ctx);
  47.167 +
  47.168 +    printk("MAC for xenheap is: 0x%08"PRIx64"\n", *mac);
  47.169 +
  47.170 +    /* wipe ctx to ensure key is not left in memory */
  47.171 +    memset(&ctx, 0, sizeof(ctx));
  47.172 +}
  47.173 +
  47.174 +static void tboot_gen_frametable_integrity(const uint8_t key[TB_KEY_SIZE],
  47.175 +                                           vmac_t *mac)
  47.176 +{
  47.177 +    uint8_t nonce[16] = {};
  47.178 +    vmac_ctx_t ctx;
  47.179 +
  47.180 +    vmac_set_key((uint8_t *)key, &ctx);
  47.181 +    *mac = vmac((uint8_t *)frame_table,
  47.182 +                PFN_UP(max_page * sizeof(*frame_table)), nonce, NULL, &ctx);
  47.183 +
  47.184 +    printk("MAC for frametable is: 0x%08"PRIx64"\n", *mac);
  47.185 +
  47.186 +    /* wipe ctx to ensure key is not left in memory */
  47.187 +    memset(&ctx, 0, sizeof(ctx));
  47.188 +}
  47.189 +
  47.190  void tboot_shutdown(uint32_t shutdown_type)
  47.191  {
  47.192      uint32_t map_base, map_size;
  47.193 @@ -130,38 +276,60 @@ void tboot_shutdown(uint32_t shutdown_ty
  47.194  
  47.195      local_irq_disable();
  47.196  
  47.197 -    /* if this is S3 then set regions to MAC */
  47.198 -    if ( shutdown_type == TB_SHUTDOWN_S3 ) {
  47.199 -        g_tboot_shared->num_mac_regions = 4;
  47.200 -        /* S3 resume code (and other real mode trampoline code) */
  47.201 -        g_tboot_shared->mac_regions[0].start =
  47.202 -            (uint64_t)bootsym_phys(trampoline_start);
  47.203 -        g_tboot_shared->mac_regions[0].end =
  47.204 -            (uint64_t)bootsym_phys(trampoline_end);
  47.205 -        /* hypervisor code + data */
  47.206 -        g_tboot_shared->mac_regions[1].start = (uint64_t)__pa(&_stext);
  47.207 -        g_tboot_shared->mac_regions[1].end = (uint64_t)__pa(&__init_begin);
  47.208 -        /* per-cpu data */
  47.209 -        g_tboot_shared->mac_regions[2].start = (uint64_t)__pa(&__per_cpu_start);
  47.210 -        g_tboot_shared->mac_regions[2].end = (uint64_t)__pa(&__per_cpu_end);
  47.211 -        /* bss */
  47.212 -        g_tboot_shared->mac_regions[3].start = (uint64_t)__pa(&__bss_start);
  47.213 -        g_tboot_shared->mac_regions[3].end = (uint64_t)__pa(&_end);
  47.214 -    }
  47.215 +    /* we may be called from an interrupt context, so to prevent */
  47.216 +    /* 'ASSERT(!in_irq());' in alloc_domheap_pages(), decrease count */
  47.217 +    while ( in_irq() )
  47.218 +        irq_exit();
  47.219  
  47.220      /* Create identity map for tboot shutdown code. */
  47.221 +    /* do before S3 integrity because mapping tboot may change xenheap */
  47.222      map_base = PFN_DOWN(g_tboot_shared->tboot_base);
  47.223      map_size = PFN_UP(g_tboot_shared->tboot_size);
  47.224  
  47.225      err = map_pages_to_xen(map_base << PAGE_SHIFT, map_base, map_size,
  47.226                             __PAGE_HYPERVISOR);
  47.227 -    if ( err != 0 )
  47.228 -    {
  47.229 +    if ( err != 0 ) {
  47.230          printk("error (0x%x) mapping tboot pages (mfns) @ 0x%x, 0x%x\n", err,
  47.231                 map_base, map_size);
  47.232          return;
  47.233      }
  47.234  
  47.235 +    /* if this is S3 then set regions to MAC */
  47.236 +    if ( shutdown_type == TB_SHUTDOWN_S3 ) {
  47.237 +        /*
  47.238 +         * Xen regions for tboot to MAC
  47.239 +         */
  47.240 +        g_tboot_shared->num_mac_regions = 5;
  47.241 +        /* S3 resume code (and other real mode trampoline code) */
  47.242 +        g_tboot_shared->mac_regions[0].start = bootsym_phys(trampoline_start);
  47.243 +        g_tboot_shared->mac_regions[0].size = bootsym_phys(trampoline_end) -
  47.244 +                                              bootsym_phys(trampoline_start);
  47.245 +        /* hypervisor code + data */
  47.246 +        g_tboot_shared->mac_regions[1].start = (uint64_t)__pa(&_stext);
  47.247 +        g_tboot_shared->mac_regions[1].size = __pa(&__init_begin) -
  47.248 +                                              __pa(&_stext);
  47.249 +        /* per-cpu data */
  47.250 +        g_tboot_shared->mac_regions[2].start = (uint64_t)__pa(&__per_cpu_start);
  47.251 +        g_tboot_shared->mac_regions[2].size = __pa(&__per_cpu_end) -
  47.252 +                                              __pa(&__per_cpu_start);
  47.253 +        /* bss */
  47.254 +        g_tboot_shared->mac_regions[3].start = (uint64_t)__pa(&__bss_start);
  47.255 +        g_tboot_shared->mac_regions[3].size = __pa(&_end) - __pa(&__bss_start);
  47.256 +        /* boot allocator bitmap */
  47.257 +        g_tboot_shared->mac_regions[4].start = (uint64_t)__pa(&_end);
  47.258 +        g_tboot_shared->mac_regions[4].size = allocator_bitmap_end -
  47.259 +                                              __pa(&_end);
  47.260 +
  47.261 +        /*
  47.262 +         * MAC domains and other Xen memory
  47.263 +         */
  47.264 +        /* Xen has no better entropy source for MAC key than tboot's */
  47.265 +        /* MAC domains first in case it perturbs xenheap */
  47.266 +        tboot_gen_domain_integrity(g_tboot_shared->s3_key, &domain_mac);
  47.267 +        tboot_gen_frametable_integrity(g_tboot_shared->s3_key, &frametable_mac);
  47.268 +        tboot_gen_xenheap_integrity(g_tboot_shared->s3_key, &xenheap_mac);
  47.269 +    }
  47.270 +
  47.271      write_ptbase(idle_vcpu[0]);
  47.272  
  47.273      ((void(*)(void))(unsigned long)g_tboot_shared->shutdown_entry)();
  47.274 @@ -264,6 +432,29 @@ int __init tboot_parse_dmar_table(acpi_t
  47.275      return rc;
  47.276  }
  47.277  
  47.278 +int tboot_s3_resume(void)
  47.279 +{
  47.280 +    vmac_t mac;
  47.281 +
  47.282 +    if ( !tboot_in_measured_env() )
  47.283 +        return 0;
  47.284 +
  47.285 +    /* need to do these in reverse order of shutdown */
  47.286 +    tboot_gen_xenheap_integrity(g_tboot_shared->s3_key, &mac);
  47.287 +    if ( mac != xenheap_mac )
  47.288 +        return -1;
  47.289 +
  47.290 +    tboot_gen_frametable_integrity(g_tboot_shared->s3_key, &mac);
  47.291 +    if ( mac != frametable_mac )
  47.292 +        return -2;
  47.293 +
  47.294 +    tboot_gen_domain_integrity(g_tboot_shared->s3_key, &mac);
  47.295 +    if ( mac != domain_mac )
  47.296 +        return 0; /* -3 */
  47.297 +
  47.298 +    return 0;
  47.299 +}
  47.300 +
  47.301  /*
  47.302   * Local variables:
  47.303   * mode: C

    48.1 --- a/xen/arch/x86/time.c	Mon Mar 02 18:26:56 2009 +0900
    48.2 +++ b/xen/arch/x86/time.c	Fri Mar 06 12:22:22 2009 +0900
    48.3 @@ -1087,38 +1087,69 @@ static void local_time_calibration(void)
    48.4   */
    48.5  struct calibration_rendezvous {
    48.6      cpumask_t cpu_calibration_map;
    48.7 -    atomic_t nr_cpus;
    48.8 +    atomic_t count_start;
    48.9 +    atomic_t count_end;
   48.10      s_time_t master_stime;
   48.11      u64 master_tsc_stamp;
   48.12  };
   48.13  
   48.14 +#define NR_LOOPS 5
   48.15 +
   48.16  static void time_calibration_rendezvous(void *_r)
   48.17  {
   48.18 +    int i;
   48.19      struct cpu_calibration *c = &this_cpu(cpu_calibration);
   48.20      struct calibration_rendezvous *r = _r;
   48.21      unsigned int total_cpus = cpus_weight(r->cpu_calibration_map);
   48.22  
   48.23 -    if ( smp_processor_id() == 0 )
   48.24 +    /* 
   48.25 +     * Loop is used here to get rid of the cache's side effect to enlarge
   48.26 +     * the TSC difference among CPUs.
   48.27 +     */
   48.28 +    for ( i = 0; i < NR_LOOPS; i++ )
   48.29      {
   48.30 -        while ( atomic_read(&r->nr_cpus) != (total_cpus - 1) )
   48.31 -            cpu_relax();
   48.32 -        r->master_stime = read_platform_stime();
   48.33 -        rdtscll(r->master_tsc_stamp);
   48.34 -        mb(); /* write r->master_* /then/ signal */
   48.35 -        atomic_inc(&r->nr_cpus);
   48.36 -        c->local_tsc_stamp = r->master_tsc_stamp;
   48.37 -    }
   48.38 -    else
   48.39 -    {
   48.40 -        atomic_inc(&r->nr_cpus);
   48.41 -        while ( atomic_read(&r->nr_cpus) != total_cpus )
   48.42 -            cpu_relax();
   48.43 -        mb(); /* receive signal /then/ read r->master_* */
   48.44 -        if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) )
   48.45 -            wrmsrl(MSR_IA32_TSC, r->master_tsc_stamp);
   48.46 -        rdtscll(c->local_tsc_stamp);
   48.47 +        if ( smp_processor_id() == 0 )
   48.48 +        {
   48.49 +            while ( atomic_read(&r->count_start) != (total_cpus - 1) )
   48.50 +                mb();
   48.51 +   
   48.52 +            if ( r->master_stime == 0 )
   48.53 +            {
   48.54 +                r->master_stime = read_platform_stime();
   48.55 +                if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) )
   48.56 +                    rdtscll(r->master_tsc_stamp);
   48.57 +            }
   48.58 +            atomic_set(&r->count_end, 0);
   48.59 +            wmb();
   48.60 +            atomic_inc(&r->count_start);
   48.61 +    
   48.62 +            if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && 
   48.63 +                 i == NR_LOOPS - 1 )
   48.64 +                write_tsc((u32)r->master_tsc_stamp, (u32)(r->master_tsc_stamp >> 32));
   48.65 +    
   48.66 +            while (atomic_read(&r->count_end) != total_cpus - 1)
   48.67 +                mb();
   48.68 +            atomic_set(&r->count_start, 0);
   48.69 +            wmb();
   48.70 +            atomic_inc(&r->count_end);
   48.71 +        }
   48.72 +        else
   48.73 +        {
   48.74 +            atomic_inc(&r->count_start);
   48.75 +            while ( atomic_read(&r->count_start) != total_cpus )
   48.76 +                mb();
   48.77 +    
   48.78 +            if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && 
   48.79 +                 i == NR_LOOPS - 1 )
   48.80 +                write_tsc((u32)r->master_tsc_stamp, (u32)(r->master_tsc_stamp >> 32));
   48.81 +    
   48.82 +            atomic_inc(&r->count_end);
   48.83 +            while (atomic_read(&r->count_end) != total_cpus)
   48.84 +                mb();
   48.85 +        }
   48.86      }
   48.87  
   48.88 +    rdtscll(c->local_tsc_stamp);
   48.89      c->stime_local_stamp = get_s_time();
   48.90      c->stime_master_stamp = r->master_stime;
   48.91  
   48.92 @@ -1129,7 +1160,9 @@ static void time_calibration(void *unuse
   48.93  {
   48.94      struct calibration_rendezvous r = {
   48.95          .cpu_calibration_map = cpu_online_map,
   48.96 -        .nr_cpus = ATOMIC_INIT(0)
   48.97 +        .count_start = ATOMIC_INIT(0),
   48.98 +        .count_end = ATOMIC_INIT(0),
   48.99 +        .master_stime = 0
  48.100      };
  48.101  
  48.102      /* @wait=1 because we must wait for all cpus before freeing @r. */

    49.1 --- a/xen/arch/x86/traps.c	Mon Mar 02 18:26:56 2009 +0900
    49.2 +++ b/xen/arch/x86/traps.c	Fri Mar 06 12:22:22 2009 +0900
    49.3 @@ -2187,10 +2187,17 @@ static int emulate_privileged_op(struct 
    49.4          case MSR_IA32_MPERF:
    49.5          case MSR_IA32_APERF:
    49.6          case MSR_IA32_PERF_CTL:
    49.7 +            if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL )
    49.8 +                goto fail;
    49.9 +            if ( !is_cpufreq_controller(v->domain) )
   49.10 +                break;
   49.11 +            if ( wrmsr_safe(regs->ecx, eax, edx) != 0 )
   49.12 +                goto fail;
   49.13 +            break;
   49.14          case MSR_IA32_THERM_CONTROL:
   49.15              if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL )
   49.16                  goto fail;
   49.17 -            if ( !is_cpufreq_controller(v->domain) )
   49.18 +            if ( (v->domain->domain_id != 0) || !v->domain->is_pinned )
   49.19                  break;
   49.20              if ( wrmsr_safe(regs->ecx, eax, edx) != 0 )
   49.21                  goto fail;
   49.22 @@ -3095,7 +3102,8 @@ long register_guest_nmi_callback(unsigne
   49.23  
   49.24      t->vector  = TRAP_nmi;
   49.25      t->flags   = 0;
   49.26 -    t->cs      = !IS_COMPAT(d) ? FLAT_KERNEL_CS : FLAT_COMPAT_KERNEL_CS;
   49.27 +    t->cs      = (is_pv_32on64_domain(d) ?
   49.28 +                  FLAT_COMPAT_KERNEL_CS : FLAT_KERNEL_CS);
   49.29      t->address = address;
   49.30      TI_SET_IF(t, 1);
   49.31  

    50.1 --- a/xen/common/domain.c	Mon Mar 02 18:26:56 2009 +0900
    50.2 +++ b/xen/common/domain.c	Fri Mar 06 12:22:22 2009 +0900
    50.3 @@ -465,7 +465,9 @@ void domain_shutdown(struct domain *d, u
    50.4  
    50.5      for_each_vcpu ( d, v )
    50.6      {
    50.7 -        if ( v->defer_shutdown )
    50.8 +        if ( reason == SHUTDOWN_crash )
    50.9 +            v->defer_shutdown = 0;
   50.10 +        else if ( v->defer_shutdown )
   50.11              continue;
   50.12          vcpu_pause_nosync(v);
   50.13          v->paused_for_shutdown = 1;

    51.1 --- a/xen/common/domctl.c	Mon Mar 02 18:26:56 2009 +0900
    51.2 +++ b/xen/common/domctl.c	Fri Mar 06 12:22:22 2009 +0900
    51.3 @@ -242,13 +242,15 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc
    51.4          if ( (c.nat = xmalloc(struct vcpu_guest_context)) == NULL )
    51.5              goto svc_out;
    51.6  
    51.7 -        if ( !IS_COMPAT(v->domain) )
    51.8 +#ifdef CONFIG_COMPAT
    51.9 +        if ( !is_pv_32on64_vcpu(v) )
   51.10              ret = copy_from_guest(c.nat, op->u.vcpucontext.ctxt, 1);
   51.11 -#ifdef CONFIG_COMPAT
   51.12          else
   51.13              ret = copy_from_guest(c.cmp,
   51.14                                    guest_handle_cast(op->u.vcpucontext.ctxt,
   51.15                                                      void), 1);
   51.16 +#else
   51.17 +        ret = copy_from_guest(c.nat, op->u.vcpucontext.ctxt, 1);
   51.18  #endif
   51.19          ret = ret ? -EFAULT : 0;
   51.20  
   51.21 @@ -339,7 +341,8 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc
   51.22          ret = -EINVAL;
   51.23          if ( supervisor_mode_kernel ||
   51.24               (op->u.createdomain.flags &
   51.25 -             ~(XEN_DOMCTL_CDF_hvm_guest | XEN_DOMCTL_CDF_hap)) )
   51.26 +             ~(XEN_DOMCTL_CDF_hvm_guest | XEN_DOMCTL_CDF_hap |
   51.27 +               XEN_DOMCTL_CDF_s3_integrity)) )
   51.28              break;
   51.29  
   51.30          dom = op->domain;
   51.31 @@ -371,6 +374,8 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc
   51.32              domcr_flags |= DOMCRF_hvm;
   51.33          if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_hap )
   51.34              domcr_flags |= DOMCRF_hap;
   51.35 +        if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_s3_integrity )
   51.36 +            domcr_flags |= DOMCRF_s3_integrity;
   51.37  
   51.38          ret = -ENOMEM;
   51.39          d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref);
   51.40 @@ -593,12 +598,14 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc
   51.41          if ( v != current )
   51.42              vcpu_unpause(v);
   51.43  
   51.44 -        if ( !IS_COMPAT(v->domain) )
   51.45 +#ifdef CONFIG_COMPAT
   51.46 +        if ( !is_pv_32on64_vcpu(v) )
   51.47              ret = copy_to_guest(op->u.vcpucontext.ctxt, c.nat, 1);
   51.48 -#ifdef CONFIG_COMPAT
   51.49          else
   51.50              ret = copy_to_guest(guest_handle_cast(op->u.vcpucontext.ctxt,
   51.51                                                    void), c.cmp, 1);
   51.52 +#else
   51.53 +        ret = copy_to_guest(op->u.vcpucontext.ctxt, c.nat, 1);
   51.54  #endif
   51.55  
   51.56          if ( copy_to_guest(u_domctl, op, 1) || ret )

    52.1 --- a/xen/common/event_channel.c	Mon Mar 02 18:26:56 2009 +0900
    52.2 +++ b/xen/common/event_channel.c	Fri Mar 06 12:22:22 2009 +0900
    52.3 @@ -548,7 +548,7 @@ static int evtchn_set_pending(struct vcp
    52.4          return 1;
    52.5  
    52.6      if ( !test_bit        (port, &shared_info(d, evtchn_mask)) &&
    52.7 -         !test_and_set_bit(port / BITS_PER_GUEST_LONG(d),
    52.8 +         !test_and_set_bit(port / BITS_PER_EVTCHN_WORD(d),
    52.9                             &vcpu_info(v, evtchn_pending_sel)) )
   52.10      {
   52.11          vcpu_mark_events_pending(v);
   52.12 @@ -783,7 +783,7 @@ int evtchn_unmask(unsigned int port)
   52.13       */
   52.14      if ( test_and_clear_bit(port, &shared_info(d, evtchn_mask)) &&
   52.15           test_bit          (port, &shared_info(d, evtchn_pending)) &&
   52.16 -         !test_and_set_bit (port / BITS_PER_GUEST_LONG(d),
   52.17 +         !test_and_set_bit (port / BITS_PER_EVTCHN_WORD(d),
   52.18                              &vcpu_info(v, evtchn_pending_sel)) )
   52.19      {
   52.20          vcpu_mark_events_pending(v);

    53.1 --- a/xen/common/grant_table.c	Mon Mar 02 18:26:56 2009 +0900
    53.2 +++ b/xen/common/grant_table.c	Fri Mar 06 12:22:22 2009 +0900
    53.3 @@ -195,7 +195,7 @@ static void
    53.4  __gnttab_map_grant_ref(
    53.5      struct gnttab_map_grant_ref *op)
    53.6  {
    53.7 -    struct domain *ld, *rd;
    53.8 +    struct domain *ld, *rd, *owner;
    53.9      struct vcpu   *led;
   53.10      int            handle;
   53.11      unsigned long  frame = 0, nr_gets = 0;
   53.12 @@ -336,8 +336,13 @@ static void
   53.13  
   53.14      spin_unlock(&rd->grant_table->lock);
   53.15  
   53.16 -    if ( is_iomem_page(frame) )
   53.17 +    if ( !mfn_valid(frame) ||
   53.18 +         (owner = page_get_owner_and_reference(mfn_to_page(frame))) == dom_io )
   53.19      {
   53.20 +        /* Only needed the reference to confirm dom_io ownership. */
   53.21 +        if ( mfn_valid(frame) )
   53.22 +            put_page(mfn_to_page(frame));
   53.23 +
   53.24          if ( !iomem_access_permitted(rd, frame, frame) )
   53.25          {
   53.26              gdprintk(XENLOG_WARNING,
   53.27 @@ -352,20 +357,11 @@ static void
   53.28          if ( rc != GNTST_okay )
   53.29              goto undo_out;
   53.30      }
   53.31 -    else
   53.32 +    else if ( owner == rd )
   53.33      {
   53.34 -        if ( unlikely(!mfn_valid(frame)) ||
   53.35 -             unlikely(!(gnttab_host_mapping_get_page_type(op, ld, rd) ?
   53.36 -                        get_page_and_type(mfn_to_page(frame), rd,
   53.37 -                                          PGT_writable_page) :
   53.38 -                        get_page(mfn_to_page(frame), rd))) )
   53.39 -        {
   53.40 -            if ( !rd->is_dying )
   53.41 -                gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n",
   53.42 -                         frame);
   53.43 -            rc = GNTST_general_error;
   53.44 -            goto undo_out;
   53.45 -        }
   53.46 +        if ( gnttab_host_mapping_get_page_type(op, ld, rd) &&
   53.47 +             !get_page_type(mfn_to_page(frame), PGT_writable_page) )
   53.48 +            goto could_not_pin;
   53.49  
   53.50          nr_gets++;
   53.51          if ( op->flags & GNTMAP_host_map )
   53.52 @@ -383,6 +379,17 @@ static void
   53.53              }
   53.54          }
   53.55      }
   53.56 +    else
   53.57 +    {
   53.58 +    could_not_pin:
   53.59 +        if ( !rd->is_dying )
   53.60 +            gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n",
   53.61 +                     frame);
   53.62 +        if ( owner != NULL )
   53.63 +            put_page(mfn_to_page(frame));
   53.64 +        rc = GNTST_general_error;
   53.65 +        goto undo_out;
   53.66 +    }
   53.67  
   53.68      if ( need_iommu(ld) &&
   53.69           !(old_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) &&

    54.1 --- a/xen/common/keyhandler.c	Mon Mar 02 18:26:56 2009 +0900
    54.2 +++ b/xen/common/keyhandler.c	Fri Mar 06 12:22:22 2009 +0900
    54.3 @@ -225,7 +225,7 @@ static void dump_domains(unsigned char k
    54.4                     test_bit(v->virq_to_evtchn[VIRQ_DEBUG], 
    54.5                              &shared_info(d, evtchn_mask)),
    54.6                     test_bit(v->virq_to_evtchn[VIRQ_DEBUG] /
    54.7 -                            BITS_PER_GUEST_LONG(d),
    54.8 +                            BITS_PER_EVTCHN_WORD(d),
    54.9                              &vcpu_info(v, evtchn_pending_sel)));
   54.10              send_guest_vcpu_virq(v, VIRQ_DEBUG);
   54.11          }

    55.1 --- a/xen/common/trace.c	Mon Mar 02 18:26:56 2009 +0900
    55.2 +++ b/xen/common/trace.c	Fri Mar 06 12:22:22 2009 +0900
    55.3 @@ -37,10 +37,8 @@
    55.4  #define xen_t_buf t_buf
    55.5  CHECK_t_buf;
    55.6  #undef xen_t_buf
    55.7 -#define TB_COMPAT IS_COMPAT(dom0)
    55.8  #else
    55.9  #define compat_t_rec t_rec
   55.10 -#define TB_COMPAT 0
   55.11  #endif
   55.12  
   55.13  /* opt_tbuf_size: trace buffer size (in pages) */

    56.1 --- a/xen/common/xencomm.c	Mon Mar 02 18:26:56 2009 +0900
    56.2 +++ b/xen/common/xencomm.c	Fri Mar 06 12:22:22 2009 +0900
    56.3 @@ -51,24 +51,16 @@ xencomm_get_page(unsigned long paddr, st
    56.4          return -EFAULT;
    56.5          
    56.6      *page = maddr_to_page(maddr);
    56.7 -    if ( get_page(*page, current->domain) == 0 )
    56.8 +    if ( !get_page(*page, current->domain) )
    56.9      {
   56.10 -        if ( page_get_owner(*page) != current->domain )
   56.11 -        {
   56.12 -            /*
   56.13 -             * This page might be a page granted by another domain, or
   56.14 -             * this page is freed with decrease reservation hypercall at
   56.15 -             * the same time.
   56.16 -             */
   56.17 -            gdprintk(XENLOG_WARNING,
   56.18 -                     "bad page is passed. paddr 0x%lx maddr 0x%lx\n",
   56.19 -                     paddr, maddr);
   56.20 -            return -EFAULT;
   56.21 -        }
   56.22 -
   56.23 -        /* Try again. */
   56.24 -        cpu_relax();
   56.25 -        return -EAGAIN;
   56.26 +        /*
   56.27 +         * This page might be a page granted by another domain, or this page 
   56.28 +         * is freed with decrease reservation hypercall at the same time.
   56.29 +         */
   56.30 +        gdprintk(XENLOG_WARNING,
   56.31 +                 "bad page is passed. paddr 0x%lx maddr 0x%lx\n",
   56.32 +                 paddr, maddr);
   56.33 +        return -EFAULT;
   56.34      }
   56.35  
   56.36      return 0;

    57.1 --- a/xen/common/xenoprof.c	Mon Mar 02 18:26:56 2009 +0900
    57.2 +++ b/xen/common/xenoprof.c	Fri Mar 06 12:22:22 2009 +0900
    57.3 @@ -208,7 +208,7 @@ static int alloc_xenoprof_struct(
    57.4      bufsize = sizeof(struct xenoprof_buf);
    57.5      i = sizeof(struct event_log);
    57.6  #ifdef CONFIG_COMPAT
    57.7 -    d->xenoprof->is_compat = IS_COMPAT(is_passive ? dom0 : d);
    57.8 +    d->xenoprof->is_compat = is_pv_32on64_domain(is_passive ? dom0 : d);
    57.9      if ( XENOPROF_COMPAT(d->xenoprof) )
   57.10      {
   57.11          bufsize = sizeof(struct compat_oprof_buf);

    58.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    58.2 +++ b/xen/crypto/Makefile	Fri Mar 06 12:22:22 2009 +0900
    58.3 @@ -0,0 +1,2 @@
    58.4 +obj-y += rijndael.o
    58.5 +obj-y += vmac.o

    59.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    59.2 +++ b/xen/crypto/rijndael.c	Fri Mar 06 12:22:22 2009 +0900
    59.3 @@ -0,0 +1,1269 @@
    59.4 +/*	$OpenBSD: rijndael.c,v 1.19 2008/06/09 07:49:45 djm Exp $ */
    59.5 +
    59.6 +/**
    59.7 + * rijndael-alg-fst.c
    59.8 + *
    59.9 + * @version 3.0 (December 2000)
   59.10 + *
   59.11 + * Optimised ANSI C code for the Rijndael cipher (now AES)
   59.12 + *
   59.13 + * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
   59.14 + * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
   59.15 + * @author Paulo Barreto <paulo.barreto@terra.com.br>
   59.16 + *
   59.17 + * This code is hereby placed in the public domain.
   59.18 + *
   59.19 + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
   59.20 + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   59.21 + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   59.22 + * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
   59.23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   59.24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   59.25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
   59.26 + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
   59.27 + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
   59.28 + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
   59.29 + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   59.30 + */
   59.31 +
   59.32 +/* start for Xen */
   59.33 +#include <xen/config.h>
   59.34 +#include <xen/init.h>
   59.35 +#include <xen/types.h>
   59.36 +#include <xen/lib.h>
   59.37 +#include <crypto/rijndael.h>
   59.38 +/* end for Xen */
   59.39 +
   59.40 +#undef FULL_UNROLL
   59.41 +
   59.42 +/*
   59.43 +Te0[x] = S [x].[02, 01, 01, 03];
   59.44 +Te1[x] = S [x].[03, 02, 01, 01];
   59.45 +Te2[x] = S [x].[01, 03, 02, 01];
   59.46 +Te3[x] = S [x].[01, 01, 03, 02];
   59.47 +Te4[x] = S [x].[01, 01, 01, 01];
   59.48 +
   59.49 +Td0[x] = Si[x].[0e, 09, 0d, 0b];
   59.50 +Td1[x] = Si[x].[0b, 0e, 09, 0d];
   59.51 +Td2[x] = Si[x].[0d, 0b, 0e, 09];
   59.52 +Td3[x] = Si[x].[09, 0d, 0b, 0e];
   59.53 +Td4[x] = Si[x].[01, 01, 01, 01];
   59.54 +*/
   59.55 +
   59.56 +static const u32 Te0[256] = {
   59.57 +    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
   59.58 +    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
   59.59 +    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
   59.60 +    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
   59.61 +    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
   59.62 +    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
   59.63 +    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
   59.64 +    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
   59.65 +    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
   59.66 +    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
   59.67 +    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
   59.68 +    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
   59.69 +    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
   59.70 +    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
   59.71 +    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
   59.72 +    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
   59.73 +    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
   59.74 +    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
   59.75 +    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
   59.76 +    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
   59.77 +    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
   59.78 +    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
   59.79 +    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
   59.80 +    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
   59.81 +    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
   59.82 +    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
   59.83 +    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
   59.84 +    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
   59.85 +    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
   59.86 +    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
   59.87 +    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
   59.88 +    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
   59.89 +    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
   59.90 +    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
   59.91 +    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
   59.92 +    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
   59.93 +    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
   59.94 +    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
   59.95 +    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
   59.96 +    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
   59.97 +    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
   59.98 +    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
   59.99 +    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
  59.100 +    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
  59.101 +    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
  59.102 +    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
  59.103 +    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
  59.104 +    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
  59.105 +    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
  59.106 +    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
  59.107 +    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
  59.108 +    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
  59.109 +    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
  59.110 +    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
  59.111 +    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
  59.112 +    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
  59.113 +    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
  59.114 +    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
  59.115 +    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
  59.116 +    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
  59.117 +    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
  59.118 +    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
  59.119 +    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
  59.120 +    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
  59.121 +};
  59.122 +static const u32 Te1[256] = {
  59.123 +    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
  59.124 +    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
  59.125 +    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
  59.126 +    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
  59.127 +    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
  59.128 +    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
  59.129 +    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
  59.130 +    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
  59.131 +    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
  59.132 +    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
  59.133 +    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
  59.134 +    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
  59.135 +    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
  59.136 +    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
  59.137 +    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
  59.138 +    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
  59.139 +    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
  59.140 +    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
  59.141 +    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
  59.142 +    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
  59.143 +    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
  59.144 +    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
  59.145 +    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
  59.146 +    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
  59.147 +    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
  59.148 +    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
  59.149 +    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
  59.150 +    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
  59.151 +    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
  59.152 +    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
  59.153 +    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
  59.154 +    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
  59.155 +    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
  59.156 +    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
  59.157 +    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
  59.158 +    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
  59.159 +    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
  59.160 +    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
  59.161 +    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
  59.162 +    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
  59.163 +    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
  59.164 +    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
  59.165 +    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
  59.166 +    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
  59.167 +    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
  59.168 +    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
  59.169 +    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
  59.170 +    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
  59.171 +    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
  59.172 +    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
  59.173 +    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
  59.174 +    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
  59.175 +    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
  59.176 +    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
  59.177 +    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
  59.178 +    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
  59.179 +    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
  59.180 +    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
  59.181 +    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
  59.182 +    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
  59.183 +    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
  59.184 +    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
  59.185 +    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
  59.186 +    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
  59.187 +};
  59.188 +static const u32 Te2[256] = {
  59.189 +    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
  59.190 +    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
  59.191 +    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
  59.192 +    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
  59.193 +    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
  59.194 +    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
  59.195 +    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
  59.196 +    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
  59.197 +    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
  59.198 +    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
  59.199 +    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
  59.200 +    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
  59.201 +    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
  59.202 +    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
  59.203 +    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
  59.204 +    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
  59.205 +    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
  59.206 +    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
  59.207 +    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
  59.208 +    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
  59.209 +    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
  59.210 +    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
  59.211 +    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
  59.212 +    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
  59.213 +    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
  59.214 +    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
  59.215 +    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
  59.216 +    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
  59.217 +    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
  59.218 +    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
  59.219 +    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
  59.220 +    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
  59.221 +    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
  59.222 +    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
  59.223 +    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
  59.224 +    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
  59.225 +    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
  59.226 +    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
  59.227 +    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
  59.228 +    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
  59.229 +    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
  59.230 +    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
  59.231 +    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
  59.232 +    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
  59.233 +    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
  59.234 +    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
  59.235 +    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
  59.236 +    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
  59.237 +    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
  59.238 +    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
  59.239 +    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
  59.240 +    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
  59.241 +    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
  59.242 +    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
  59.243 +    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
  59.244 +    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
  59.245 +    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
  59.246 +    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
  59.247 +    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
  59.248 +    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
  59.249 +    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
  59.250 +    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
  59.251 +    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
  59.252 +    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
  59.253 +};
  59.254 +static const u32 Te3[256] = {
  59.255 +    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
  59.256 +    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
  59.257 +    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
  59.258 +    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
  59.259 +    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
  59.260 +    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
  59.261 +    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
  59.262 +    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
  59.263 +    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
  59.264 +    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
  59.265 +    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
  59.266 +    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
  59.267 +    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
  59.268 +    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
  59.269 +    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
  59.270 +    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
  59.271 +    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
  59.272 +    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
  59.273 +    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
  59.274 +    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
  59.275 +    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
  59.276 +    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
  59.277 +    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
  59.278 +    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
  59.279 +    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
  59.280 +    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
  59.281 +    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
  59.282 +    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
  59.283 +    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
  59.284 +    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
  59.285 +    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
  59.286 +    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
  59.287 +    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
  59.288 +    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
  59.289 +    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
  59.290 +    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
  59.291 +    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
  59.292 +    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
  59.293 +    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
  59.294 +    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
  59.295 +    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
  59.296 +    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
  59.297 +    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
  59.298 +    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
  59.299 +    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
  59.300 +    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
  59.301 +    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
  59.302 +    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
  59.303 +    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
  59.304 +    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
  59.305 +    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
  59.306 +    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
  59.307 +    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
  59.308 +    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
  59.309 +    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
  59.310 +    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
  59.311 +    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
  59.312 +    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
  59.313 +    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
  59.314 +    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
  59.315 +    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
  59.316 +    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
  59.317 +    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
  59.318 +    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
  59.319 +};
  59.320 +static const u32 Te4[256] = {
  59.321 +    0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
  59.322 +    0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
  59.323 +    0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
  59.324 +    0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
  59.325 +    0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
  59.326 +    0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
  59.327 +    0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
  59.328 +    0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
  59.329 +    0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
  59.330 +    0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
  59.331 +    0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
  59.332 +    0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
  59.333 +    0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
  59.334 +    0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
  59.335 +    0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
  59.336 +    0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
  59.337 +    0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
  59.338 +    0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
  59.339 +    0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
  59.340 +    0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
  59.341 +    0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
  59.342 +    0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
  59.343 +    0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
  59.344 +    0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
  59.345 +    0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
  59.346 +    0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
  59.347 +    0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
  59.348 +    0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
  59.349 +    0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
  59.350 +    0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
  59.351 +    0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
  59.352 +    0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
  59.353 +    0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
  59.354 +    0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
  59.355 +    0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
  59.356 +    0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
  59.357 +    0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
  59.358 +    0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
  59.359 +    0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
  59.360 +    0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
  59.361 +    0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
  59.362 +    0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
  59.363 +    0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
  59.364 +    0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
  59.365 +    0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
  59.366 +    0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
  59.367 +    0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
  59.368 +    0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
  59.369 +    0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
  59.370 +    0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
  59.371 +    0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
  59.372 +    0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
  59.373 +    0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
  59.374 +    0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
  59.375 +    0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
  59.376 +    0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
  59.377 +    0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
  59.378 +    0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
  59.379 +    0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
  59.380 +    0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
  59.381 +    0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
  59.382 +    0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
  59.383 +    0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
  59.384 +    0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U,
  59.385 +};
  59.386 +static const u32 Td0[256] = {
  59.387 +    0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
  59.388 +    0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
  59.389 +    0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
  59.390 +    0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
  59.391 +    0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
  59.392 +    0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
  59.393 +    0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
  59.394 +    0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
  59.395 +    0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
  59.396 +    0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
  59.397 +    0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
  59.398 +    0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
  59.399 +    0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
  59.400 +    0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
  59.401 +    0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
  59.402 +    0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
  59.403 +    0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
  59.404 +    0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
  59.405 +    0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
  59.406 +    0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
  59.407 +    0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
  59.408 +    0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
  59.409 +    0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
  59.410 +    0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
  59.411 +    0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
  59.412 +    0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
  59.413 +    0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
  59.414 +    0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
  59.415 +    0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
  59.416 +    0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
  59.417 +    0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
  59.418 +    0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
  59.419 +    0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
  59.420 +    0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
  59.421 +    0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
  59.422 +    0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
  59.423 +    0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
  59.424 +    0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
  59.425 +    0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
  59.426 +    0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
  59.427 +    0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
  59.428 +    0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
  59.429 +    0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
  59.430 +    0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
  59.431 +    0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
  59.432 +    0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
  59.433 +    0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
  59.434 +    0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
  59.435 +    0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
  59.436 +    0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
  59.437 +    0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
  59.438 +    0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
  59.439 +    0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
  59.440 +    0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
  59.441 +    0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
  59.442 +    0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
  59.443 +    0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
  59.444 +    0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
  59.445 +    0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
  59.446 +    0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
  59.447 +    0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
  59.448 +    0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
  59.449 +    0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
  59.450 +    0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
  59.451 +};
  59.452 +static const u32 Td1[256] = {
  59.453 +    0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
  59.454 +    0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
  59.455 +    0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
  59.456 +    0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
  59.457 +    0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
  59.458 +    0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
  59.459 +    0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
  59.460 +    0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
  59.461 +    0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
  59.462 +    0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
  59.463 +    0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
  59.464 +    0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
  59.465 +    0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
  59.466 +    0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
  59.467 +    0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
  59.468 +    0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
  59.469 +    0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
  59.470 +    0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
  59.471 +    0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
  59.472 +    0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
  59.473 +    0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
  59.474 +    0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
  59.475 +    0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
  59.476 +    0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
  59.477 +    0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
  59.478 +    0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
  59.479 +    0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
  59.480 +    0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
  59.481 +    0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
  59.482 +    0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
  59.483 +    0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
  59.484 +    0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
  59.485 +    0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
  59.486 +    0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
  59.487 +    0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
  59.488 +    0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
  59.489 +    0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
  59.490 +    0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
  59.491 +    0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
  59.492 +    0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
  59.493 +    0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
  59.494 +    0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
  59.495 +    0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
  59.496 +    0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
  59.497 +    0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
  59.498 +    0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
  59.499 +    0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
  59.500 +    0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
  59.501 +    0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
  59.502 +    0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
  59.503 +    0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
  59.504 +    0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
  59.505 +    0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
  59.506 +    0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
  59.507 +    0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
  59.508 +    0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
  59.509 +    0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
  59.510 +    0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
  59.511 +    0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
  59.512 +    0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
  59.513 +    0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
  59.514 +    0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
  59.515 +    0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
  59.516 +    0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
  59.517 +};
  59.518 +static const u32 Td2[256] = {
  59.519 +    0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
  59.520 +    0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
  59.521 +    0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
  59.522 +    0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
  59.523 +    0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
  59.524 +    0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
  59.525 +    0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
  59.526 +    0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
  59.527 +    0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
  59.528 +    0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
  59.529 +    0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
  59.530 +    0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
  59.531 +    0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
  59.532 +    0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
  59.533 +    0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
  59.534 +    0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
  59.535 +    0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
  59.536 +    0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
  59.537 +    0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
  59.538 +    0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
  59.539 +    0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
  59.540 +    0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
  59.541 +    0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
  59.542 +    0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
  59.543 +    0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
  59.544 +    0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
  59.545 +    0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
  59.546 +    0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
  59.547 +    0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
  59.548 +    0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
  59.549 +    0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
  59.550 +    0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
  59.551 +    0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
  59.552 +    0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
  59.553 +    0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
  59.554 +    0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
  59.555 +    0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
  59.556 +    0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
  59.557 +    0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
  59.558 +    0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
  59.559 +    0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
  59.560 +    0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
  59.561 +    0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
  59.562 +    0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
  59.563 +    0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
  59.564 +    0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
  59.565 +    0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
  59.566 +    0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
  59.567 +    0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
  59.568 +    0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
  59.569 +    0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
  59.570 +    0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
  59.571 +    0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
  59.572 +    0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
  59.573 +    0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
  59.574 +    0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
  59.575 +    0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
  59.576 +    0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
  59.577 +    0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
  59.578 +    0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
  59.579 +    0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
  59.580 +    0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
  59.581 +    0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
  59.582 +    0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
  59.583 +};
  59.584 +static const u32 Td3[256] = {
  59.585 +    0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
  59.586 +    0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
  59.587 +    0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
  59.588 +    0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
  59.589 +    0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
  59.590 +    0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
  59.591 +    0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
  59.592 +    0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
  59.593 +    0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
  59.594 +    0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
  59.595 +    0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
  59.596 +    0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
  59.597 +    0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
  59.598 +    0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
  59.599 +    0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
  59.600 +    0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
  59.601 +    0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
  59.602 +    0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
  59.603 +    0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
  59.604 +    0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
  59.605 +    0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
  59.606 +    0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
  59.607 +    0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
  59.608 +    0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
  59.609 +    0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
  59.610 +    0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
  59.611 +    0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
  59.612 +    0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
  59.613 +    0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
  59.614 +    0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
  59.615 +    0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
  59.616 +    0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
  59.617 +    0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
  59.618 +    0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
  59.619 +    0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
  59.620 +    0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
  59.621 +    0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
  59.622 +    0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
  59.623 +    0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
  59.624 +    0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
  59.625 +    0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
  59.626 +    0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
  59.627 +    0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
  59.628 +    0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
  59.629 +    0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
  59.630 +    0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
  59.631 +    0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
  59.632 +    0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
  59.633 +    0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
  59.634 +    0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
  59.635 +    0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
  59.636 +    0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
  59.637 +    0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
  59.638 +    0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
  59.639 +    0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
  59.640 +    0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
  59.641 +    0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
  59.642 +    0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
  59.643 +    0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
  59.644 +    0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
  59.645 +    0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
  59.646 +    0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
  59.647 +    0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
  59.648 +    0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
  59.649 +};
  59.650 +static const u32 Td4[256] = {
  59.651 +    0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U,
  59.652 +    0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U,
  59.653 +    0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU,
  59.654 +    0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU,
  59.655 +    0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U,
  59.656 +    0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U,
  59.657 +    0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U,
  59.658 +    0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU,
  59.659 +    0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U,
  59.660 +    0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU,
  59.661 +    0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU,
  59.662 +    0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU,
  59.663 +    0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U,
  59.664 +    0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U,
  59.665 +    0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U,
  59.666 +    0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U,
  59.667 +    0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U,
  59.668 +    0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U,
  59.669 +    0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU,
  59.670 +    0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U,
  59.671 +    0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U,
  59.672 +    0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU,
  59.673 +    0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U,
  59.674 +    0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U,
  59.675 +    0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U,
  59.676 +    0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU,
  59.677 +    0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U,
  59.678 +    0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U,
  59.679 +    0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU,
  59.680 +    0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U,
  59.681 +    0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U,
  59.682 +    0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU,
  59.683 +    0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U,
  59.684 +    0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU,
  59.685 +    0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU,
  59.686 +    0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U,
  59.687 +    0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U,
  59.688 +    0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U,
  59.689 +    0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U,
  59.690 +    0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU,
  59.691 +    0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U,
  59.692 +    0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U,
  59.693 +    0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU,
  59.694 +    0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU,
  59.695 +    0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU,
  59.696 +    0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U,
  59.697 +    0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU,
  59.698 +    0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U,
  59.699 +    0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U,
  59.700 +    0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U,
  59.701 +    0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U,
  59.702 +    0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU,
  59.703 +    0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U,
  59.704 +    0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU,
  59.705 +    0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU,
  59.706 +    0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU,
  59.707 +    0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU,
  59.708 +    0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U,
  59.709 +    0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU,
  59.710 +    0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U,
  59.711 +    0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU,
  59.712 +    0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U,
  59.713 +    0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U,
  59.714 +    0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU,
  59.715 +};
  59.716 +static const u32 rcon[] = {
  59.717 +	0x01000000, 0x02000000, 0x04000000, 0x08000000,
  59.718 +	0x10000000, 0x20000000, 0x40000000, 0x80000000,
  59.719 +	0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
  59.720 +};
  59.721 +
  59.722 +#define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] <<  8) ^ ((u32)(pt)[3]))
  59.723 +#define PUTU32(ct, st) { (ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); (ct)[2] = (u8)((st) >>  8); (ct)[3] = (u8)(st); }
  59.724 +
  59.725 +/**
  59.726 + * Expand the cipher key into the encryption key schedule.
  59.727 + *
  59.728 + * @return	the number of rounds for the given cipher key size.
  59.729 + */
  59.730 +int
  59.731 +rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], int keyBits)
  59.732 +{
  59.733 +   	int i = 0;
  59.734 +	u32 temp;
  59.735 +
  59.736 +	rk[0] = GETU32(cipherKey     );
  59.737 +	rk[1] = GETU32(cipherKey +  4);
  59.738 +	rk[2] = GETU32(cipherKey +  8);
  59.739 +	rk[3] = GETU32(cipherKey + 12);
  59.740 +	if (keyBits == 128) {
  59.741 +		for (;;) {
  59.742 +			temp  = rk[3];
  59.743 +			rk[4] = rk[0] ^
  59.744 +				(Te4[(temp >> 16) & 0xff] & 0xff000000) ^
  59.745 +				(Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
  59.746 +				(Te4[(temp      ) & 0xff] & 0x0000ff00) ^
  59.747 +				(Te4[(temp >> 24)       ] & 0x000000ff) ^
  59.748 +				rcon[i];
  59.749 +			rk[5] = rk[1] ^ rk[4];
  59.750 +			rk[6] = rk[2] ^ rk[5];
  59.751 +			rk[7] = rk[3] ^ rk[6];
  59.752 +			if (++i == 10) {
  59.753 +				return 10;
  59.754 +			}
  59.755 +			rk += 4;
  59.756 +		}
  59.757 +	}
  59.758 +	rk[4] = GETU32(cipherKey + 16);
  59.759 +	rk[5] = GETU32(cipherKey + 20);
  59.760 +	if (keyBits == 192) {
  59.761 +		for (;;) {
  59.762 +			temp = rk[ 5];
  59.763 +			rk[ 6] = rk[ 0] ^
  59.764 +				(Te4[(temp >> 16) & 0xff] & 0xff000000) ^
  59.765 +				(Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
  59.766 +				(Te4[(temp      ) & 0xff] & 0x0000ff00) ^
  59.767 +				(Te4[(temp >> 24)       ] & 0x000000ff) ^
  59.768 +				rcon[i];
  59.769 +			rk[ 7] = rk[ 1] ^ rk[ 6];
  59.770 +			rk[ 8] = rk[ 2] ^ rk[ 7];
  59.771 +			rk[ 9] = rk[ 3] ^ rk[ 8];
  59.772 +			if (++i == 8) {
  59.773 +				return 12;
  59.774 +			}
  59.775 +			rk[10] = rk[ 4] ^ rk[ 9];
  59.776 +			rk[11] = rk[ 5] ^ rk[10];
  59.777 +			rk += 6;
  59.778 +		}
  59.779 +	}
  59.780 +	rk[6] = GETU32(cipherKey + 24);
  59.781 +	rk[7] = GETU32(cipherKey + 28);
  59.782 +	if (keyBits == 256) {
  59.783 +		for (;;) {
  59.784 +			temp = rk[ 7];
  59.785 +			rk[ 8] = rk[ 0] ^
  59.786 +				(Te4[(temp >> 16) & 0xff] & 0xff000000) ^
  59.787 +				(Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
  59.788 +				(Te4[(temp      ) & 0xff] & 0x0000ff00) ^
  59.789 +				(Te4[(temp >> 24)       ] & 0x000000ff) ^
  59.790 +				rcon[i];
  59.791 +			rk[ 9] = rk[ 1] ^ rk[ 8];
  59.792 +			rk[10] = rk[ 2] ^ rk[ 9];
  59.793 +			rk[11] = rk[ 3] ^ rk[10];
  59.794 +			if (++i == 7) {
  59.795 +				return 14;
  59.796 +			}
  59.797 +			temp = rk[11];
  59.798 +			rk[12] = rk[ 4] ^
  59.799 +				(Te4[(temp >> 24)       ] & 0xff000000) ^
  59.800 +				(Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^
  59.801 +				(Te4[(temp >>  8) & 0xff] & 0x0000ff00) ^
  59.802 +				(Te4[(temp      ) & 0xff] & 0x000000ff);
  59.803 +			rk[13] = rk[ 5] ^ rk[12];
  59.804 +			rk[14] = rk[ 6] ^ rk[13];
  59.805 +		     	rk[15] = rk[ 7] ^ rk[14];
  59.806 +			rk += 8;
  59.807 +		}
  59.808 +	}
  59.809 +	return 0;
  59.810 +}
  59.811 +
  59.812 +/**
  59.813 + * Expand the cipher key into the decryption key schedule.
  59.814 + *
  59.815 + * @return	the number of rounds for the given cipher key size.
  59.816 + */
  59.817 +int
  59.818 +rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], int keyBits)
  59.819 +{
  59.820 +	int Nr, i, j;
  59.821 +	u32 temp;
  59.822 +
  59.823 +	/* expand the cipher key: */
  59.824 +	Nr = rijndaelKeySetupEnc(rk, cipherKey, keyBits);
  59.825 +
  59.826 +	/* invert the order of the round keys: */
  59.827 +	for (i = 0, j = 4*Nr; i < j; i += 4, j -= 4) {
  59.828 +		temp = rk[i    ]; rk[i    ] = rk[j    ]; rk[j    ] = temp;
  59.829 +		temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
  59.830 +		temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
  59.831 +		temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp;
  59.832 +	}
  59.833 +	/* apply the inverse MixColumn transform to all round keys but the first and the last: */
  59.834 +	for (i = 1; i < Nr; i++) {
  59.835 +		rk += 4;
  59.836 +		rk[0] =
  59.837 +			Td0[Te4[(rk[0] >> 24)       ] & 0xff] ^
  59.838 +			Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^
  59.839 +			Td2[Te4[(rk[0] >>  8) & 0xff] & 0xff] ^
  59.840 +			Td3[Te4[(rk[0]      ) & 0xff] & 0xff];
  59.841 +		rk[1] =
  59.842 +			Td0[Te4[(rk[1] >> 24)       ] & 0xff] ^
  59.843 +			Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^
  59.844 +			Td2[Te4[(rk[1] >>  8) & 0xff] & 0xff] ^
  59.845 +			Td3[Te4[(rk[1]      ) & 0xff] & 0xff];
  59.846 +		rk[2] =
  59.847 +			Td0[Te4[(rk[2] >> 24)       ] & 0xff] ^
  59.848 +			Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^
  59.849 +			Td2[Te4[(rk[2] >>  8) & 0xff] & 0xff] ^
  59.850 +			Td3[Te4[(rk[2]      ) & 0xff] & 0xff];
  59.851 +		rk[3] =
  59.852 +			Td0[Te4[(rk[3] >> 24)       ] & 0xff] ^
  59.853 +			Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^
  59.854 +			Td2[Te4[(rk[3] >>  8) & 0xff] & 0xff] ^
  59.855 +			Td3[Te4[(rk[3]      ) & 0xff] & 0xff];
  59.856 +	}
  59.857 +	return Nr;
  59.858 +}
  59.859 +
  59.860 +void
  59.861 +rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16],
  59.862 +    u8 ct[16])
  59.863 +{
  59.864 +	u32 s0, s1, s2, s3, t0, t1, t2, t3;
  59.865 +#ifndef FULL_UNROLL
  59.866 +    int r;
  59.867 +#endif /* ?FULL_UNROLL */
  59.868 +
  59.869 +    /*
  59.870 +	 * map byte array block to cipher state
  59.871 +	 * and add initial round key:
  59.872 +	 */
  59.873 +	s0 = GETU32(pt     ) ^ rk[0];
  59.874 +	s1 = GETU32(pt +  4) ^ rk[1];
  59.875 +	s2 = GETU32(pt +  8) ^ rk[2];
  59.876 +	s3 = GETU32(pt + 12) ^ rk[3];
  59.877 +#ifdef FULL_UNROLL
  59.878 +    /* round 1: */
  59.879 +   	t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[ 4];
  59.880 +   	t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[ 5];
  59.881 +   	t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[ 6];
  59.882 +   	t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[ 7];
  59.883 +   	/* round 2: */
  59.884 +   	s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[ 8];
  59.885 +   	s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[ 9];
  59.886 +   	s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[10];
  59.887 +   	s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[11];
  59.888 +    /* round 3: */
  59.889 +   	t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[12];
  59.890 +   	t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[13];
  59.891 +   	t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[14];
  59.892 +   	t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[15];
  59.893 +   	/* round 4: */
  59.894 +   	s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[16];
  59.895 +   	s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[17];
  59.896 +   	s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[18];
  59.897 +   	s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[19];
  59.898 +    /* round 5: */
  59.899 +   	t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[20];
  59.900 +   	t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[21];
  59.901 +   	t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[22];
  59.902 +   	t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[23];
  59.903 +   	/* round 6: */
  59.904 +   	s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[24];
  59.905 +   	s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[25];
  59.906 +   	s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[26];
  59.907 +   	s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[27];
  59.908 +    /* round 7: */
  59.909 +   	t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[28];
  59.910 +   	t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[29];
  59.911 +   	t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[30];
  59.912 +   	t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[31];
  59.913 +   	/* round 8: */
  59.914 +   	s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[32];
  59.915 +   	s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[33];
  59.916 +   	s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[34];
  59.917 +   	s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[35];
  59.918 +    /* round 9: */
  59.919 +   	t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[36];
  59.920 +   	t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[37];
  59.921 +   	t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[38];
  59.922 +   	t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[39];
  59.923 +    if (Nr > 10) {
  59.924 +	/* round 10: */
  59.925 +	s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[40];
  59.926 +	s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[41];
  59.927 +	s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[42];
  59.928 +	s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[43];
  59.929 +	/* round 11: */
  59.930 +	t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[44];
  59.931 +	t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[45];
  59.932 +	t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[46];
  59.933 +	t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[47];
  59.934 +	if (Nr > 12) {
  59.935 +	    /* round 12: */
  59.936 +	    s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[48];
  59.937 +	    s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[49];
  59.938 +	    s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[50];
  59.939 +	    s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[51];
  59.940 +	    /* round 13: */
  59.941 +	    t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[52];
  59.942 +	    t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[53];
  59.943 +	    t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[54];
  59.944 +	    t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[55];
  59.945 +	}
  59.946 +    }
  59.947 +    rk += Nr << 2;
  59.948 +#else  /* !FULL_UNROLL */
  59.949 +    /*
  59.950 +	 * Nr - 1 full rounds:
  59.951 +	 */
  59.952 +    r = Nr >> 1;
  59.953 +    for (;;) {
  59.954 +	t0 =
  59.955 +	    Te0[(s0 >> 24)       ] ^
  59.956 +	    Te1[(s1 >> 16) & 0xff] ^
  59.957 +	    Te2[(s2 >>  8) & 0xff] ^
  59.958 +	    Te3[(s3      ) & 0xff] ^
  59.959 +	    rk[4];
  59.960 +	t1 =
  59.961 +	    Te0[(s1 >> 24)       ] ^
  59.962 +	    Te1[(s2 >> 16) & 0xff] ^
  59.963 +	    Te2[(s3 >>  8) & 0xff] ^
  59.964 +	    Te3[(s0      ) & 0xff] ^
  59.965 +	    rk[5];
  59.966 +	t2 =
  59.967 +	    Te0[(s2 >> 24)       ] ^
  59.968 +	    Te1[(s3 >> 16) & 0xff] ^
  59.969 +	    Te2[(s0 >>  8) & 0xff] ^
  59.970 +	    Te3[(s1      ) & 0xff] ^
  59.971 +	    rk[6];
  59.972 +	t3 =
  59.973 +	    Te0[(s3 >> 24)       ] ^
  59.974 +	    Te1[(s0 >> 16) & 0xff] ^
  59.975 +	    Te2[(s1 >>  8) & 0xff] ^
  59.976 +	    Te3[(s2      ) & 0xff] ^
  59.977 +	    rk[7];
  59.978 +
  59.979 +	rk += 8;
  59.980 +	if (--r == 0) {
  59.981 +	    break;
  59.982 +	}
  59.983 +
  59.984 +	s0 =
  59.985 +	    Te0[(t0 >> 24)       ] ^
  59.986 +	    Te1[(t1 >> 16) & 0xff] ^
  59.987 +	    Te2[(t2 >>  8) & 0xff] ^
  59.988 +	    Te3[(t3      ) & 0xff] ^
  59.989 +	    rk[0];
  59.990 +	s1 =
  59.991 +	    Te0[(t1 >> 24)       ] ^
  59.992 +	    Te1[(t2 >> 16) & 0xff] ^
  59.993 +	    Te2[(t3 >>  8) & 0xff] ^
  59.994 +	    Te3[(t0      ) & 0xff] ^
  59.995 +	    rk[1];
  59.996 +	s2 =
  59.997 +	    Te0[(t2 >> 24)       ] ^
  59.998 +	    Te1[(t3 >> 16) & 0xff] ^
  59.999 +	    Te2[(t0 >>  8) & 0xff] ^
 59.1000 +	    Te3[(t1      ) & 0xff] ^
 59.1001 +	    rk[2];
 59.1002 +	s3 =
 59.1003 +	    Te0[(t3 >> 24)       ] ^
 59.1004 +	    Te1[(t0 >> 16) & 0xff] ^
 59.1005 +	    Te2[(t1 >>  8) & 0xff] ^
 59.1006 +	    Te3[(t2      ) & 0xff] ^
 59.1007 +	    rk[3];
 59.1008 +    }
 59.1009 +#endif /* ?FULL_UNROLL */
 59.1010 +    /*
 59.1011 +	 * apply last round and
 59.1012 +	 * map cipher state to byte array block:
 59.1013 +	 */
 59.1014 +	s0 =
 59.1015 +		(Te4[(t0 >> 24)       ] & 0xff000000) ^
 59.1016 +		(Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
 59.1017 +		(Te4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
 59.1018 +		(Te4[(t3      ) & 0xff] & 0x000000ff) ^
 59.1019 +		rk[0];
 59.1020 +	PUTU32(ct     , s0);
 59.1021 +	s1 =
 59.1022 +		(Te4[(t1 >> 24)       ] & 0xff000000) ^
 59.1023 +		(Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
 59.1024 +		(Te4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
 59.1025 +		(Te4[(t0      ) & 0xff] & 0x000000ff) ^
 59.1026 +		rk[1];
 59.1027 +	PUTU32(ct +  4, s1);
 59.1028 +	s2 =
 59.1029 +		(Te4[(t2 >> 24)       ] & 0xff000000) ^
 59.1030 +		(Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
 59.1031 +		(Te4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
 59.1032 +		(Te4[(t1      ) & 0xff] & 0x000000ff) ^
 59.1033 +		rk[2];
 59.1034 +	PUTU32(ct +  8, s2);
 59.1035 +	s3 =
 59.1036 +		(Te4[(t3 >> 24)       ] & 0xff000000) ^
 59.1037 +		(Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
 59.1038 +		(Te4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
 59.1039 +		(Te4[(t2      ) & 0xff] & 0x000000ff) ^
 59.1040 +		rk[3];
 59.1041 +	PUTU32(ct + 12, s3);
 59.1042 +}
 59.1043 +
 59.1044 +static void
 59.1045 +rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16],
 59.1046 +    u8 pt[16])
 59.1047 +{
 59.1048 +	u32 s0, s1, s2, s3, t0, t1, t2, t3;
 59.1049 +#ifndef FULL_UNROLL
 59.1050 +    int r;
 59.1051 +#endif /* ?FULL_UNROLL */
 59.1052 +
 59.1053 +    /*
 59.1054 +	 * map byte array block to cipher state
 59.1055 +	 * and add initial round key:
 59.1056 +	 */
 59.1057 +    s0 = GETU32(ct     ) ^ rk[0];
 59.1058 +    s1 = GETU32(ct +  4) ^ rk[1];
 59.1059 +    s2 = GETU32(ct +  8) ^ rk[2];
 59.1060 +    s3 = GETU32(ct + 12) ^ rk[3];
 59.1061 +#ifdef FULL_UNROLL
 59.1062 +    /* round 1: */
 59.1063 +    t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >>  8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[ 4];
 59.1064 +    t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >>  8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[ 5];
 59.1065 +    t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >>  8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[ 6];
 59.1066 +    t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >>  8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[ 7];
 59.1067 +    /* round 2: */
 59.1068 +    s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >>  8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[ 8];
 59.1069 +    s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >>  8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[ 9];
 59.1070 +    s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >>  8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[10];
 59.1071 +    s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >>  8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[11];
 59.1072 +    /* round 3: */
 59.1073 +    t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >>  8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[12];
 59.1074 +    t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >>  8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[13];
 59.1075 +    t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >>  8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[14];
 59.1076 +    t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >>  8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[15];
 59.1077 +    /* round 4: */
 59.1078 +    s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >>  8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[16];
 59.1079 +    s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >>  8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[17];
 59.1080 +    s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >>  8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[18];
 59.1081 +    s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >>  8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[19];
 59.1082 +    /* round 5: */
 59.1083 +    t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >>  8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[20];
 59.1084 +    t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >>  8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[21];
 59.1085 +    t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >>  8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[22];
 59.1086 +    t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >>  8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[23];
 59.1087 +    /* round 6: */
 59.1088 +    s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >>  8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[24];
 59.1089 +    s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >>  8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[25];
 59.1090 +    s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >>  8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[26];
 59.1091 +    s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >>  8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[27];
 59.1092 +    /* round 7: */
 59.1093 +    t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >>  8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[28];
 59.1094 +    t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >>  8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[29];
 59.1095 +    t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >>  8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[30];
 59.1096 +    t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >>  8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[31];
 59.1097 +    /* round 8: */
 59.1098 +    s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >>  8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[32];
 59.1099 +    s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >>  8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[33];
 59.1100 +    s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >>  8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[34];
 59.1101 +    s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >>  8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[35];
 59.1102 +    /* round 9: */
 59.1103 +    t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >>  8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[36];
 59.1104 +    t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >>  8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[37];
 59.1105 +    t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >>  8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[38];
 59.1106 +    t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >>  8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[39];
 59.1107 +    if (Nr > 10) {
 59.1108 +	/* round 10: */
 59.1109 +	s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >>  8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[40];
 59.1110 +	s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >>  8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[41];
 59.1111 +	s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >>  8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[42];
 59.1112 +	s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >>  8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[43];
 59.1113 +	/* round 11: */
 59.1114 +	t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >>  8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[44];
 59.1115 +	t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >>  8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[45];
 59.1116 +	t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >>  8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[46];
 59.1117 +	t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >>  8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[47];
 59.1118 +	if (Nr > 12) {
 59.1119 +	    /* round 12: */
 59.1120 +	    s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >>  8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[48];
 59.1121 +	    s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >>  8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[49];
 59.1122 +	    s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >>  8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[50];
 59.1123 +	    s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >>  8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[51];
 59.1124 +	    /* round 13: */
 59.1125 +	    t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >>  8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[52];
 59.1126 +	    t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >>  8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[53];
 59.1127 +	    t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >>  8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[54];
 59.1128 +	    t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >>  8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[55];
 59.1129 +	}
 59.1130 +    }
 59.1131 +	rk += Nr << 2;
 59.1132 +#else  /* !FULL_UNROLL */
 59.1133 +    /*
 59.1134 +     * Nr - 1 full rounds:
 59.1135 +     */
 59.1136 +    r = Nr >> 1;
 59.1137 +    for (;;) {
 59.1138 +	t0 =
 59.1139 +	    Td0[(s0 >> 24)       ] ^
 59.1140 +	    Td1[(s3 >> 16) & 0xff] ^
 59.1141 +	    Td2[(s2 >>  8) & 0xff] ^
 59.1142 +	    Td3[(s1      ) & 0xff] ^
 59.1143 +	    rk[4];
 59.1144 +	t1 =
 59.1145 +	    Td0[(s1 >> 24)       ] ^
 59.1146 +	    Td1[(s0 >> 16) & 0xff] ^
 59.1147 +	    Td2[(s3 >>  8) & 0xff] ^
 59.1148 +	    Td3[(s2      ) & 0xff] ^
 59.1149 +	    rk[5];
 59.1150 +	t2 =
 59.1151 +	    Td0[(s2 >> 24)       ] ^
 59.1152 +	    Td1[(s1 >> 16) & 0xff] ^
 59.1153 +	    Td2[(s0 >>  8) & 0xff] ^
 59.1154 +	    Td3[(s3      ) & 0xff] ^
 59.1155 +	    rk[6];
 59.1156 +	t3 =
 59.1157 +	    Td0[(s3 >> 24)       ] ^
 59.1158 +	    Td1[(s2 >> 16) & 0xff] ^
 59.1159 +	    Td2[(s1 >>  8) & 0xff] ^
 59.1160 +	    Td3[(s0      ) & 0xff] ^
 59.1161 +	    rk[7];
 59.1162 +
 59.1163 +	rk += 8;
 59.1164 +	if (--r == 0) {
 59.1165 +	    break;
 59.1166 +	}
 59.1167 +
 59.1168 +	s0 =
 59.1169 +	    Td0[(t0 >> 24)       ] ^
 59.1170 +	    Td1[(t3 >> 16) & 0xff] ^
 59.1171 +	    Td2[(t2 >>  8) & 0xff] ^
 59.1172 +	    Td3[(t1      ) & 0xff] ^
 59.1173 +	    rk[0];
 59.1174 +	s1 =
 59.1175 +	    Td0[(t1 >> 24)       ] ^
 59.1176 +	    Td1[(t0 >> 16) & 0xff] ^
 59.1177 +	    Td2[(t3 >>  8) & 0xff] ^
 59.1178 +	    Td3[(t2      ) & 0xff] ^
 59.1179 +	    rk[1];
 59.1180 +	s2 =
 59.1181 +	    Td0[(t2 >> 24)       ] ^
 59.1182 +	    Td1[(t1 >> 16) & 0xff] ^
 59.1183 +	    Td2[(t0 >>  8) & 0xff] ^
 59.1184 +	    Td3[(t3      ) & 0xff] ^
 59.1185 +	    rk[2];
 59.1186 +	s3 =
 59.1187 +	    Td0[(t3 >> 24)       ] ^
 59.1188 +	    Td1[(t2 >> 16) & 0xff] ^
 59.1189 +	    Td2[(t1 >>  8) & 0xff] ^
 59.1190 +	    Td3[(t0      ) & 0xff] ^
 59.1191 +	    rk[3];
 59.1192 +    }
 59.1193 +#endif /* ?FULL_UNROLL */
 59.1194 +    /*
 59.1195 +	 * apply last round and
 59.1196 +	 * map cipher state to byte array block:
 59.1197 +	 */
 59.1198 +   	s0 =
 59.1199 +   		(Td4[(t0 >> 24)       ] & 0xff000000) ^
 59.1200 +   		(Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
 59.1201 +   		(Td4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
 59.1202 +   		(Td4[(t1      ) & 0xff] & 0x000000ff) ^
 59.1203 +   		rk[0];
 59.1204 +	PUTU32(pt     , s0);
 59.1205 +   	s1 =
 59.1206 +   		(Td4[(t1 >> 24)       ] & 0xff000000) ^
 59.1207 +   		(Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
 59.1208 +   		(Td4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
 59.1209 +   		(Td4[(t2      ) & 0xff] & 0x000000ff) ^
 59.1210 +   		rk[1];
 59.1211 +	PUTU32(pt +  4, s1);
 59.1212 +   	s2 =
 59.1213 +   		(Td4[(t2 >> 24)       ] & 0xff000000) ^
 59.1214 +   		(Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
 59.1215 +   		(Td4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
 59.1216 +   		(Td4[(t3      ) & 0xff] & 0x000000ff) ^
 59.1217 +   		rk[2];
 59.1218 +	PUTU32(pt +  8, s2);
 59.1219 +   	s3 =
 59.1220 +   		(Td4[(t3 >> 24)       ] & 0xff000000) ^
 59.1221 +   		(Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
 59.1222 +   		(Td4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
 59.1223 +   		(Td4[(t0      ) & 0xff] & 0x000000ff) ^
 59.1224 +   		rk[3];
 59.1225 +	PUTU32(pt + 12, s3);
 59.1226 +}
 59.1227 +
 59.1228 +/* setup key context for encryption only */
 59.1229 +int
 59.1230 +rijndael_set_key_enc_only(rijndael_ctx *ctx, const u_char *key, int bits)
 59.1231 +{
 59.1232 +	int rounds;
 59.1233 +
 59.1234 +	rounds = rijndaelKeySetupEnc(ctx->ek, key, bits);
 59.1235 +	if (rounds == 0)
 59.1236 +		return -1;
 59.1237 +
 59.1238 +	ctx->Nr = rounds;
 59.1239 +	ctx->enc_only = 1;
 59.1240 +
 59.1241 +	return 0;
 59.1242 +}
 59.1243 +
 59.1244 +/* setup key context for both encryption and decryption */
 59.1245 +int
 59.1246 +rijndael_set_key(rijndael_ctx *ctx, const u_char *key, int bits)
 59.1247 +{
 59.1248 +	int rounds;
 59.1249 +
 59.1250 +	rounds = rijndaelKeySetupEnc(ctx->ek, key, bits);
 59.1251 +	if (rounds == 0)
 59.1252 +		return -1;
 59.1253 +	if (rijndaelKeySetupDec(ctx->dk, key, bits) != rounds)
 59.1254 +		return -1;
 59.1255 +
 59.1256 +	ctx->Nr = rounds;
 59.1257 +	ctx->enc_only = 0;
 59.1258 +
 59.1259 +	return 0;
 59.1260 +}
 59.1261 +
 59.1262 +void
 59.1263 +rijndael_decrypt(rijndael_ctx *ctx, const u_char *src, u_char *dst)
 59.1264 +{
 59.1265 +	rijndaelDecrypt(ctx->dk, ctx->Nr, src, dst);
 59.1266 +}
 59.1267 +
 59.1268 +void
 59.1269 +rijndael_encrypt(rijndael_ctx *ctx, const u_char *src, u_char *dst)
 59.1270 +{
 59.1271 +	rijndaelEncrypt(ctx->ek, ctx->Nr, src, dst);
 59.1272 +}