ia64/xen-unstable
changeset 19277:2a4eaa76d974
merge with xen-unstable.hg.
author | Isaku Yamahata <yamahata@valinux.co.jp> |
---|---|
date | Fri Mar 06 12:22:22 2009 +0900 (2009-03-06) |
parents | 6227bf629626 cff29d694a89 |
children | dee5bc89873e |
files |
line diff
1.1 --- a/extras/mini-os/arch/ia64/mm.c Mon Mar 02 18:26:56 2009 +0900 1.2 +++ b/extras/mini-os/arch/ia64/mm.c Fri Mar 06 12:22:22 2009 +0900 1.3 @@ -156,6 +156,12 @@ map_frames_ex(unsigned long* frames, uns 1.4 return (void*) __va(frames[0] << PAGE_SHIFT); 1.5 } 1.6 1.7 +int unmap_frames(unsigned long virt_addr, unsigned long num_frames) 1.8 +{ 1.9 + /* TODO */ 1.10 + ASSERT(0); 1.11 +} 1.12 + 1.13 void arch_init_p2m(unsigned long max_pfn) 1.14 { 1.15 printk("Warn: p2m map not implemented.\n");
2.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 2.2 +++ b/extras/mini-os/arch/x86/ioremap.c Fri Mar 06 12:22:22 2009 +0900 2.3 @@ -0,0 +1,88 @@ 2.4 +/* 2.5 + * Copyright (C) 2009, Netronome Systems, Inc. 2.6 + * 2.7 + * Permission is hereby granted, free of charge, to any person obtaining a copy 2.8 + * of this software and associated documentation files (the "Software"), to 2.9 + * deal in the Software without restriction, including without limitation the 2.10 + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 2.11 + * sell copies of the Software, and to permit persons to whom the Software is 2.12 + * furnished to do so, subject to the following conditions: 2.13 + * 2.14 + * The above copyright notice and this permission notice shall be included in 2.15 + * all copies or substantial portions of the Software. 2.16 + * 2.17 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 2.18 + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 2.19 + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 2.20 + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2.21 + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 2.22 + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 2.23 + * DEALINGS IN THE SOFTWARE. 2.24 + */ 2.25 + 2.26 + 2.27 +#include <types.h> 2.28 +#include <lib.h> 2.29 +#include <xmalloc.h> 2.30 +#include <mm.h> 2.31 +#include <ioremap.h> 2.32 + 2.33 +/* Map a physical address range into virtual address space with provided 2.34 + * flags. Return a virtual address range it is mapped to. */ 2.35 +static void *__do_ioremap(unsigned long phys_addr, unsigned long size, 2.36 + unsigned long prot) 2.37 +{ 2.38 + unsigned long va; 2.39 + unsigned long mfns, mfn; 2.40 + unsigned long num_pages, offset; 2.41 + int i; 2.42 + 2.43 + /* allow non page aligned addresses but for mapping we need to align them */ 2.44 + offset = (phys_addr & ~PAGE_MASK); 2.45 + num_pages = (offset + size + PAGE_SIZE - 1) / PAGE_SIZE; 2.46 + phys_addr &= PAGE_MASK; 2.47 + mfns = mfn = phys_addr >> PAGE_SHIFT; 2.48 + 2.49 + /* sanity checks on list of MFNs */ 2.50 + for ( i = 0; i < num_pages; i++, mfn++ ) 2.51 + { 2.52 + if ( mfn_is_ram(mfn) ) 2.53 + { 2.54 + printk("ioremap: mfn 0x%ulx is RAM\n", mfn); 2.55 + goto mfn_invalid; 2.56 + } 2.57 + } 2.58 + va = (unsigned long)map_frames_ex(&mfns, num_pages, 0, 1, 1, 2.59 + DOMID_IO, 0, prot); 2.60 + return (void *)(va + offset); 2.61 + 2.62 +mfn_invalid: 2.63 + return NULL; 2.64 +} 2.65 + 2.66 +void *ioremap(unsigned long phys_addr, unsigned long size) 2.67 +{ 2.68 + return __do_ioremap(phys_addr, size, IO_PROT); 2.69 +} 2.70 + 2.71 +void *ioremap_nocache(unsigned long phys_addr, unsigned long size) 2.72 +{ 2.73 + return __do_ioremap(phys_addr, size, IO_PROT_NOCACHE); 2.74 +} 2.75 + 2.76 +/* Un-map the io-remapped region. Currently no list of existing mappings is 2.77 + * maintained, so the caller has to supply the size */ 2.78 +void iounmap(void *virt_addr, unsigned long size) 2.79 +{ 2.80 + unsigned long num_pages; 2.81 + unsigned long va = (unsigned long)virt_addr; 2.82 + 2.83 + /* work out number of frames to unmap */ 2.84 + num_pages = ((va & ~PAGE_MASK) + size + PAGE_SIZE - 1) / PAGE_SIZE; 2.85 + 2.86 + unmap_frames(va & PAGE_MASK, num_pages); 2.87 +} 2.88 + 2.89 + 2.90 + 2.91 +/* -*- Mode:C; c-basic-offset:4; tab-width:4 indent-tabs-mode:nil -*- */
3.1 --- a/extras/mini-os/arch/x86/mm.c Mon Mar 02 18:26:56 2009 +0900 3.2 +++ b/extras/mini-os/arch/x86/mm.c Fri Mar 06 12:22:22 2009 +0900 3.3 @@ -52,20 +52,26 @@ 3.4 unsigned long *phys_to_machine_mapping; 3.5 unsigned long mfn_zero; 3.6 extern char stack[]; 3.7 -extern void page_walk(unsigned long virt_addr); 3.8 +extern void page_walk(unsigned long va); 3.9 3.10 -void new_pt_frame(unsigned long *pt_pfn, unsigned long prev_l_mfn, 3.11 - unsigned long offset, unsigned long level) 3.12 +/* 3.13 + * Make pt_pfn a new 'level' page table frame and hook it into the page 3.14 + * table at offset in previous level MFN (pref_l_mfn). pt_pfn is a guest 3.15 + * PFN. 3.16 + */ 3.17 +static void new_pt_frame(unsigned long *pt_pfn, unsigned long prev_l_mfn, 3.18 + unsigned long offset, unsigned long level) 3.19 { 3.20 pgentry_t *tab = (pgentry_t *)start_info.pt_base; 3.21 unsigned long pt_page = (unsigned long)pfn_to_virt(*pt_pfn); 3.22 pgentry_t prot_e, prot_t; 3.23 mmu_update_t mmu_updates[1]; 3.24 + int rc; 3.25 3.26 prot_e = prot_t = 0; 3.27 - DEBUG("Allocating new L%d pt frame for pt_pfn=%lx, " 3.28 - "prev_l_mfn=%lx, offset=%lx", 3.29 - level, *pt_pfn, prev_l_mfn, offset); 3.30 + DEBUG("Allocating new L%d pt frame for pfn=%lx, " 3.31 + "prev_l_mfn=%lx, offset=%lx", 3.32 + level, *pt_pfn, prev_l_mfn, offset); 3.33 3.34 /* We need to clear the page, otherwise we might fail to map it 3.35 as a page table page */ 3.36 @@ -74,56 +80,63 @@ void new_pt_frame(unsigned long *pt_pfn, 3.37 switch ( level ) 3.38 { 3.39 case L1_FRAME: 3.40 - prot_e = L1_PROT; 3.41 - prot_t = L2_PROT; 3.42 - break; 3.43 + prot_e = L1_PROT; 3.44 + prot_t = L2_PROT; 3.45 + break; 3.46 case L2_FRAME: 3.47 - prot_e = L2_PROT; 3.48 - prot_t = L3_PROT; 3.49 - break; 3.50 + prot_e = L2_PROT; 3.51 + prot_t = L3_PROT; 3.52 + break; 3.53 #if defined(__x86_64__) 3.54 case L3_FRAME: 3.55 - prot_e = L3_PROT; 3.56 - prot_t = L4_PROT; 3.57 - break; 3.58 + prot_e = L3_PROT; 3.59 + prot_t = L4_PROT; 3.60 + break; 3.61 #endif 3.62 default: 3.63 - printk("new_pt_frame() called with invalid level number %d\n", level); 3.64 - do_exit(); 3.65 - break; 3.66 + printk("new_pt_frame() called with invalid level number %d\n", level); 3.67 + do_exit(); 3.68 + break; 3.69 } 3.70 3.71 - /* Update the entry */ 3.72 + /* Make PFN a page table page */ 3.73 #if defined(__x86_64__) 3.74 tab = pte_to_virt(tab[l4_table_offset(pt_page)]); 3.75 #endif 3.76 tab = pte_to_virt(tab[l3_table_offset(pt_page)]); 3.77 3.78 mmu_updates[0].ptr = (tab[l2_table_offset(pt_page)] & PAGE_MASK) + 3.79 - sizeof(pgentry_t) * l1_table_offset(pt_page); 3.80 + sizeof(pgentry_t) * l1_table_offset(pt_page); 3.81 mmu_updates[0].val = (pgentry_t)pfn_to_mfn(*pt_pfn) << PAGE_SHIFT | 3.82 - (prot_e & ~_PAGE_RW); 3.83 - if(HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF) < 0) 3.84 + (prot_e & ~_PAGE_RW); 3.85 + 3.86 + if ( (rc = HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF)) < 0 ) 3.87 { 3.88 - printk("PTE for new page table page could not be updated\n"); 3.89 - do_exit(); 3.90 + printk("ERROR: PTE for new page table page could not be updated\n"); 3.91 + printk(" mmu_update failed with rc=%d\n", rc); 3.92 + do_exit(); 3.93 } 3.94 - 3.95 - /* Now fill the new page table page with entries. 3.96 - Update the page directory as well. */ 3.97 - mmu_updates[0].ptr = ((pgentry_t)prev_l_mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset; 3.98 + 3.99 + /* Hook the new page table page into the hierarchy */ 3.100 + mmu_updates[0].ptr = 3.101 + ((pgentry_t)prev_l_mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset; 3.102 mmu_updates[0].val = (pgentry_t)pfn_to_mfn(*pt_pfn) << PAGE_SHIFT | prot_t; 3.103 - if(HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF) < 0) 3.104 + 3.105 + if ( (rc = HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF)) < 0 ) 3.106 { 3.107 - printk("ERROR: mmu_update failed\n"); 3.108 - do_exit(); 3.109 + printk("ERROR: mmu_update failed with rc=%d\n", rc); 3.110 + do_exit(); 3.111 } 3.112 3.113 *pt_pfn += 1; 3.114 } 3.115 3.116 -/* Checks if a pagetable frame is needed (if weren't allocated by Xen) */ 3.117 -static int need_pt_frame(unsigned long virt_address, int level) 3.118 +/* 3.119 + * Checks if a pagetable frame is needed at 'level' to map a given 3.120 + * address. Note, this function is specific to the initial page table 3.121 + * building. 3.122 + */ 3.123 +static int need_pt_frame(unsigned long va, int level) 3.124 { 3.125 unsigned long hyp_virt_start = HYPERVISOR_VIRT_START; 3.126 #if defined(__x86_64__) 3.127 @@ -135,63 +148,71 @@ static int need_pt_frame(unsigned long v 3.128 /* In general frames will _not_ be needed if they were already 3.129 allocated to map the hypervisor into our VA space */ 3.130 #if defined(__x86_64__) 3.131 - if(level == L3_FRAME) 3.132 + if ( level == L3_FRAME ) 3.133 { 3.134 - if(l4_table_offset(virt_address) >= 3.135 - l4_table_offset(hyp_virt_start) && 3.136 - l4_table_offset(virt_address) <= 3.137 - l4_table_offset(hyp_virt_end)) 3.138 + if ( l4_table_offset(va) >= 3.139 + l4_table_offset(hyp_virt_start) && 3.140 + l4_table_offset(va) <= 3.141 + l4_table_offset(hyp_virt_end)) 3.142 return 0; 3.143 return 1; 3.144 - } else 3.145 + } 3.146 + else 3.147 #endif 3.148 3.149 - if(level == L2_FRAME) 3.150 + if ( level == L2_FRAME ) 3.151 { 3.152 #if defined(__x86_64__) 3.153 - if(l4_table_offset(virt_address) >= 3.154 - l4_table_offset(hyp_virt_start) && 3.155 - l4_table_offset(virt_address) <= 3.156 - l4_table_offset(hyp_virt_end)) 3.157 + if ( l4_table_offset(va) >= 3.158 + l4_table_offset(hyp_virt_start) && 3.159 + l4_table_offset(va) <= 3.160 + l4_table_offset(hyp_virt_end)) 3.161 #endif 3.162 - if(l3_table_offset(virt_address) >= 3.163 - l3_table_offset(hyp_virt_start) && 3.164 - l3_table_offset(virt_address) <= 3.165 - l3_table_offset(hyp_virt_end)) 3.166 + if ( l3_table_offset(va) >= 3.167 + l3_table_offset(hyp_virt_start) && 3.168 + l3_table_offset(va) <= 3.169 + l3_table_offset(hyp_virt_end)) 3.170 return 0; 3.171 3.172 return 1; 3.173 - } else 3.174 - 3.175 - /* Always need l1 frames */ 3.176 - if(level == L1_FRAME) 3.177 - return 1; 3.178 + } 3.179 + else 3.180 + /* Always need l1 frames */ 3.181 + if ( level == L1_FRAME ) 3.182 + return 1; 3.183 3.184 printk("ERROR: Unknown frame level %d, hypervisor %llx,%llx\n", 3.185 - level, hyp_virt_start, hyp_virt_end); 3.186 + level, hyp_virt_start, hyp_virt_end); 3.187 return -1; 3.188 } 3.189 3.190 -void build_pagetable(unsigned long *start_pfn, unsigned long *max_pfn) 3.191 +/* 3.192 + * Build the initial pagetable. 3.193 + */ 3.194 +static void build_pagetable(unsigned long *start_pfn, unsigned long *max_pfn) 3.195 { 3.196 unsigned long start_address, end_address; 3.197 unsigned long pfn_to_map, pt_pfn = *start_pfn; 3.198 static mmu_update_t mmu_updates[L1_PAGETABLE_ENTRIES + 1]; 3.199 pgentry_t *tab = (pgentry_t *)start_info.pt_base, page; 3.200 - unsigned long mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base)); 3.201 + unsigned long pt_mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base)); 3.202 unsigned long offset; 3.203 int count = 0; 3.204 + int rc; 3.205 3.206 - pfn_to_map = (start_info.nr_pt_frames - NOT_L1_FRAMES) * L1_PAGETABLE_ENTRIES; 3.207 + pfn_to_map = 3.208 + (start_info.nr_pt_frames - NOT_L1_FRAMES) * L1_PAGETABLE_ENTRIES; 3.209 3.210 - if (*max_pfn >= virt_to_pfn(HYPERVISOR_VIRT_START)) 3.211 + if ( *max_pfn >= virt_to_pfn(HYPERVISOR_VIRT_START) ) 3.212 { 3.213 printk("WARNING: Mini-OS trying to use Xen virtual space. " 3.214 "Truncating memory from %dMB to ", 3.215 - ((unsigned long)pfn_to_virt(*max_pfn) - (unsigned long)&_text)>>20); 3.216 + ((unsigned long)pfn_to_virt(*max_pfn) - 3.217 + (unsigned long)&_text)>>20); 3.218 *max_pfn = virt_to_pfn(HYPERVISOR_VIRT_START - PAGE_SIZE); 3.219 printk("%dMB\n", 3.220 - ((unsigned long)pfn_to_virt(*max_pfn) - (unsigned long)&_text)>>20); 3.221 + ((unsigned long)pfn_to_virt(*max_pfn) - 3.222 + (unsigned long)&_text)>>20); 3.223 } 3.224 3.225 start_address = (unsigned long)pfn_to_virt(pfn_to_map); 3.226 @@ -200,49 +221,53 @@ void build_pagetable(unsigned long *star 3.227 /* We worked out the virtual memory range to map, now mapping loop */ 3.228 printk("Mapping memory range 0x%lx - 0x%lx\n", start_address, end_address); 3.229 3.230 - while(start_address < end_address) 3.231 + while ( start_address < end_address ) 3.232 { 3.233 tab = (pgentry_t *)start_info.pt_base; 3.234 - mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base)); 3.235 + pt_mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base)); 3.236 3.237 #if defined(__x86_64__) 3.238 offset = l4_table_offset(start_address); 3.239 /* Need new L3 pt frame */ 3.240 - if(!(start_address & L3_MASK)) 3.241 - if(need_pt_frame(start_address, L3_FRAME)) 3.242 - new_pt_frame(&pt_pfn, mfn, offset, L3_FRAME); 3.243 + if ( !(start_address & L3_MASK) ) 3.244 + if ( need_pt_frame(start_address, L3_FRAME) ) 3.245 + new_pt_frame(&pt_pfn, pt_mfn, offset, L3_FRAME); 3.246 3.247 page = tab[offset]; 3.248 - mfn = pte_to_mfn(page); 3.249 - tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT); 3.250 + pt_mfn = pte_to_mfn(page); 3.251 + tab = to_virt(mfn_to_pfn(pt_mfn) << PAGE_SHIFT); 3.252 #endif 3.253 offset = l3_table_offset(start_address); 3.254 /* Need new L2 pt frame */ 3.255 - if(!(start_address & L2_MASK)) 3.256 - if(need_pt_frame(start_address, L2_FRAME)) 3.257 - new_pt_frame(&pt_pfn, mfn, offset, L2_FRAME); 3.258 + if ( !(start_address & L2_MASK) ) 3.259 + if ( need_pt_frame(start_address, L2_FRAME) ) 3.260 + new_pt_frame(&pt_pfn, pt_mfn, offset, L2_FRAME); 3.261 3.262 page = tab[offset]; 3.263 - mfn = pte_to_mfn(page); 3.264 - tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT); 3.265 + pt_mfn = pte_to_mfn(page); 3.266 + tab = to_virt(mfn_to_pfn(pt_mfn) << PAGE_SHIFT); 3.267 offset = l2_table_offset(start_address); 3.268 /* Need new L1 pt frame */ 3.269 - if(!(start_address & L1_MASK)) 3.270 - if(need_pt_frame(start_address, L1_FRAME)) 3.271 - new_pt_frame(&pt_pfn, mfn, offset, L1_FRAME); 3.272 + if ( !(start_address & L1_MASK) ) 3.273 + if ( need_pt_frame(start_address, L1_FRAME) ) 3.274 + new_pt_frame(&pt_pfn, pt_mfn, offset, L1_FRAME); 3.275 3.276 page = tab[offset]; 3.277 - mfn = pte_to_mfn(page); 3.278 + pt_mfn = pte_to_mfn(page); 3.279 offset = l1_table_offset(start_address); 3.280 3.281 - mmu_updates[count].ptr = ((pgentry_t)mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset; 3.282 - mmu_updates[count].val = (pgentry_t)pfn_to_mfn(pfn_to_map++) << PAGE_SHIFT | L1_PROT; 3.283 + mmu_updates[count].ptr = 3.284 + ((pgentry_t)pt_mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset; 3.285 + mmu_updates[count].val = 3.286 + (pgentry_t)pfn_to_mfn(pfn_to_map++) << PAGE_SHIFT | L1_PROT; 3.287 count++; 3.288 - if (count == L1_PAGETABLE_ENTRIES || pfn_to_map == *max_pfn) 3.289 + if ( count == L1_PAGETABLE_ENTRIES || pfn_to_map == *max_pfn ) 3.290 { 3.291 - if(HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF) < 0) 3.292 + rc = HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF); 3.293 + if ( rc < 0 ) 3.294 { 3.295 - printk("PTE could not be updated\n"); 3.296 + printk("ERROR: build_pagetable(): PTE could not be updated\n"); 3.297 + printk(" mmu_update failed with rc=%d\n", rc); 3.298 do_exit(); 3.299 } 3.300 count = 0; 3.301 @@ -253,20 +278,26 @@ void build_pagetable(unsigned long *star 3.302 *start_pfn = pt_pfn; 3.303 } 3.304 3.305 +/* 3.306 + * Mark portion of the address space read only. 3.307 + */ 3.308 extern void shared_info; 3.309 static void set_readonly(void *text, void *etext) 3.310 { 3.311 - unsigned long start_address = ((unsigned long) text + PAGE_SIZE - 1) & PAGE_MASK; 3.312 + unsigned long start_address = 3.313 + ((unsigned long) text + PAGE_SIZE - 1) & PAGE_MASK; 3.314 unsigned long end_address = (unsigned long) etext; 3.315 static mmu_update_t mmu_updates[L1_PAGETABLE_ENTRIES + 1]; 3.316 pgentry_t *tab = (pgentry_t *)start_info.pt_base, page; 3.317 unsigned long mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base)); 3.318 unsigned long offset; 3.319 int count = 0; 3.320 + int rc; 3.321 3.322 printk("setting %p-%p readonly\n", text, etext); 3.323 3.324 - while (start_address + PAGE_SIZE <= end_address) { 3.325 + while ( start_address + PAGE_SIZE <= end_address ) 3.326 + { 3.327 tab = (pgentry_t *)start_info.pt_base; 3.328 mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base)); 3.329 3.330 @@ -287,20 +318,25 @@ static void set_readonly(void *text, voi 3.331 3.332 offset = l1_table_offset(start_address); 3.333 3.334 - if (start_address != (unsigned long)&shared_info) { 3.335 - mmu_updates[count].ptr = ((pgentry_t)mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset; 3.336 - mmu_updates[count].val = tab[offset] & ~_PAGE_RW; 3.337 - count++; 3.338 - } else 3.339 - printk("skipped %p\n", start_address); 3.340 + if ( start_address != (unsigned long)&shared_info ) 3.341 + { 3.342 + mmu_updates[count].ptr = 3.343 + ((pgentry_t)mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset; 3.344 + mmu_updates[count].val = tab[offset] & ~_PAGE_RW; 3.345 + count++; 3.346 + } 3.347 + else 3.348 + printk("skipped %p\n", start_address); 3.349 3.350 start_address += PAGE_SIZE; 3.351 3.352 - if (count == L1_PAGETABLE_ENTRIES || start_address + PAGE_SIZE > end_address) 3.353 + if ( count == L1_PAGETABLE_ENTRIES || 3.354 + start_address + PAGE_SIZE > end_address ) 3.355 { 3.356 - if(HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF) < 0) 3.357 + rc = HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF); 3.358 + if ( rc < 0 ) 3.359 { 3.360 - printk("PTE could not be updated\n"); 3.361 + printk("ERROR: set_readonly(): PTE could not be updated\n"); 3.362 do_exit(); 3.363 } 3.364 count = 0; 3.365 @@ -308,41 +344,73 @@ static void set_readonly(void *text, voi 3.366 } 3.367 3.368 { 3.369 - mmuext_op_t op = { 3.370 - .cmd = MMUEXT_TLB_FLUSH_ALL, 3.371 - }; 3.372 - int count; 3.373 - HYPERVISOR_mmuext_op(&op, 1, &count, DOMID_SELF); 3.374 + mmuext_op_t op = { 3.375 + .cmd = MMUEXT_TLB_FLUSH_ALL, 3.376 + }; 3.377 + int count; 3.378 + HYPERVISOR_mmuext_op(&op, 1, &count, DOMID_SELF); 3.379 } 3.380 } 3.381 3.382 -void mem_test(unsigned long *start_add, unsigned long *end_add) 3.383 +/* 3.384 + * A useful mem testing function. Write the address to every address in the 3.385 + * range provided and read back the value. If verbose, print page walk to 3.386 + * some VA 3.387 + * 3.388 + * If we get MEM_TEST_MAX_ERRORS we might as well stop 3.389 + */ 3.390 +#define MEM_TEST_MAX_ERRORS 10 3.391 +int mem_test(unsigned long *start_va, unsigned long *end_va, int verbose) 3.392 { 3.393 unsigned long mask = 0x10000; 3.394 unsigned long *pointer; 3.395 - 3.396 - for(pointer = start_add; pointer < end_add; pointer++) 3.397 + int error_count = 0; 3.398 + 3.399 + /* write values and print page walks */ 3.400 + if ( verbose && (((unsigned long)start_va) & 0xfffff) ) 3.401 { 3.402 - if(!(((unsigned long)pointer) & 0xfffff)) 3.403 + printk("MemTest Start: 0x%lx\n", start_va); 3.404 + page_walk((unsigned long)start_va); 3.405 + } 3.406 + for ( pointer = start_va; pointer < end_va; pointer++ ) 3.407 + { 3.408 + if ( verbose && !(((unsigned long)pointer) & 0xfffff) ) 3.409 { 3.410 printk("Writing to %lx\n", pointer); 3.411 page_walk((unsigned long)pointer); 3.412 } 3.413 *pointer = (unsigned long)pointer & ~mask; 3.414 } 3.415 - 3.416 - for(pointer = start_add; pointer < end_add; pointer++) 3.417 + if ( verbose && (((unsigned long)end_va) & 0xfffff) ) 3.418 { 3.419 - if(((unsigned long)pointer & ~mask) != *pointer) 3.420 + printk("MemTest End: %lx\n", end_va-1); 3.421 + page_walk((unsigned long)end_va-1); 3.422 + } 3.423 + 3.424 + /* verify values */ 3.425 + for ( pointer = start_va; pointer < end_va; pointer++ ) 3.426 + { 3.427 + if ( ((unsigned long)pointer & ~mask) != *pointer ) 3.428 + { 3.429 printk("Read error at 0x%lx. Read: 0x%lx, should read 0x%lx\n", 3.430 - (unsigned long)pointer, 3.431 - *pointer, 3.432 - ((unsigned long)pointer & ~mask)); 3.433 + (unsigned long)pointer, *pointer, 3.434 + ((unsigned long)pointer & ~mask)); 3.435 + error_count++; 3.436 + if ( error_count >= MEM_TEST_MAX_ERRORS ) 3.437 + { 3.438 + printk("mem_test: too many errors\n"); 3.439 + return -1; 3.440 + } 3.441 + } 3.442 } 3.443 - 3.444 + return 0; 3.445 } 3.446 3.447 -static pgentry_t *get_pgt(unsigned long addr) 3.448 + 3.449 +/* 3.450 + * get the PTE for virtual address va if it exists. Otherwise NULL. 3.451 + */ 3.452 +static pgentry_t *get_pgt(unsigned long va) 3.453 { 3.454 unsigned long mfn; 3.455 pgentry_t *tab; 3.456 @@ -352,67 +420,78 @@ static pgentry_t *get_pgt(unsigned long 3.457 mfn = virt_to_mfn(start_info.pt_base); 3.458 3.459 #if defined(__x86_64__) 3.460 - offset = l4_table_offset(addr); 3.461 - if (!(tab[offset] & _PAGE_PRESENT)) 3.462 + offset = l4_table_offset(va); 3.463 + if ( !(tab[offset] & _PAGE_PRESENT) ) 3.464 return NULL; 3.465 mfn = pte_to_mfn(tab[offset]); 3.466 tab = mfn_to_virt(mfn); 3.467 #endif 3.468 - offset = l3_table_offset(addr); 3.469 - if (!(tab[offset] & _PAGE_PRESENT)) 3.470 + offset = l3_table_offset(va); 3.471 + if ( !(tab[offset] & _PAGE_PRESENT) ) 3.472 + return NULL; 3.473 + mfn = pte_to_mfn(tab[offset]); 3.474 + tab = mfn_to_virt(mfn); 3.475 + offset = l2_table_offset(va); 3.476 + if ( !(tab[offset] & _PAGE_PRESENT) ) 3.477 return NULL; 3.478 mfn = pte_to_mfn(tab[offset]); 3.479 tab = mfn_to_virt(mfn); 3.480 - offset = l2_table_offset(addr); 3.481 - if (!(tab[offset] & _PAGE_PRESENT)) 3.482 - return NULL; 3.483 - mfn = pte_to_mfn(tab[offset]); 3.484 - tab = mfn_to_virt(mfn); 3.485 - offset = l1_table_offset(addr); 3.486 + offset = l1_table_offset(va); 3.487 return &tab[offset]; 3.488 } 3.489 3.490 -pgentry_t *need_pgt(unsigned long addr) 3.491 + 3.492 +/* 3.493 + * return a valid PTE for a given virtual address. If PTE does not exist, 3.494 + * allocate page-table pages. 3.495 + */ 3.496 +pgentry_t *need_pgt(unsigned long va) 3.497 { 3.498 - unsigned long mfn; 3.499 + unsigned long pt_mfn; 3.500 pgentry_t *tab; 3.501 unsigned long pt_pfn; 3.502 unsigned offset; 3.503 3.504 tab = (pgentry_t *)start_info.pt_base; 3.505 - mfn = virt_to_mfn(start_info.pt_base); 3.506 + pt_mfn = virt_to_mfn(start_info.pt_base); 3.507 3.508 #if defined(__x86_64__) 3.509 - offset = l4_table_offset(addr); 3.510 - if (!(tab[offset] & _PAGE_PRESENT)) { 3.511 + offset = l4_table_offset(va); 3.512 + if ( !(tab[offset] & _PAGE_PRESENT) ) 3.513 + { 3.514 pt_pfn = virt_to_pfn(alloc_page()); 3.515 - new_pt_frame(&pt_pfn, mfn, offset, L3_FRAME); 3.516 + new_pt_frame(&pt_pfn, pt_mfn, offset, L3_FRAME); 3.517 } 3.518 ASSERT(tab[offset] & _PAGE_PRESENT); 3.519 - mfn = pte_to_mfn(tab[offset]); 3.520 - tab = mfn_to_virt(mfn); 3.521 + pt_mfn = pte_to_mfn(tab[offset]); 3.522 + tab = mfn_to_virt(pt_mfn); 3.523 #endif 3.524 - offset = l3_table_offset(addr); 3.525 - if (!(tab[offset] & _PAGE_PRESENT)) { 3.526 + offset = l3_table_offset(va); 3.527 + if ( !(tab[offset] & _PAGE_PRESENT) ) 3.528 + { 3.529 pt_pfn = virt_to_pfn(alloc_page()); 3.530 - new_pt_frame(&pt_pfn, mfn, offset, L2_FRAME); 3.531 + new_pt_frame(&pt_pfn, pt_mfn, offset, L2_FRAME); 3.532 } 3.533 ASSERT(tab[offset] & _PAGE_PRESENT); 3.534 - mfn = pte_to_mfn(tab[offset]); 3.535 - tab = mfn_to_virt(mfn); 3.536 - offset = l2_table_offset(addr); 3.537 - if (!(tab[offset] & _PAGE_PRESENT)) { 3.538 + pt_mfn = pte_to_mfn(tab[offset]); 3.539 + tab = mfn_to_virt(pt_mfn); 3.540 + offset = l2_table_offset(va); 3.541 + if ( !(tab[offset] & _PAGE_PRESENT) ) 3.542 + { 3.543 pt_pfn = virt_to_pfn(alloc_page()); 3.544 - new_pt_frame(&pt_pfn, mfn, offset, L1_FRAME); 3.545 + new_pt_frame(&pt_pfn, pt_mfn, offset, L1_FRAME); 3.546 } 3.547 ASSERT(tab[offset] & _PAGE_PRESENT); 3.548 - mfn = pte_to_mfn(tab[offset]); 3.549 - tab = mfn_to_virt(mfn); 3.550 + pt_mfn = pte_to_mfn(tab[offset]); 3.551 + tab = mfn_to_virt(pt_mfn); 3.552 3.553 - offset = l1_table_offset(addr); 3.554 + offset = l1_table_offset(va); 3.555 return &tab[offset]; 3.556 } 3.557 3.558 +/* 3.559 + * Reserve an area of virtual address space for mappings and Heap 3.560 + */ 3.561 static unsigned long demand_map_area_start; 3.562 #ifdef __x86_64__ 3.563 #define DEMAND_MAP_PAGES ((128ULL << 30) / PAGE_SIZE) 3.564 @@ -437,7 +516,8 @@ void arch_init_demand_mapping_area(unsig 3.565 3.566 demand_map_area_start = (unsigned long) pfn_to_virt(cur_pfn); 3.567 cur_pfn += DEMAND_MAP_PAGES; 3.568 - printk("Demand map pfns at %lx-%lx.\n", demand_map_area_start, pfn_to_virt(cur_pfn)); 3.569 + printk("Demand map pfns at %lx-%lx.\n", 3.570 + demand_map_area_start, pfn_to_virt(cur_pfn)); 3.571 3.572 #ifdef HAVE_LIBC 3.573 cur_pfn++; 3.574 @@ -448,104 +528,204 @@ void arch_init_demand_mapping_area(unsig 3.575 #endif 3.576 } 3.577 3.578 +unsigned long allocate_ondemand(unsigned long n, unsigned long alignment) 3.579 +{ 3.580 + unsigned long x; 3.581 + unsigned long y = 0; 3.582 + 3.583 + /* Find a properly aligned run of n contiguous frames */ 3.584 + for ( x = 0; 3.585 + x <= DEMAND_MAP_PAGES - n; 3.586 + x = (x + y + 1 + alignment - 1) & ~(alignment - 1) ) 3.587 + { 3.588 + unsigned long addr = demand_map_area_start + x * PAGE_SIZE; 3.589 + pgentry_t *pgt = get_pgt(addr); 3.590 + for ( y = 0; y < n; y++, addr += PAGE_SIZE ) 3.591 + { 3.592 + if ( !(addr & L1_MASK) ) 3.593 + pgt = get_pgt(addr); 3.594 + if ( pgt ) 3.595 + { 3.596 + if ( *pgt & _PAGE_PRESENT ) 3.597 + break; 3.598 + pgt++; 3.599 + } 3.600 + } 3.601 + if ( y == n ) 3.602 + break; 3.603 + } 3.604 + if ( y != n ) 3.605 + { 3.606 + printk("Failed to find %ld frames!\n", n); 3.607 + return 0; 3.608 + } 3.609 + return demand_map_area_start + x * PAGE_SIZE; 3.610 +} 3.611 + 3.612 +/* 3.613 + * Map an array of MFNs contiguously into virtual address space starting at 3.614 + * va. map f[i*stride]+i*increment for i in 0..n-1. 3.615 + */ 3.616 #define MAP_BATCH ((STACK_SIZE / 2) / sizeof(mmu_update_t)) 3.617 -void do_map_frames(unsigned long addr, 3.618 - unsigned long *f, unsigned long n, unsigned long stride, 3.619 - unsigned long increment, domid_t id, int may_fail, unsigned long prot) 3.620 +void do_map_frames(unsigned long va, 3.621 + unsigned long *mfns, unsigned long n, 3.622 + unsigned long stride, unsigned long incr, 3.623 + domid_t id, int may_fail, 3.624 + unsigned long prot) 3.625 { 3.626 pgentry_t *pgt = NULL; 3.627 unsigned long done = 0; 3.628 unsigned long i; 3.629 int rc; 3.630 3.631 - while (done < n) { 3.632 - unsigned long todo; 3.633 + if ( !mfns ) 3.634 + { 3.635 + printk("do_map_frames: no mfns supplied\n"); 3.636 + return; 3.637 + } 3.638 + DEBUG("va=%p n=0x%lx, mfns[0]=0x%lx stride=0x%lx incr=0x%lx prot=0x%lx\n", 3.639 + va, n, mfns[0], stride, incr, prot); 3.640 + 3.641 + while ( done < n ) 3.642 + { 3.643 + unsigned long todo; 3.644 3.645 - if (may_fail) 3.646 - todo = 1; 3.647 - else 3.648 - todo = n - done; 3.649 + if ( may_fail ) 3.650 + todo = 1; 3.651 + else 3.652 + todo = n - done; 3.653 3.654 - if (todo > MAP_BATCH) 3.655 - todo = MAP_BATCH; 3.656 + if ( todo > MAP_BATCH ) 3.657 + todo = MAP_BATCH; 3.658 3.659 - { 3.660 - mmu_update_t mmu_updates[todo]; 3.661 + { 3.662 + mmu_update_t mmu_updates[todo]; 3.663 3.664 - for (i = 0; i < todo; i++, addr += PAGE_SIZE, pgt++) { 3.665 - if (!pgt || !(addr & L1_MASK)) 3.666 - pgt = need_pgt(addr); 3.667 - mmu_updates[i].ptr = virt_to_mach(pgt); 3.668 - mmu_updates[i].val = ((pgentry_t)(f[(done + i) * stride] + (done + i) * increment) << PAGE_SHIFT) | prot; 3.669 - } 3.670 + for ( i = 0; i < todo; i++, va += PAGE_SIZE, pgt++) 3.671 + { 3.672 + if ( !pgt || !(va & L1_MASK) ) 3.673 + pgt = need_pgt(va); 3.674 + 3.675 + mmu_updates[i].ptr = virt_to_mach(pgt) | MMU_NORMAL_PT_UPDATE; 3.676 + mmu_updates[i].val = ((pgentry_t)(mfns[(done + i) * stride] + 3.677 + (done + i) * incr) 3.678 + << PAGE_SHIFT) | prot; 3.679 + } 3.680 3.681 - rc = HYPERVISOR_mmu_update(mmu_updates, todo, NULL, id); 3.682 - if (rc < 0) { 3.683 - if (may_fail) 3.684 - f[done * stride] |= 0xF0000000; 3.685 - else { 3.686 - printk("Map %ld (%lx, ...) at %p failed: %d.\n", todo, f[done * stride] + done * increment, addr, rc); 3.687 + rc = HYPERVISOR_mmu_update(mmu_updates, todo, NULL, id); 3.688 + if ( rc < 0 ) 3.689 + { 3.690 + if (may_fail) 3.691 + mfns[done * stride] |= 0xF0000000; 3.692 + else { 3.693 + printk("Map %ld (%lx, ...) at %p failed: %d.\n", 3.694 + todo, mfns[done * stride] + done * incr, va, rc); 3.695 do_exit(); 3.696 - } 3.697 - } 3.698 - } 3.699 - 3.700 - done += todo; 3.701 + } 3.702 + } 3.703 + } 3.704 + done += todo; 3.705 } 3.706 } 3.707 3.708 -unsigned long allocate_ondemand(unsigned long n, unsigned long alignment) 3.709 +/* 3.710 + * Map an array of MFNs contiguous into virtual address space. Virtual 3.711 + * addresses are allocated from the on demand area. 3.712 + */ 3.713 +void *map_frames_ex(unsigned long *mfns, unsigned long n, 3.714 + unsigned long stride, unsigned long incr, 3.715 + unsigned long alignment, 3.716 + domid_t id, int may_fail, unsigned long prot) 3.717 { 3.718 - unsigned long x; 3.719 - unsigned long y = 0; 3.720 + unsigned long va = allocate_ondemand(n, alignment); 3.721 + 3.722 + if ( !va ) 3.723 + return NULL; 3.724 + 3.725 + do_map_frames(va, mfns, n, stride, incr, id, may_fail, prot); 3.726 + 3.727 + return (void *)va; 3.728 +} 3.729 + 3.730 +/* 3.731 + * Unmap nun_frames frames mapped at virtual address va. 3.732 + */ 3.733 +#define UNMAP_BATCH ((STACK_SIZE / 2) / sizeof(multicall_entry_t)) 3.734 +int unmap_frames(unsigned long va, unsigned long num_frames) 3.735 +{ 3.736 + int n = UNMAP_BATCH; 3.737 + multicall_entry_t call[n]; 3.738 + int ret; 3.739 + int i; 3.740 + 3.741 + ASSERT(!((unsigned long)va & ~PAGE_MASK)); 3.742 3.743 - /* Find a properly aligned run of n contiguous frames */ 3.744 - for (x = 0; x <= DEMAND_MAP_PAGES - n; x = (x + y + 1 + alignment - 1) & ~(alignment - 1)) { 3.745 - unsigned long addr = demand_map_area_start + x * PAGE_SIZE; 3.746 - pgentry_t *pgt = get_pgt(addr); 3.747 - for (y = 0; y < n; y++, addr += PAGE_SIZE) { 3.748 - if (!(addr & L1_MASK)) 3.749 - pgt = get_pgt(addr); 3.750 - if (pgt) { 3.751 - if (*pgt & _PAGE_PRESENT) 3.752 - break; 3.753 - pgt++; 3.754 + DEBUG("va=%p, num=0x%lx\n", va, num_frames); 3.755 + 3.756 + while ( num_frames ) { 3.757 + if ( n > num_frames ) 3.758 + n = num_frames; 3.759 + 3.760 + for ( i = 0; i < n; i++ ) 3.761 + { 3.762 + int arg = 0; 3.763 + /* simply update the PTE for the VA and invalidate TLB */ 3.764 + call[i].op = __HYPERVISOR_update_va_mapping; 3.765 + call[i].args[arg++] = va; 3.766 + call[i].args[arg++] = 0; 3.767 +#ifdef __i386__ 3.768 + call[i].args[arg++] = 0; 3.769 +#endif 3.770 + call[i].args[arg++] = UVMF_INVLPG; 3.771 + 3.772 + va += PAGE_SIZE; 3.773 + } 3.774 + 3.775 + ret = HYPERVISOR_multicall(call, n); 3.776 + if ( ret ) 3.777 + { 3.778 + printk("update_va_mapping hypercall failed with rc=%d.\n", ret); 3.779 + return -ret; 3.780 + } 3.781 + 3.782 + for ( i = 0; i < n; i++ ) 3.783 + { 3.784 + if ( call[i].result ) 3.785 + { 3.786 + printk("update_va_mapping failed for with rc=%d.\n", ret); 3.787 + return -(call[i].result); 3.788 } 3.789 } 3.790 - if (y == n) 3.791 - break; 3.792 + num_frames -= n; 3.793 } 3.794 - if (y != n) { 3.795 - printk("Failed to find %ld frames!\n", n); 3.796 - return 0; 3.797 - } 3.798 - return demand_map_area_start + x * PAGE_SIZE; 3.799 + return 0; 3.800 } 3.801 3.802 -void *map_frames_ex(unsigned long *f, unsigned long n, unsigned long stride, 3.803 - unsigned long increment, unsigned long alignment, domid_t id, 3.804 - int may_fail, unsigned long prot) 3.805 +/* 3.806 + * Check if a given MFN refers to real memory 3.807 + */ 3.808 +static long system_ram_end_mfn; 3.809 +int mfn_is_ram(unsigned long mfn) 3.810 { 3.811 - unsigned long addr = allocate_ondemand(n, alignment); 3.812 - 3.813 - if (!addr) 3.814 - return NULL; 3.815 - 3.816 - /* Found it at x. Map it in. */ 3.817 - do_map_frames(addr, f, n, stride, increment, id, may_fail, prot); 3.818 - 3.819 - return (void *)addr; 3.820 + /* very crude check if a given MFN is memory or not. Probably should 3.821 + * make this a little more sophisticated ;) */ 3.822 + return (mfn <= system_ram_end_mfn) ? 1 : 0; 3.823 } 3.824 3.825 + 3.826 +/* 3.827 + * Clear some of the bootstrap memory 3.828 + */ 3.829 static void clear_bootstrap(void) 3.830 { 3.831 pte_t nullpte = { }; 3.832 + int rc; 3.833 3.834 /* Use first page as the CoW zero page */ 3.835 memset(&_text, 0, PAGE_SIZE); 3.836 mfn_zero = virt_to_mfn((unsigned long) &_text); 3.837 - if (HYPERVISOR_update_va_mapping(0, nullpte, UVMF_INVLPG)) 3.838 - printk("Unable to unmap NULL page\n"); 3.839 + if ( (rc = HYPERVISOR_update_va_mapping(0, nullpte, UVMF_INVLPG)) ) 3.840 + printk("Unable to unmap NULL page. rc=%d\n", rc); 3.841 } 3.842 3.843 void arch_init_p2m(unsigned long max_pfn) 3.844 @@ -570,19 +750,19 @@ void arch_init_p2m(unsigned long max_pfn 3.845 unsigned long pfn; 3.846 3.847 l3_list = (unsigned long *)alloc_page(); 3.848 - for(pfn=0; pfn<max_pfn; pfn++) 3.849 + for ( pfn=0; pfn<max_pfn; pfn++ ) 3.850 { 3.851 - if(!(pfn % (L1_P2M_ENTRIES * L2_P2M_ENTRIES))) 3.852 + if ( !(pfn % (L1_P2M_ENTRIES * L2_P2M_ENTRIES)) ) 3.853 { 3.854 l2_list = (unsigned long*)alloc_page(); 3.855 - if((pfn >> L3_P2M_SHIFT) > 0) 3.856 + if ( (pfn >> L3_P2M_SHIFT) > 0 ) 3.857 { 3.858 printk("Error: Too many pfns.\n"); 3.859 do_exit(); 3.860 } 3.861 l3_list[(pfn >> L2_P2M_SHIFT)] = virt_to_mfn(l2_list); 3.862 } 3.863 - if(!(pfn % (L1_P2M_ENTRIES))) 3.864 + if ( !(pfn % (L1_P2M_ENTRIES)) ) 3.865 { 3.866 l1_list = (unsigned long*)alloc_page(); 3.867 l2_list[(pfn >> L1_P2M_SHIFT) & L2_P2M_MASK] = 3.868 @@ -601,30 +781,34 @@ void arch_init_mm(unsigned long* start_p 3.869 3.870 unsigned long start_pfn, max_pfn, virt_pfns; 3.871 3.872 - printk(" _text: %p\n", &_text); 3.873 - printk(" _etext: %p\n", &_etext); 3.874 - printk(" _erodata: %p\n", &_erodata); 3.875 - printk(" _edata: %p\n", &_edata); 3.876 - printk(" stack start: %p\n", stack); 3.877 - printk(" _end: %p\n", &_end); 3.878 + printk(" _text: %p(VA)\n", &_text); 3.879 + printk(" _etext: %p(VA)\n", &_etext); 3.880 + printk(" _erodata: %p(VA)\n", &_erodata); 3.881 + printk(" _edata: %p(VA)\n", &_edata); 3.882 + printk("stack start: %p(VA)\n", stack); 3.883 + printk(" _end: %p(VA)\n", &_end); 3.884 3.885 /* First page follows page table pages and 3 more pages (store page etc) */ 3.886 start_pfn = PFN_UP(to_phys(start_info.pt_base)) + 3.887 - start_info.nr_pt_frames + 3; 3.888 + start_info.nr_pt_frames + 3; 3.889 max_pfn = start_info.nr_pages; 3.890 3.891 /* We need room for demand mapping and heap, clip available memory */ 3.892 virt_pfns = DEMAND_MAP_PAGES + HEAP_PAGES; 3.893 - if (max_pfn + virt_pfns + 1 < max_pfn) 3.894 + if ( max_pfn + virt_pfns + 1 < max_pfn ) 3.895 max_pfn = -(virt_pfns + 1); 3.896 3.897 - printk(" start_pfn: %lx\n", start_pfn); 3.898 - printk(" max_pfn: %lx\n", max_pfn); 3.899 + printk(" start_pfn: %lx\n", start_pfn); 3.900 + printk(" max_pfn: %lx\n", max_pfn); 3.901 3.902 build_pagetable(&start_pfn, &max_pfn); 3.903 clear_bootstrap(); 3.904 set_readonly(&_text, &_erodata); 3.905 3.906 + /* get the number of physical pages the system has. Used to check for 3.907 + * system memory. */ 3.908 + system_ram_end_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL); 3.909 + 3.910 *start_pfn_p = start_pfn; 3.911 *max_pfn_p = max_pfn; 3.912 }
4.1 --- a/extras/mini-os/arch/x86/setup.c Mon Mar 02 18:26:56 2009 +0900 4.2 +++ b/extras/mini-os/arch/x86/setup.c Fri Mar 06 12:22:22 2009 +0900 4.3 @@ -63,10 +63,12 @@ void failsafe_callback(void); 4.4 static 4.5 shared_info_t *map_shared_info(unsigned long pa) 4.6 { 4.7 - if ( HYPERVISOR_update_va_mapping( 4.8 - (unsigned long)shared_info, __pte(pa | 7), UVMF_INVLPG) ) 4.9 + int rc; 4.10 + 4.11 + if ( (rc = HYPERVISOR_update_va_mapping( 4.12 + (unsigned long)shared_info, __pte(pa | 7), UVMF_INVLPG)) ) 4.13 { 4.14 - printk("Failed to map shared_info!!\n"); 4.15 + printk("Failed to map shared_info!! rc=%d\n", rc); 4.16 do_exit(); 4.17 } 4.18 return (shared_info_t *)shared_info;
5.1 --- a/extras/mini-os/events.c Mon Mar 02 18:26:56 2009 +0900 5.2 +++ b/extras/mini-os/events.c Fri Mar 06 12:22:22 2009 +0900 5.3 @@ -42,19 +42,23 @@ void unbind_all_ports(void) 5.4 int cpu = 0; 5.5 shared_info_t *s = HYPERVISOR_shared_info; 5.6 vcpu_info_t *vcpu_info = &s->vcpu_info[cpu]; 5.7 + int rc; 5.8 5.9 - for (i = 0; i < NR_EVS; i++) 5.10 + for ( i = 0; i < NR_EVS; i++ ) 5.11 { 5.12 - if (i == start_info.console.domU.evtchn || 5.13 - i == start_info.store_evtchn) 5.14 + if ( i == start_info.console.domU.evtchn || 5.15 + i == start_info.store_evtchn) 5.16 continue; 5.17 - if (test_and_clear_bit(i, bound_ports)) 5.18 + 5.19 + if ( test_and_clear_bit(i, bound_ports) ) 5.20 { 5.21 struct evtchn_close close; 5.22 printk("port %d still bound!\n", i); 5.23 mask_evtchn(i); 5.24 close.port = i; 5.25 - HYPERVISOR_event_channel_op(EVTCHNOP_close, &close); 5.26 + rc = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close); 5.27 + if ( rc ) 5.28 + printk("WARN: close_port %s failed rc=%d. ignored\n", i, rc); 5.29 clear_evtchn(i); 5.30 } 5.31 } 5.32 @@ -71,8 +75,9 @@ int do_event(evtchn_port_t port, struct 5.33 5.34 clear_evtchn(port); 5.35 5.36 - if (port >= NR_EVS) { 5.37 - printk("Port number too large: %d\n", port); 5.38 + if ( port >= NR_EVS ) 5.39 + { 5.40 + printk("WARN: do_event(): Port number too large: %d\n", port); 5.41 return 1; 5.42 } 5.43 5.44 @@ -89,9 +94,9 @@ int do_event(evtchn_port_t port, struct 5.45 evtchn_port_t bind_evtchn(evtchn_port_t port, evtchn_handler_t handler, 5.46 void *data) 5.47 { 5.48 - if(ev_actions[port].handler != default_handler) 5.49 + if ( ev_actions[port].handler != default_handler ) 5.50 printk("WARN: Handler for port %d already registered, replacing\n", 5.51 - port); 5.52 + port); 5.53 5.54 ev_actions[port].data = data; 5.55 wmb(); 5.56 @@ -104,8 +109,9 @@ evtchn_port_t bind_evtchn(evtchn_port_t 5.57 void unbind_evtchn(evtchn_port_t port ) 5.58 { 5.59 struct evtchn_close close; 5.60 + int rc; 5.61 5.62 - if (ev_actions[port].handler == default_handler) 5.63 + if ( ev_actions[port].handler == default_handler ) 5.64 printk("WARN: No handler for port %d when unbinding\n", port); 5.65 mask_evtchn(port); 5.66 clear_evtchn(port); 5.67 @@ -116,37 +122,43 @@ void unbind_evtchn(evtchn_port_t port ) 5.68 clear_bit(port, bound_ports); 5.69 5.70 close.port = port; 5.71 - HYPERVISOR_event_channel_op(EVTCHNOP_close, &close); 5.72 + rc = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close); 5.73 + if ( rc ) 5.74 + printk("WARN: close_port %s failed rc=%d. ignored\n", port, rc); 5.75 + 5.76 } 5.77 5.78 evtchn_port_t bind_virq(uint32_t virq, evtchn_handler_t handler, void *data) 5.79 { 5.80 evtchn_bind_virq_t op; 5.81 + int rc; 5.82 5.83 /* Try to bind the virq to a port */ 5.84 op.virq = virq; 5.85 op.vcpu = smp_processor_id(); 5.86 5.87 - if ( HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &op) != 0 ) 5.88 + if ( (rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &op)) != 0 ) 5.89 { 5.90 - printk("Failed to bind virtual IRQ %d\n", virq); 5.91 + printk("Failed to bind virtual IRQ %d with rc=%d\n", virq, rc); 5.92 return -1; 5.93 } 5.94 bind_evtchn(op.port, handler, data); 5.95 return op.port; 5.96 } 5.97 5.98 -evtchn_port_t bind_pirq(uint32_t pirq, int will_share, evtchn_handler_t handler, void *data) 5.99 +evtchn_port_t bind_pirq(uint32_t pirq, int will_share, 5.100 + evtchn_handler_t handler, void *data) 5.101 { 5.102 evtchn_bind_pirq_t op; 5.103 + int rc; 5.104 5.105 /* Try to bind the pirq to a port */ 5.106 op.pirq = pirq; 5.107 op.flags = will_share ? BIND_PIRQ__WILL_SHARE : 0; 5.108 5.109 - if ( HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &op) != 0 ) 5.110 + if ( (rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &op)) != 0 ) 5.111 { 5.112 - printk("Failed to bind physical IRQ %d\n", pirq); 5.113 + printk("Failed to bind physical IRQ %d with rc=%d\n", pirq, rc); 5.114 return -1; 5.115 } 5.116 bind_evtchn(op.port, handler, data); 5.117 @@ -173,7 +185,8 @@ void init_events(void) 5.118 asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0)); 5.119 wrmsrl(0xc0000101, &cpu0_pda); /* 0xc0000101 is MSR_GS_BASE */ 5.120 cpu0_pda.irqcount = -1; 5.121 - cpu0_pda.irqstackptr = (void*) (((unsigned long)irqstack + 2 * STACK_SIZE) & ~(STACK_SIZE - 1)); 5.122 + cpu0_pda.irqstackptr = (void*) (((unsigned long)irqstack + 2 * STACK_SIZE) 5.123 + & ~(STACK_SIZE - 1)); 5.124 #endif 5.125 /* initialize event handler */ 5.126 for ( i = 0; i < NR_EVS; i++ ) 5.127 @@ -207,15 +220,19 @@ void default_handler(evtchn_port_t port, 5.128 int evtchn_alloc_unbound(domid_t pal, evtchn_handler_t handler, 5.129 void *data, evtchn_port_t *port) 5.130 { 5.131 - int err; 5.132 + int rc; 5.133 + 5.134 evtchn_alloc_unbound_t op; 5.135 op.dom = DOMID_SELF; 5.136 op.remote_dom = pal; 5.137 - err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, &op); 5.138 - if (err) 5.139 - return err; 5.140 + rc = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, &op); 5.141 + if ( rc ) 5.142 + { 5.143 + printk("ERROR: alloc_unbound failed with rc=%d", rc); 5.144 + return rc; 5.145 + } 5.146 *port = bind_evtchn(op.port, handler, data); 5.147 - return err; 5.148 + return rc; 5.149 } 5.150 5.151 /* Connect to a port so as to allow the exchange of notifications with 5.152 @@ -225,15 +242,28 @@ int evtchn_bind_interdomain(domid_t pal, 5.153 evtchn_handler_t handler, void *data, 5.154 evtchn_port_t *local_port) 5.155 { 5.156 - int err; 5.157 + int rc; 5.158 evtchn_port_t port; 5.159 evtchn_bind_interdomain_t op; 5.160 op.remote_dom = pal; 5.161 op.remote_port = remote_port; 5.162 - err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &op); 5.163 - if (err) 5.164 - return err; 5.165 + rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &op); 5.166 + if ( rc ) 5.167 + { 5.168 + printk("ERROR: bind_interdomain failed with rc=%d", rc); 5.169 + return rc; 5.170 + } 5.171 port = op.local_port; 5.172 *local_port = bind_evtchn(port, handler, data); 5.173 - return err; 5.174 + return rc; 5.175 } 5.176 + 5.177 +/* 5.178 + * Local variables: 5.179 + * mode: C 5.180 + * c-set-style: "BSD" 5.181 + * c-basic-offset: 4 5.182 + * tab-width: 4 5.183 + * indent-tabs-mode: nil 5.184 + * End: 5.185 + */
6.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 6.2 +++ b/extras/mini-os/include/ioremap.h Fri Mar 06 12:22:22 2009 +0900 6.3 @@ -0,0 +1,33 @@ 6.4 +/** 6.5 + * Copyright (C) 2009 Netronome Systems, Inc. All rights reserved. 6.6 + * 6.7 + * Permission is hereby granted, free of charge, to any person obtaining a copy 6.8 + * of this software and associated documentation files (the "Software"), to 6.9 + * deal in the Software without restriction, including without limitation the 6.10 + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 6.11 + * sell copies of the Software, and to permit persons to whom the Software is 6.12 + * furnished to do so, subject to the following conditions: 6.13 + * 6.14 + * The above copyright notice and this permission notice shall be included in 6.15 + * all copies or substantial portions of the Software. 6.16 + * 6.17 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 6.18 + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 6.19 + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 6.20 + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 6.21 + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 6.22 + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 6.23 + * DEALINGS IN THE SOFTWARE. 6.24 + */ 6.25 + 6.26 + 6.27 +#ifndef _IOREMAP_H_ 6.28 +#define _IOREMAP_H_ 6.29 + 6.30 +void *ioremap(unsigned long phys_addr, unsigned long size); 6.31 +void *ioremap_nocache(unsigned long phys_addr, unsigned long size); 6.32 +void iounmap(void *virt_addr, unsigned long size); 6.33 + 6.34 +#endif /* _IOREMAP_H_ */ 6.35 + 6.36 +/* -*- Mode:C; c-basic-offset:4; tab-width:4 indent-tabs-mode:nil -*- */
7.1 --- a/extras/mini-os/include/mm.h Mon Mar 02 18:26:56 2009 +0900 7.2 +++ b/extras/mini-os/include/mm.h Fri Mar 06 12:22:22 2009 +0900 7.3 @@ -71,6 +71,7 @@ void *map_frames_ex(unsigned long *f, un 7.4 void do_map_frames(unsigned long addr, 7.5 unsigned long *f, unsigned long n, unsigned long stride, 7.6 unsigned long increment, domid_t id, int may_fail, unsigned long prot); 7.7 +int unmap_frames(unsigned long va, unsigned long num_frames); 7.8 #ifdef HAVE_LIBC 7.9 extern unsigned long heap, brk, heap_mapped, heap_end; 7.10 #endif
8.1 --- a/extras/mini-os/include/x86/arch_mm.h Mon Mar 02 18:26:56 2009 +0900 8.2 +++ b/extras/mini-os/include/x86/arch_mm.h Fri Mar 06 12:22:22 2009 +0900 8.3 @@ -133,6 +133,10 @@ typedef unsigned long pgentry_t; 8.4 #define L4_PROT (_PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_DIRTY|_PAGE_USER) 8.5 #endif /* __i386__ || __x86_64__ */ 8.6 8.7 +/* flags for ioremap */ 8.8 +#define IO_PROT (L1_PROT) 8.9 +#define IO_PROT_NOCACHE (L1_PROT | _PAGE_PCD) 8.10 + 8.11 #include "arch_limits.h" 8.12 #define PAGE_SIZE __PAGE_SIZE 8.13 #define PAGE_SHIFT __PAGE_SHIFT 8.14 @@ -222,5 +226,6 @@ static __inline__ paddr_t machine_to_phy 8.15 #define do_map_zero(start, n) do_map_frames(start, &mfn_zero, n, 0, 0, DOMID_SELF, 0, L1_PROT_RO) 8.16 8.17 pgentry_t *need_pgt(unsigned long addr); 8.18 +int mfn_is_ram(unsigned long mfn); 8.19 8.20 #endif /* _ARCH_MM_H_ */
9.1 --- a/extras/mini-os/kernel.c Mon Mar 02 18:26:56 2009 +0900 9.2 +++ b/extras/mini-os/kernel.c Fri Mar 06 12:22:22 2009 +0900 9.3 @@ -490,14 +490,16 @@ void start_kernel(start_info_t *si) 9.4 9.5 /* print out some useful information */ 9.6 printk("Xen Minimal OS!\n"); 9.7 - printk("start_info: %p\n", si); 9.8 - printk(" nr_pages: %lu", si->nr_pages); 9.9 - printk(" shared_inf: %08lx\n", si->shared_info); 9.10 - printk(" pt_base: %p", (void *)si->pt_base); 9.11 - printk(" mod_start: 0x%lx\n", si->mod_start); 9.12 - printk(" mod_len: %lu\n", si->mod_len); 9.13 - printk(" flags: 0x%x\n", (unsigned int)si->flags); 9.14 - printk(" cmd_line: %s\n", 9.15 + printk(" start_info: %p(VA)\n", si); 9.16 + printk(" nr_pages: 0x%lx\n", si->nr_pages); 9.17 + printk(" shared_inf: 0x%08lx(MA)\n", si->shared_info); 9.18 + printk(" pt_base: %p(VA)\n", (void *)si->pt_base); 9.19 + printk("nr_pt_frames: 0x%lx\n", si->nr_pt_frames); 9.20 + printk(" mfn_list: %p(VA)\n", (void *)si->mfn_list); 9.21 + printk(" mod_start: 0x%lx(VA)\n", si->mod_start); 9.22 + printk(" mod_len: %lu\n", si->mod_len); 9.23 + printk(" flags: 0x%x\n", (unsigned int)si->flags); 9.24 + printk(" cmd_line: %s\n", 9.25 si->cmd_line ? (const char *)si->cmd_line : "NULL"); 9.26 9.27 /* Set up events. */
10.1 --- a/extras/mini-os/lib/sys.c Mon Mar 02 18:26:56 2009 +0900 10.2 +++ b/extras/mini-os/lib/sys.c Fri Mar 06 12:22:22 2009 +0900 10.3 @@ -1206,47 +1206,15 @@ void *mmap(void *start, size_t length, i 10.4 } else ASSERT(0); 10.5 } 10.6 10.7 -#define UNMAP_BATCH ((STACK_SIZE / 2) / sizeof(multicall_entry_t)) 10.8 int munmap(void *start, size_t length) 10.9 { 10.10 int total = length / PAGE_SIZE; 10.11 - ASSERT(!((unsigned long)start & ~PAGE_MASK)); 10.12 - while (total) { 10.13 - int n = UNMAP_BATCH; 10.14 - if (n > total) 10.15 - n = total; 10.16 - { 10.17 - int i; 10.18 - multicall_entry_t call[n]; 10.19 - unsigned char (*data)[PAGE_SIZE] = start; 10.20 - int ret; 10.21 + int ret; 10.22 10.23 - for (i = 0; i < n; i++) { 10.24 - int arg = 0; 10.25 - call[i].op = __HYPERVISOR_update_va_mapping; 10.26 - call[i].args[arg++] = (unsigned long) &data[i]; 10.27 - call[i].args[arg++] = 0; 10.28 -#ifdef __i386__ 10.29 - call[i].args[arg++] = 0; 10.30 -#endif 10.31 - call[i].args[arg++] = UVMF_INVLPG; 10.32 - } 10.33 - 10.34 - ret = HYPERVISOR_multicall(call, n); 10.35 - if (ret) { 10.36 - errno = -ret; 10.37 - return -1; 10.38 - } 10.39 - 10.40 - for (i = 0; i < n; i++) { 10.41 - if (call[i].result) { 10.42 - errno = call[i].result; 10.43 - return -1; 10.44 - } 10.45 - } 10.46 - } 10.47 - start = (char *)start + n * PAGE_SIZE; 10.48 - total -= n; 10.49 + ret = unmap_frames((unsigned long)start, (unsigned long)total); 10.50 + if (ret) { 10.51 + errno = ret; 10.52 + return -1; 10.53 } 10.54 return 0; 10.55 }
11.1 --- a/tools/blktap/drivers/block-qcow2.c Mon Mar 02 18:26:56 2009 +0900 11.2 +++ b/tools/blktap/drivers/block-qcow2.c Fri Mar 06 12:22:22 2009 +0900 11.3 @@ -1980,6 +1980,91 @@ static int qcow_validate_parent(struct d 11.4 return 0; 11.5 } 11.6 11.7 +int qcow2_create(const char *filename, uint64_t total_size, 11.8 + const char *backing_file, int flags) 11.9 +{ 11.10 + int fd, header_size, backing_filename_len, l1_size, i, shift, l2_bits; 11.11 + QCowHeader header; 11.12 + uint64_t tmp, offset; 11.13 + QCowCreateState s1, *s = &s1; 11.14 + 11.15 + memset(s, 0, sizeof(*s)); 11.16 + 11.17 + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); 11.18 + if (fd < 0) 11.19 + return -1; 11.20 + memset(&header, 0, sizeof(header)); 11.21 + header.magic = cpu_to_be32(QCOW_MAGIC); 11.22 + header.version = cpu_to_be32(QCOW_VERSION); 11.23 + header.size = cpu_to_be64(total_size * 512); 11.24 + header_size = sizeof(header); 11.25 + backing_filename_len = 0; 11.26 + if (backing_file) { 11.27 + header.backing_file_offset = cpu_to_be64(header_size); 11.28 + backing_filename_len = strlen(backing_file); 11.29 + header.backing_file_size = cpu_to_be32(backing_filename_len); 11.30 + header_size += backing_filename_len; 11.31 + } 11.32 + s->cluster_bits = 12; /* 4 KB clusters */ 11.33 + s->cluster_size = 1 << s->cluster_bits; 11.34 + header.cluster_bits = cpu_to_be32(s->cluster_bits); 11.35 + header_size = (header_size + 7) & ~7; 11.36 + if (flags & BLOCK_FLAG_ENCRYPT) { 11.37 + header.crypt_method = cpu_to_be32(QCOW_CRYPT_AES); 11.38 + } else { 11.39 + header.crypt_method = cpu_to_be32(QCOW_CRYPT_NONE); 11.40 + } 11.41 + l2_bits = s->cluster_bits - 3; 11.42 + shift = s->cluster_bits + l2_bits; 11.43 + l1_size = (((total_size * 512) + (1LL << shift) - 1) >> shift); 11.44 + offset = align_offset(header_size, s->cluster_size); 11.45 + s->l1_table_offset = offset; 11.46 + header.l1_table_offset = cpu_to_be64(s->l1_table_offset); 11.47 + header.l1_size = cpu_to_be32(l1_size); 11.48 + offset += align_offset(l1_size * sizeof(uint64_t), s->cluster_size); 11.49 + 11.50 + s->refcount_table = qemu_mallocz(s->cluster_size); 11.51 + s->refcount_block = qemu_mallocz(s->cluster_size); 11.52 + 11.53 + s->refcount_table_offset = offset; 11.54 + header.refcount_table_offset = cpu_to_be64(offset); 11.55 + header.refcount_table_clusters = cpu_to_be32(1); 11.56 + offset += s->cluster_size; 11.57 + 11.58 + s->refcount_table[0] = cpu_to_be64(offset); 11.59 + s->refcount_block_offset = offset; 11.60 + offset += s->cluster_size; 11.61 + 11.62 + /* update refcounts */ 11.63 + create_refcount_update(s, 0, header_size); 11.64 + create_refcount_update(s, s->l1_table_offset, l1_size * sizeof(uint64_t)); 11.65 + create_refcount_update(s, s->refcount_table_offset, s->cluster_size); 11.66 + create_refcount_update(s, s->refcount_block_offset, s->cluster_size); 11.67 + 11.68 + /* write all the data */ 11.69 + write(fd, &header, sizeof(header)); 11.70 + if (backing_file) { 11.71 + write(fd, backing_file, backing_filename_len); 11.72 + } 11.73 + lseek(fd, s->l1_table_offset, SEEK_SET); 11.74 + tmp = 0; 11.75 + for(i = 0;i < l1_size; i++) { 11.76 + write(fd, &tmp, sizeof(tmp)); 11.77 + } 11.78 + lseek(fd, s->refcount_table_offset, SEEK_SET); 11.79 + write(fd, s->refcount_table, s->cluster_size); 11.80 + 11.81 + lseek(fd, s->refcount_block_offset, SEEK_SET); 11.82 + write(fd, s->refcount_block, s->cluster_size); 11.83 + 11.84 + qemu_free(s->refcount_table); 11.85 + qemu_free(s->refcount_block); 11.86 + close(fd); 11.87 + return 0; 11.88 +} 11.89 + 11.90 + 11.91 + 11.92 struct tap_disk tapdisk_qcow2 = { 11.93 "qcow2", 11.94 sizeof(BDRVQcowState),
12.1 --- a/tools/blktap/drivers/qcow-create.c Mon Mar 02 18:26:56 2009 +0900 12.2 +++ b/tools/blktap/drivers/qcow-create.c Fri Mar 06 12:22:22 2009 +0900 12.3 @@ -52,7 +52,7 @@ static void help(void) 12.4 { 12.5 fprintf(stderr, "Qcow-utils: v1.0.0\n"); 12.6 fprintf(stderr, 12.7 - "usage: qcow-create [-h help] [-r reserve] <SIZE(MB)> <FILENAME> " 12.8 + "usage: qcow-create [-h help] [-r reserve] [-f format] <SIZE(MB)> <FILENAME> " 12.9 "[<BACKING_FILENAME>]\n"); 12.10 exit(-1); 12.11 } 12.12 @@ -61,11 +61,13 @@ int main(int argc, char *argv[]) 12.13 { 12.14 int ret = -1, c, backed = 0; 12.15 int sparse = 1; 12.16 + char *fmt = "qcow"; 12.17 uint64_t size; 12.18 char filename[MAX_NAME_LEN], bfilename[MAX_NAME_LEN]; 12.19 + char *tmpfile; 12.20 12.21 for(;;) { 12.22 - c = getopt(argc, argv, "hr"); 12.23 + c = getopt(argc, argv, "hrf"); 12.24 if (c == -1) 12.25 break; 12.26 switch(c) { 12.27 @@ -73,6 +75,9 @@ int main(int argc, char *argv[]) 12.28 help(); 12.29 exit(0); 12.30 break; 12.31 + case 'f': 12.32 + fmt = argv[optind++]; 12.33 + break; 12.34 case 'r': 12.35 sparse = 0; 12.36 break; 12.37 @@ -105,11 +110,16 @@ int main(int argc, char *argv[]) 12.38 } 12.39 } 12.40 12.41 - DFPRINTF("Creating file size %llu, name %s\n",(long long unsigned)size, filename); 12.42 - if (!backed) 12.43 - ret = qcow_create(filename,size,NULL,sparse); 12.44 - else 12.45 - ret = qcow_create(filename,size,bfilename,sparse); 12.46 + tmpfile = backed ? bfilename: NULL; 12.47 + if (!strcmp(fmt, "qcow")) { 12.48 + ret = qcow_create(filename, size, tmpfile, sparse); 12.49 + } else if(!strcmp(fmt, "qcow2")) { 12.50 + ret = qcow2_create(filename, size, tmpfile, sparse); 12.51 + } else { 12.52 + fprintf(stderr,"Unsupport format:%s\n", fmt); 12.53 + exit(-1); 12.54 + } 12.55 + DFPRINTF("Creating file size %llu, name %s\n",(long long unsigned)size, filename); 12.56 12.57 if (ret < 0) 12.58 DPRINTF("Unable to create QCOW file\n");
13.1 --- a/tools/blktap/drivers/tapdisk.h Mon Mar 02 18:26:56 2009 +0900 13.2 +++ b/tools/blktap/drivers/tapdisk.h Fri Mar 06 12:22:22 2009 +0900 13.3 @@ -266,4 +266,7 @@ typedef struct fd_list_entry { 13.4 13.5 int qcow_create(const char *filename, uint64_t total_size, 13.6 const char *backing_file, int flags); 13.7 + 13.8 +int qcow2_create(const char *filename, uint64_t total_size, 13.9 + const char *backing_file, int flags); 13.10 #endif /*TAPDISK_H_*/
14.1 --- a/tools/examples/xmexample.hvm Mon Mar 02 18:26:56 2009 +0900 14.2 +++ b/tools/examples/xmexample.hvm Fri Mar 06 12:22:22 2009 +0900 14.3 @@ -7,11 +7,11 @@ 14.4 #============================================================================ 14.5 14.6 import os, re 14.7 + 14.8 +arch_libdir = 'lib' 14.9 arch = os.uname()[4] 14.10 -if re.search('64', arch): 14.11 +if os.uname()[0] == 'Linux' and re.search('64', arch): 14.12 arch_libdir = 'lib64' 14.13 -else: 14.14 - arch_libdir = 'lib' 14.15 14.16 #---------------------------------------------------------------------------- 14.17 # Kernel image file.
15.1 --- a/tools/examples/xmexample.vti Mon Mar 02 18:26:56 2009 +0900 15.2 +++ b/tools/examples/xmexample.vti Fri Mar 06 12:22:22 2009 +0900 15.3 @@ -7,8 +7,10 @@ 15.4 #============================================================================ 15.5 15.6 import os, re 15.7 +arch_libdir = 'lib' 15.8 arch = os.uname()[4] 15.9 -arch_libdir = 'lib' 15.10 +if os.uname()[0] == 'Linux' and re.search('64', arch): 15.11 + arch_libdir = 'lib64' 15.12 15.13 #---------------------------------------------------------------------------- 15.14 # Kernel image file.
16.1 --- a/tools/firmware/Makefile Mon Mar 02 18:26:56 2009 +0900 16.2 +++ b/tools/firmware/Makefile Fri Mar 06 12:22:22 2009 +0900 16.3 @@ -15,13 +15,13 @@ SUBDIRS += hvmloader 16.4 .PHONY: all 16.5 all: 16.6 @set -e; if [ $$((`( bcc -v 2>&1 | grep version || echo 0.0.0 ) | cut -d' ' -f 3 | awk -F. '{ printf "0x%02x%02x%02x", $$1, $$2, $$3}'`)) -lt $$((0x00100e)) ] ; then \ 16.7 - echo "==========================================================="; \ 16.8 - echo "Require dev86 package version >= 0.16.14 to build firmware!"; \ 16.9 - echo "(visit http://www.cix.co.uk/~mayday for more information)"; \ 16.10 - echo "==========================================================="; \ 16.11 - else \ 16.12 + echo "==========================================================================="; \ 16.13 + echo "Require dev86 rpm or bin86 & bcc debs version >= 0.16.14 to build firmware!"; \ 16.14 + echo "(visit http://www.debath.co.uk/dev86/ for more information)"; \ 16.15 + echo "==========================================================================="; \ 16.16 + false ; \ 16.17 + fi 16.18 $(MAKE) subdirs-$@; \ 16.19 - fi 16.20 16.21 16.22 .PHONY: install
17.1 --- a/tools/hotplug/Linux/init.d/xend Mon Mar 02 18:26:56 2009 +0900 17.2 +++ b/tools/hotplug/Linux/init.d/xend Fri Mar 06 12:22:22 2009 +0900 17.3 @@ -39,11 +39,13 @@ function await_daemons_up 17.4 17.5 case "$1" in 17.6 start) 17.7 + touch /var/lock/subsys/xend 17.8 xend start 17.9 await_daemons_up 17.10 ;; 17.11 stop) 17.12 xend stop 17.13 + rm -f /var/lock/subsys/xend 17.14 ;; 17.15 status) 17.16 xend status
18.1 --- a/tools/hotplug/Linux/xen-hotplug-cleanup Mon Mar 02 18:26:56 2009 +0900 18.2 +++ b/tools/hotplug/Linux/xen-hotplug-cleanup Fri Mar 06 12:22:22 2009 +0900 18.3 @@ -11,6 +11,13 @@ dir=$(dirname "$0") 18.4 # This is pretty horrible, but there's not really a nicer way of solving this. 18.5 claim_lock "block" 18.6 18.7 +# split backend/DEVCLASS/VMID/DEVID on slashes 18.8 +path_array=( ${XENBUS_PATH//\// } ) 18.9 +# get /vm/UUID path 18.10 +vm=$(xenstore-read "/local/domain/${path_array[2]}/vm") 18.11 +# construct /vm/UUID/device/DEVCLASS/DEVID 18.12 +vm_dev="$vm/device/${path_array[1]}/${path_array[3]}" 18.13 + 18.14 # remove device frontend store entries 18.15 xenstore-rm -t \ 18.16 $(xenstore-read "$XENBUS_PATH/frontend" 2>/dev/null) 2>/dev/null || true 18.17 @@ -19,4 +26,7 @@ xenstore-rm -t \ 18.18 xenstore-rm -t "$XENBUS_PATH" 2>/dev/null || true 18.19 xenstore-rm -t "error/$XENBUS_PATH" 2>/dev/null || true 18.20 18.21 +# remove device path from /vm/UUID 18.22 +xenstore-rm -t "$vm_dev" 2>/dev/null || true 18.23 + 18.24 release_lock "block"
19.1 --- a/tools/libxc/xc_domain.c Mon Mar 02 18:26:56 2009 +0900 19.2 +++ b/tools/libxc/xc_domain.c Fri Mar 06 12:22:22 2009 +0900 19.3 @@ -920,7 +920,8 @@ int xc_domain_update_msi_irq( 19.4 uint32_t domid, 19.5 uint32_t gvec, 19.6 uint32_t pirq, 19.7 - uint32_t gflags) 19.8 + uint32_t gflags, 19.9 + uint64_t gtable) 19.10 { 19.11 int rc; 19.12 xen_domctl_bind_pt_irq_t *bind; 19.13 @@ -936,6 +937,7 @@ int xc_domain_update_msi_irq( 19.14 bind->machine_irq = pirq; 19.15 bind->u.msi.gvec = gvec; 19.16 bind->u.msi.gflags = gflags; 19.17 + bind->u.msi.gtable = gtable; 19.18 19.19 rc = do_domctl(xc_handle, &domctl); 19.20 return rc;
20.1 --- a/tools/libxc/xenctrl.h Mon Mar 02 18:26:56 2009 +0900 20.2 +++ b/tools/libxc/xenctrl.h Fri Mar 06 12:22:22 2009 +0900 20.3 @@ -1092,7 +1092,8 @@ int xc_domain_update_msi_irq( 20.4 uint32_t domid, 20.5 uint32_t gvec, 20.6 uint32_t pirq, 20.7 - uint32_t gflags); 20.8 + uint32_t gflags, 20.9 + uint64_t gtable); 20.10 20.11 int xc_domain_unbind_msi_irq(int xc_handle, 20.12 uint32_t domid,
21.1 --- a/tools/libxen/src/xen_common.c Mon Mar 02 18:26:56 2009 +0900 21.2 +++ b/tools/libxen/src/xen_common.c Fri Mar 06 12:22:22 2009 +0900 21.3 @@ -90,6 +90,8 @@ add_param(xmlNode *, const char *, const 21.4 static xmlNode * 21.5 add_param_struct(xmlNode *); 21.6 static xmlNode * 21.7 +add_param_array(xmlNode *); 21.8 +static xmlNode * 21.9 add_struct_array(xmlNode *, const char *); 21.10 static xmlNode * 21.11 add_nested_struct(xmlNode *, const char *); 21.12 @@ -1292,7 +1294,7 @@ make_body_add_type(enum abstract_typenam 21.13 const struct abstract_type *member_type = v->type->child; 21.14 arbitrary_set *set_val = v->u.struct_val; 21.15 abstract_value v; 21.16 - xmlNode *data_node = add_param_struct(params_node); 21.17 + xmlNode *data_node = add_param_array(params_node); 21.18 21.19 for (size_t i = 0; i < set_val->size; i++) 21.20 { 21.21 @@ -1611,6 +1613,16 @@ add_param_struct(xmlNode *params_node) 21.22 } 21.23 21.24 21.25 +static xmlNode * 21.26 +add_param_array(xmlNode *params_node) 21.27 +{ 21.28 + xmlNode *param_node = add_container(params_node, "param"); 21.29 + xmlNode *value_node = add_container(param_node, "value"); 21.30 + 21.31 + return xmlNewChild(value_node, NULL, BAD_CAST "array", NULL); 21.32 +} 21.33 + 21.34 + 21.35 static void 21.36 add_struct_member(xmlNode *struct_node, const char *name, const char *type, 21.37 const char *value)
22.1 --- a/tools/python/xen/lowlevel/acm/acm.c Mon Mar 02 18:26:56 2009 +0900 22.2 +++ b/tools/python/xen/lowlevel/acm/acm.c Fri Mar 06 12:22:22 2009 +0900 22.3 @@ -68,8 +68,6 @@ static void *__getssid(int domid, uint32 22.4 goto out2; 22.5 } else { 22.6 *buflen = SSID_BUFFER_SIZE; 22.7 - free(buf); 22.8 - buf = NULL; 22.9 goto out2; 22.10 } 22.11 out2:
23.1 --- a/tools/python/xen/util/blkif.py Mon Mar 02 18:26:56 2009 +0900 23.2 +++ b/tools/python/xen/util/blkif.py Fri Mar 06 12:22:22 2009 +0900 23.3 @@ -33,7 +33,7 @@ def blkdev_name_to_number(name): 23.4 major = scsi_major[((ord(n[7:8]) - ord('a') + 1) * 26 + (ord(n[8:9]) - ord('a'))) / 16 ] 23.5 minor = (((ord(n[7:8]) - ord('a') + 1 ) * 26 + (ord(n[8:9]) - ord('a'))) % 16) * 16 + int(n[9:] or 0) 23.6 devnum = major * 256 + minor 23.7 - elif re.match( '/dev/hd[a-t]([1-9]|[1-5][0-9]|6[0-3])?', n): 23.8 + elif re.match( '/dev/hd[a-t]([1-9]|[1-5][0-9]|6[0-3])?$', n): 23.9 ide_majors = [ 3, 22, 33, 34, 56, 57, 88, 89, 90, 91 ] 23.10 major = ide_majors[(ord(n[7:8]) - ord('a')) / 2] 23.11 minor = ((ord(n[7:8]) - ord('a')) % 2) * 64 + int(n[8:] or 0)
24.1 --- a/tools/python/xen/util/pci.py Mon Mar 02 18:26:56 2009 +0900 24.2 +++ b/tools/python/xen/util/pci.py Fri Mar 06 12:22:22 2009 +0900 24.3 @@ -66,9 +66,10 @@ PCI_EXP_DEVCTL_FLR = (0x1 << 15) 24.4 24.5 PCI_CAP_ID_PM = 0x01 24.6 PCI_PM_CTRL = 4 24.7 -PCI_PM_CTRL_NO_SOFT_RESET = 0x0004 24.8 +PCI_PM_CTRL_NO_SOFT_RESET = 0x0008 24.9 PCI_PM_CTRL_STATE_MASK = 0x0003 24.10 PCI_D3hot = 3 24.11 +PCI_D0hot = 0 24.12 24.13 VENDOR_INTEL = 0x8086 24.14 PCI_CAP_ID_VENDOR_SPECIFIC_CAP = 0x09 24.15 @@ -234,7 +235,7 @@ def find_all_devices_owned_by_pciback(): 24.16 return dev_list 24.17 24.18 def transform_list(target, src): 24.19 - ''' src: its element is pci string (Format: xxxx:xx:xx:x). 24.20 + ''' src: its element is pci string (Format: xxxx:xx:xx.x). 24.21 target: its element is pci string, or a list of pci string. 24.22 24.23 If all the elements in src are in target, we remove them from target 24.24 @@ -467,12 +468,12 @@ class PciDevice: 24.25 os.lseek(fd, PCI_CB_BRIDGE_CONTROL, 0) 24.26 br_cntl |= PCI_BRIDGE_CTL_BUS_RESET 24.27 os.write(fd, struct.pack('H', br_cntl)) 24.28 - time.sleep(0.200) 24.29 + time.sleep(0.100) 24.30 # De-assert Secondary Bus Reset 24.31 os.lseek(fd, PCI_CB_BRIDGE_CONTROL, 0) 24.32 br_cntl &= ~PCI_BRIDGE_CTL_BUS_RESET 24.33 os.write(fd, struct.pack('H', br_cntl)) 24.34 - time.sleep(0.200) 24.35 + time.sleep(0.100) 24.36 os.close(fd) 24.37 24.38 # Restore the config spaces 24.39 @@ -483,18 +484,25 @@ class PciDevice: 24.40 if pos == 0: 24.41 return False 24.42 24.43 + # No_Soft_Reset - When set 1, this bit indicates that 24.44 + # devices transitioning from D3hot to D0 because of 24.45 + # PowerState commands do not perform an internal reset. 24.46 + pm_ctl = self.pci_conf_read32(pos + PCI_PM_CTRL) 24.47 + if (pm_ctl & PCI_PM_CTRL_NO_SOFT_RESET) == 1: 24.48 + return False 24.49 + 24.50 (pci_list, cfg_list) = save_pci_conf_space([self.name]) 24.51 24.52 - # Enter D3hot without soft reset 24.53 - pm_ctl = self.pci_conf_read32(pos + PCI_PM_CTRL) 24.54 - pm_ctl |= PCI_PM_CTRL_NO_SOFT_RESET 24.55 + # Enter D3hot 24.56 pm_ctl &= ~PCI_PM_CTRL_STATE_MASK 24.57 pm_ctl |= PCI_D3hot 24.58 self.pci_conf_write32(pos + PCI_PM_CTRL, pm_ctl) 24.59 time.sleep(0.010) 24.60 24.61 # From D3hot to D0 24.62 - self.pci_conf_write32(pos + PCI_PM_CTRL, 0) 24.63 + pm_ctl &= ~PCI_PM_CTRL_STATE_MASK 24.64 + pm_ctl |= PCI_D0hot 24.65 + self.pci_conf_write32(pos + PCI_PM_CTRL, pm_ctl) 24.66 time.sleep(0.010) 24.67 24.68 restore_pci_conf_space((pci_list, cfg_list)) 24.69 @@ -516,7 +524,7 @@ class PciDevice: 24.70 (pci_list, cfg_list) = save_pci_conf_space([self.name]) 24.71 24.72 self.pci_conf_write8(pos + PCI_USB_FLRCTRL, 1) 24.73 - time.sleep(0.010) 24.74 + time.sleep(0.100) 24.75 24.76 restore_pci_conf_space((pci_list, cfg_list)) 24.77 24.78 @@ -636,7 +644,7 @@ class PciDevice: 24.79 self.dev_type = DEV_TYPE_PCI_BRIDGE 24.80 else: 24.81 creg = self.pci_conf_read16(pos + PCI_EXP_FLAGS) 24.82 - if ((creg & PCI_EXP_TYPE_PCI_BRIDGE) >> 4) == \ 24.83 + if ((creg & PCI_EXP_FLAGS_TYPE) >> 4) == \ 24.84 PCI_EXP_TYPE_PCI_BRIDGE: 24.85 self.dev_type = DEV_TYPE_PCI_BRIDGE 24.86 else: 24.87 @@ -701,7 +709,7 @@ class PciDevice: 24.88 pos = self.find_cap_offset(PCI_CAP_ID_EXP) 24.89 self.pci_conf_write32(pos + PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_FLR) 24.90 # We must sleep at least 100ms for the completion of FLR 24.91 - time.sleep(0.200) 24.92 + time.sleep(0.100) 24.93 restore_pci_conf_space((pci_list, cfg_list)) 24.94 else: 24.95 if self.bus == 0: 24.96 @@ -722,7 +730,7 @@ class PciDevice: 24.97 # We use Advanced Capability to do FLR. 24.98 pos = self.find_cap_offset(PCI_CAP_ID_AF) 24.99 self.pci_conf_write8(pos + PCI_AF_CTL, PCI_AF_CTL_FLR) 24.100 - time.sleep(0.200) 24.101 + time.sleep(0.100) 24.102 restore_pci_conf_space((pci_list, cfg_list)) 24.103 else: 24.104 if self.bus == 0:
25.1 --- a/tools/python/xen/xend/XendCheckpoint.py Mon Mar 02 18:26:56 2009 +0900 25.2 +++ b/tools/python/xen/xend/XendCheckpoint.py Fri Mar 06 12:22:22 2009 +0900 25.3 @@ -66,6 +66,13 @@ def insert_after(list, pred, value): 25.4 25.5 25.6 def save(fd, dominfo, network, live, dst, checkpoint=False, node=-1): 25.7 + try: 25.8 + if not os.path.isdir("/var/lib/xen"): 25.9 + os.makedirs("/var/lib/xen") 25.10 + except Exception, exn: 25.11 + log.exception("Can't create directory '/var/lib/xen'") 25.12 + raise XendError("Can't create directory '/var/lib/xen'") 25.13 + 25.14 write_exact(fd, SIGNATURE, "could not write guest state file: signature") 25.15 25.16 sxprep = dominfo.sxpr() 25.17 @@ -166,6 +173,13 @@ def save(fd, dominfo, network, live, dst 25.18 25.19 25.20 def restore(xd, fd, dominfo = None, paused = False, relocating = False): 25.21 + try: 25.22 + if not os.path.isdir("/var/lib/xen"): 25.23 + os.makedirs("/var/lib/xen") 25.24 + except Exception, exn: 25.25 + log.exception("Can't create directory '/var/lib/xen'") 25.26 + raise XendError("Can't create directory '/var/lib/xen'") 25.27 + 25.28 signature = read_exact(fd, len(SIGNATURE), 25.29 "not a valid guest state file: signature read") 25.30 if signature != SIGNATURE:
26.1 --- a/tools/python/xen/xend/XendConfig.py Mon Mar 02 18:26:56 2009 +0900 26.2 +++ b/tools/python/xen/xend/XendConfig.py Fri Mar 06 12:22:22 2009 +0900 26.3 @@ -216,6 +216,7 @@ XENAPI_CFG_TYPES = { 26.4 'cpuid_check' : dict, 26.5 'machine_address_size': int, 26.6 'suppress_spurious_page_faults': bool0, 26.7 + 's3_integrity' : int, 26.8 } 26.9 26.10 # List of legacy configuration keys that have no equivalent in the 26.11 @@ -1275,6 +1276,7 @@ class XendConfig(dict): 26.12 vscsi_dict = self.vscsi_convert_sxp_to_dict(config) 26.13 vscsi_devs = vscsi_dict['devs'] 26.14 vscsi_mode = vscsi_dict['feature-host'] 26.15 + vscsi_be = vscsi_dict.get('backend', None) 26.16 26.17 # create XenAPI DSCSI objects. 26.18 for vscsi_dev in vscsi_devs: 26.19 @@ -1294,6 +1296,8 @@ class XendConfig(dict): 26.20 'feature-host': vscsi_mode, 26.21 'uuid': vscsi_devs_uuid 26.22 } 26.23 + if vscsi_be is not None: 26.24 + vscsi_info['backend'] = vscsi_be 26.25 target['devices'][vscsi_devs_uuid] = (dev_type, vscsi_info) 26.26 log.debug("XendConfig: reading device: %s,%s" % \ 26.27 (vscsi_devs, vscsi_mode)) 26.28 @@ -1621,6 +1625,7 @@ class XendConfig(dict): 26.29 # [device, 26.30 # [vscsi, 26.31 # [feature-host, 0], 26.32 + # [backend, 0], 26.33 # [dev, 26.34 # [devid, 0], [p-devname, sdb], [p-dev, 1:0:0:1], 26.35 # [v-dev, 0:0:0:0], [state, 1] 26.36 @@ -1632,6 +1637,7 @@ class XendConfig(dict): 26.37 # ], 26.38 # [vscsi, 26.39 # [feature-host, 1], 26.40 + # [backend, 0], 26.41 # [dev, 26.42 # [devid, 1], [p-devname, sdg], [p-dev, 2:0:0:0], 26.43 # [v-dev, 1:0:0:0], [state, 1] 26.44 @@ -1653,6 +1659,7 @@ class XendConfig(dict): 26.45 # [device, 26.46 # [vscsi, 26.47 # [feature-host, 0], 26.48 + # [backend, 0], 26.49 # [dev, 26.50 # [devid, 0], [p-devname, sdd], [p-dev, 1:0:0:3], 26.51 # [v-dev, 0:0:0:2], [state, 1] 26.52 @@ -1668,7 +1675,7 @@ class XendConfig(dict): 26.53 # 26.54 # { devs: [ {devid: 0, p-devname: sdd, p-dev: 1:0:0:3, 26.55 # v-dev: 0:0:0:2, state: 1} ], 26.56 - # feature-host: 1 } 26.57 + # feature-host: 1 , backend: 0 } 26.58 26.59 dev_config = {} 26.60 26.61 @@ -1689,6 +1696,11 @@ class XendConfig(dict): 26.62 26.63 vscsi_mode = sxp.children(dev_sxp, 'feature-host')[0] 26.64 dev_config['feature-host'] = vscsi_mode[1] 26.65 + try: 26.66 + vscsi_be = sxp.children(dev_sxp, 'backend')[0] 26.67 + dev_config['backend'] = vscsi_be[1] 26.68 + except IndexError: 26.69 + pass 26.70 26.71 return dev_config 26.72 26.73 @@ -1803,6 +1815,7 @@ class XendConfig(dict): 26.74 vscsi_dict = self.vscsi_convert_sxp_to_dict(config) 26.75 vscsi_devs = vscsi_dict['devs'] 26.76 vscsi_mode = vscsi_dict['feature-host'] 26.77 + vscsi_be = vscsi_dict.get('backend', None) 26.78 26.79 # destroy existing XenAPI DSCSI objects 26.80 for dscsi_uuid in XendDSCSI.get_by_VM(self['uuid']): 26.81 @@ -1826,6 +1839,8 @@ class XendConfig(dict): 26.82 'feature-host': vscsi_mode, 26.83 'uuid': dev_uuid 26.84 } 26.85 + if vscsi_be is not None: 26.86 + vscsi_info['backend'] = vscsi_be 26.87 self['devices'][dev_uuid] = (dev_type, vscsi_info) 26.88 return True 26.89 26.90 @@ -1919,6 +1934,8 @@ class XendConfig(dict): 26.91 elif dev_type == 'vscsi': 26.92 sxpr = ['vscsi', ['uuid', dev_info['uuid']], 26.93 ['feature-host', dev_info['feature-host']]] 26.94 + if dev_info.has_key('backend'): 26.95 + sxpr.append(['backend', dev_info['backend']]) 26.96 for pci_dev_info in dev_info['devs']: 26.97 pci_dev_sxpr = ['dev'] 26.98 for opt, val in pci_dev_info.items():
27.1 --- a/tools/python/xen/xend/XendDomainInfo.py Mon Mar 02 18:26:56 2009 +0900 27.2 +++ b/tools/python/xen/xend/XendDomainInfo.py Fri Mar 06 12:22:22 2009 +0900 27.3 @@ -900,6 +900,11 @@ class XendDomainInfo: 27.4 new_dev_sxp = ['vscsi'] 27.5 cur_mode = sxp.children(cur_dev_sxp, 'feature-host')[0] 27.6 new_dev_sxp.append(cur_mode) 27.7 + try: 27.8 + cur_be = sxp.children(cur_dev_sxp, 'backend')[0] 27.9 + new_dev_sxp.append(cur_be) 27.10 + except IndexError: 27.11 + pass 27.12 27.13 for cur_dev in sxp.children(cur_dev_sxp, 'dev'): 27.14 if state == xenbusState['Closing']: 27.15 @@ -2207,12 +2212,17 @@ class XendDomainInfo: 27.16 if security.has_authorization(ssidref) == False: 27.17 raise VmError("VM is not authorized to run.") 27.18 27.19 + s3_integrity = 0 27.20 + if self.info.has_key('s3_integrity'): 27.21 + s3_integrity = self.info['s3_integrity'] 27.22 + flags = (int(hvm) << 0) | (int(hap) << 1) | (int(s3_integrity) << 2) 27.23 + 27.24 try: 27.25 self.domid = xc.domain_create( 27.26 domid = 0, 27.27 ssidref = ssidref, 27.28 handle = uuid.fromString(self.info['uuid']), 27.29 - flags = (int(hvm) << 0) | (int(hap) << 1), 27.30 + flags = flags, 27.31 target = self.info.target()) 27.32 except Exception, e: 27.33 # may get here if due to ACM the operation is not permitted
28.1 --- a/tools/python/xen/xend/server/DevController.py Mon Mar 02 18:26:56 2009 +0900 28.2 +++ b/tools/python/xen/xend/server/DevController.py Fri Mar 06 12:22:22 2009 +0900 28.3 @@ -235,8 +235,8 @@ class DevController: 28.4 xstransact.Remove(backpath) 28.5 xstransact.Remove(frontpath) 28.6 28.7 - # xstransact.Remove(self.devicePath()) ?? Below is the same ? 28.8 - self.vm._removeVm("device/%s/%d" % (self.deviceClass, dev)) 28.9 + # xstransact.Remove(self.devicePath()) ?? Below is the same ? 28.10 + self.vm._removeVm("device/%s/%d" % (self.deviceClass, dev)) 28.11 28.12 def configurations(self, transaction = None): 28.13 return map(lambda x: self.configuration(x, transaction), self.deviceIDs(transaction))
29.1 --- a/tools/python/xen/xm/create.py Mon Mar 02 18:26:56 2009 +0900 29.2 +++ b/tools/python/xen/xm/create.py Fri Mar 06 12:22:22 2009 +0900 29.3 @@ -579,6 +579,11 @@ gopts.var('hap', val='HAP', 29.4 use="""Hap status (0=hap is disabled; 29.5 1=hap is enabled.""") 29.6 29.7 +gopts.var('s3_integrity', val='TBOOT_MEMORY_PROTECT', 29.8 + fn=set_int, default=1, 29.9 + use="""Should domain memory integrity be verified during S3? 29.10 + (0=protection is disabled; 1=protection is enabled.""") 29.11 + 29.12 gopts.var('cpuid', val="IN[,SIN]:eax=EAX,ebx=EBX,ecx=ECX,edx=EDX", 29.13 fn=append_value, default=[], 29.14 use="""Cpuid description.""") 29.15 @@ -832,6 +837,10 @@ def configure_security(config, vals): 29.16 elif num > 1: 29.17 err("VM config error: Multiple access_control definitions!") 29.18 29.19 +def configure_mem_prot(config_image, vals): 29.20 + """Create the config for S3 memory integrity verification under tboot. 29.21 + """ 29.22 + config_image.append(['s3_integrity', vals.s3_integrity]) 29.23 29.24 def configure_vtpm(config_devs, vals): 29.25 """Create the config for virtual TPM interfaces. 29.26 @@ -964,6 +973,7 @@ def make_config(vals): 29.27 else: 29.28 config.append(['bootloader_args', '-q']) 29.29 config.append(['image', config_image]) 29.30 + configure_mem_prot(config, vals); 29.31 29.32 config_devs = [] 29.33 configure_disks(config_devs, vals)
30.1 --- a/tools/python/xen/xm/xenapi_create.py Mon Mar 02 18:26:56 2009 +0900 30.2 +++ b/tools/python/xen/xm/xenapi_create.py Fri Mar 06 12:22:22 2009 +0900 30.3 @@ -269,6 +269,8 @@ class xenapi_create: 30.4 vm.attributes["is_a_template"].value == 'true', 30.5 "auto_power_on": 30.6 vm.attributes["auto_power_on"].value == 'true', 30.7 + "s3_integrity": 30.8 + vm.attributes["s3_integrity"].value, 30.9 "memory_static_max": 30.10 get_child_node_attribute(vm, "memory", "static_max"), 30.11 "memory_static_min": 30.12 @@ -650,6 +652,8 @@ class sxp2xml: 30.13 = str(get_child_by_name(config, "vcpus", 1)) 30.14 vm.attributes["vcpus_at_startup"] \ 30.15 = str(get_child_by_name(config, "vcpus", 1)) 30.16 + vm.attributes["s3_integrity"] \ 30.17 + = str(get_child_by_name(config, "s3_integrity", 0)) 30.18 30.19 sec_data = get_child_by_name(config, "security") 30.20 if sec_data:
31.1 --- a/xen/Rules.mk Mon Mar 02 18:26:56 2009 +0900 31.2 +++ b/xen/Rules.mk Fri Mar 06 12:22:22 2009 +0900 31.3 @@ -38,6 +38,7 @@ ALL_OBJS-y += $(BASEDIR)/c 31.4 ALL_OBJS-y += $(BASEDIR)/drivers/built_in.o 31.5 ALL_OBJS-y += $(BASEDIR)/xsm/built_in.o 31.6 ALL_OBJS-y += $(BASEDIR)/arch/$(TARGET_ARCH)/built_in.o 31.7 +ALL_OBJS-$(x86) += $(BASEDIR)/crypto/built_in.o 31.8 31.9 CFLAGS-y += -g -D__XEN__ 31.10 CFLAGS-$(XSM_ENABLE) += -DXSM_ENABLE
32.1 --- a/xen/arch/ia64/vmx/vmx_interrupt.c Mon Mar 02 18:26:56 2009 +0900 32.2 +++ b/xen/arch/ia64/vmx/vmx_interrupt.c Fri Mar 06 12:22:22 2009 +0900 32.3 @@ -154,3 +154,14 @@ void hvm_isa_irq_deassert(struct domain 32.4 { 32.5 /* dummy */ 32.6 } 32.7 + 32.8 +int msixtbl_pt_register(struct domain *d, int pirq, uint64_t gtable) 32.9 +{ 32.10 + /* dummy */ 32.11 + return -ENOSYS; 32.12 +} 32.13 + 32.14 +void msixtbl_pt_unregister(struct domain *d, int pirq) 32.15 +{ 32.16 + /* dummy */ 32.17 +}
33.1 --- a/xen/arch/ia64/xen/mm.c Mon Mar 02 18:26:56 2009 +0900 33.2 +++ b/xen/arch/ia64/xen/mm.c Fri Mar 06 12:22:22 2009 +0900 33.3 @@ -3236,9 +3236,56 @@ int get_page_type(struct page_info *page 33.4 return 1; 33.5 } 33.6 33.7 -int page_is_conventional_ram(unsigned long mfn) 33.8 +int page_is_ram_type(unsigned long mfn, unsigned long type) 33.9 { 33.10 - return (efi_mem_type(pfn_to_paddr(mfn)) == EFI_CONVENTIONAL_MEMORY); 33.11 + u32 mem_type = efi_mem_type(pfn_to_paddr(mfn)); 33.12 + 33.13 + if (type & RAM_TYPE_CONVENTIONAL) 33.14 + { 33.15 + switch (mem_type) 33.16 + { 33.17 + case EFI_BOOT_SERVICES_CODE: 33.18 + case EFI_BOOT_SERVICES_DATA: 33.19 + case EFI_LOADER_CODE: 33.20 + case EFI_LOADER_DATA: 33.21 + case EFI_CONVENTIONAL_MEMORY: 33.22 + return 1; 33.23 + default: 33.24 + break; 33.25 + } 33.26 + } 33.27 + if (type & RAM_TYPE_RESERVED) 33.28 + { 33.29 + switch (mem_type) 33.30 + { 33.31 + case EFI_RUNTIME_SERVICES_CODE: 33.32 + case EFI_RUNTIME_SERVICES_DATA: 33.33 + case EFI_RESERVED_TYPE: 33.34 + case EFI_MEMORY_MAPPED_IO: 33.35 + case EFI_MEMORY_MAPPED_IO_PORT_SPACE: 33.36 + case EFI_PAL_CODE: 33.37 + return 1; 33.38 + default: 33.39 + break; 33.40 + } 33.41 + } 33.42 + if (type & RAM_TYPE_ACPI) 33.43 + { 33.44 + switch (mem_type) 33.45 + { 33.46 + case EFI_ACPI_RECLAIM_MEMORY: 33.47 + case EFI_ACPI_MEMORY_NVS: 33.48 + return 1; 33.49 + default: 33.50 + break; 33.51 + } 33.52 + } 33.53 + else if (type & RAM_TYPE_UNUSABLE) 33.54 + { 33.55 + return (mem_type == EFI_UNUSABLE_MEMORY); 33.56 + } 33.57 + 33.58 + return 0; 33.59 } 33.60 33.61
34.1 --- a/xen/arch/x86/acpi/cpu_idle.c Mon Mar 02 18:26:56 2009 +0900 34.2 +++ b/xen/arch/x86/acpi/cpu_idle.c Fri Mar 06 12:22:22 2009 +0900 34.3 @@ -55,6 +55,7 @@ static void (*lapic_timer_on)(void); 34.4 34.5 extern u32 pmtmr_ioport; 34.6 extern void (*pm_idle) (void); 34.7 +extern void (*dead_idle) (void); 34.8 34.9 static void (*pm_idle_save) (void) __read_mostly; 34.10 unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER - 1; 34.11 @@ -369,6 +370,43 @@ static void acpi_processor_idle(void) 34.12 cpuidle_current_governor->reflect(power); 34.13 } 34.14 34.15 +static void acpi_dead_idle(void) 34.16 +{ 34.17 + struct acpi_processor_power *power; 34.18 + struct acpi_processor_cx *cx; 34.19 + int unused; 34.20 + 34.21 + if ( (power = processor_powers[smp_processor_id()]) == NULL ) 34.22 + goto default_halt; 34.23 + 34.24 + if ( (cx = &power->states[power->count-1]) == NULL ) 34.25 + goto default_halt; 34.26 + 34.27 + for ( ; ; ) 34.28 + { 34.29 + if ( !power->flags.bm_check && cx->type == ACPI_STATE_C3 ) 34.30 + ACPI_FLUSH_CPU_CACHE(); 34.31 + 34.32 + switch ( cx->entry_method ) 34.33 + { 34.34 + case ACPI_CSTATE_EM_FFH: 34.35 + /* Not treat interrupt as break event */ 34.36 + mwait_idle_with_hints(cx->address, 0); 34.37 + break; 34.38 + case ACPI_CSTATE_EM_SYSIO: 34.39 + inb(cx->address); 34.40 + unused = inl(pmtmr_ioport); 34.41 + break; 34.42 + default: 34.43 + goto default_halt; 34.44 + } 34.45 + } 34.46 + 34.47 +default_halt: 34.48 + for ( ; ; ) 34.49 + halt(); 34.50 +} 34.51 + 34.52 static int init_cx_pminfo(struct acpi_processor_power *acpi_power) 34.53 { 34.54 int i; 34.55 @@ -740,6 +778,11 @@ long set_cx_pminfo(uint32_t cpu, struct 34.56 pm_idle_save = pm_idle; 34.57 pm_idle = acpi_processor_idle; 34.58 } 34.59 + 34.60 + if ( cpu_id == 0 ) 34.61 + { 34.62 + dead_idle = acpi_dead_idle; 34.63 + } 34.64 34.65 return 0; 34.66 }
35.1 --- a/xen/arch/x86/acpi/power.c Mon Mar 02 18:26:56 2009 +0900 35.2 +++ b/xen/arch/x86/acpi/power.c Fri Mar 06 12:22:22 2009 +0900 35.3 @@ -190,6 +190,7 @@ static int enter_state(u32 state) 35.4 case ACPI_STATE_S3: 35.5 do_suspend_lowlevel(); 35.6 system_reset_counter++; 35.7 + error = tboot_s3_resume(); 35.8 break; 35.9 case ACPI_STATE_S5: 35.10 acpi_enter_sleep_state(ACPI_STATE_S5); 35.11 @@ -206,7 +207,10 @@ static int enter_state(u32 state) 35.12 35.13 device_power_up(); 35.14 35.15 - printk(XENLOG_INFO "Finishing wakeup from ACPI S%d state.", state); 35.16 + printk(XENLOG_INFO "Finishing wakeup from ACPI S%d state.\n", state); 35.17 + 35.18 + if ( (state == ACPI_STATE_S3) && error ) 35.19 + panic("Memory integrity was lost on resume (%d)\n", error); 35.20 35.21 done: 35.22 spin_debug_enable(); 35.23 @@ -318,7 +322,7 @@ static void tboot_sleep(u8 sleep_state) 35.24 35.25 tboot_shutdown(shutdown_type); 35.26 } 35.27 - 35.28 + 35.29 /* System is really put into sleep state by this stub */ 35.30 acpi_status asmlinkage acpi_enter_sleep_state(u8 sleep_state) 35.31 {
36.1 --- a/xen/arch/x86/cpu/mcheck/mce_intel.c Mon Mar 02 18:26:56 2009 +0900 36.2 +++ b/xen/arch/x86/cpu/mcheck/mce_intel.c Fri Mar 06 12:22:22 2009 +0900 36.3 @@ -359,12 +359,6 @@ static int do_cmci_discover(int i) 36.4 return 0; 36.5 } 36.6 set_bit(i, __get_cpu_var(mce_banks_owned)); 36.7 - /* Clear Corected Error Counter field, make sure CMCI could 36.8 - * be triggered on the new owner 36.9 - */ 36.10 - msr = MSR_IA32_MC0_STATUS + 4 * i; 36.11 - rdmsrl(msr, val); 36.12 - wrmsrl(msr, val & ~MCi_STATUS_ERRCOUNT); 36.13 out: 36.14 clear_bit(i, __get_cpu_var(no_cmci_banks)); 36.15 return 1; 36.16 @@ -374,6 +368,7 @@ static void cmci_discover(void) 36.17 { 36.18 unsigned long flags; 36.19 int i; 36.20 + struct mc_info *mi = NULL; 36.21 36.22 printk(KERN_DEBUG "CMCI: find owner on CPU%d\n", smp_processor_id()); 36.23 36.24 @@ -385,6 +380,18 @@ static void cmci_discover(void) 36.25 36.26 spin_unlock_irqrestore(&cmci_discover_lock, flags); 36.27 36.28 + /* In case CMCI happended when do owner change. 36.29 + * If CMCI happened yet not processed immediately, 36.30 + * MCi_status (error_count bit 38~52) is not cleared, 36.31 + * the CMCI interrupt will never be triggered again. 36.32 + */ 36.33 + mi = machine_check_poll(MC_FLAG_CMCI); 36.34 + if (mi) { 36.35 + x86_mcinfo_dump(mi); 36.36 + if (dom0 && guest_enabled_event(dom0->vcpu[0], VIRQ_MCA)) 36.37 + send_guest_global_virq(dom0, VIRQ_MCA); 36.38 + } 36.39 + 36.40 printk(KERN_DEBUG "CMCI: CPU%d owner_map[%lx], no_cmci_map[%lx]\n", 36.41 smp_processor_id(), 36.42 *((unsigned long *)__get_cpu_var(mce_banks_owned)),
37.1 --- a/xen/arch/x86/cpu/mcheck/x86_mca.h Mon Mar 02 18:26:56 2009 +0900 37.2 +++ b/xen/arch/x86/cpu/mcheck/x86_mca.h Fri Mar 06 12:22:22 2009 +0900 37.3 @@ -46,8 +46,6 @@ 37.4 #define MCi_STATUS_MSEC 0x00000000ffff0000ULL 37.5 /* Other information */ 37.6 #define MCi_STATUS_OTHER 0x01ffffff00000000ULL 37.7 -/*Corrected Error Count*/ 37.8 -#define MCi_STATUS_ERRCOUNT 0x001FFFC0000000000ULL 37.9 /* processor context corrupt */ 37.10 #define MCi_STATUS_PCC 0x0200000000000000ULL 37.11 /* MSR_K8_MCi_ADDR register valid */
38.1 --- a/xen/arch/x86/domain.c Mon Mar 02 18:26:56 2009 +0900 38.2 +++ b/xen/arch/x86/domain.c Fri Mar 06 12:22:22 2009 +0900 38.3 @@ -58,7 +58,9 @@ DEFINE_PER_CPU(u64, efer); 38.4 DEFINE_PER_CPU(unsigned long, cr4); 38.5 38.6 static void default_idle(void); 38.7 +static void default_dead_idle(void); 38.8 void (*pm_idle) (void) = default_idle; 38.9 +void (*dead_idle) (void) = default_dead_idle; 38.10 38.11 static void paravirt_ctxt_switch_from(struct vcpu *v); 38.12 static void paravirt_ctxt_switch_to(struct vcpu *v); 38.13 @@ -84,6 +86,12 @@ static void default_idle(void) 38.14 local_irq_enable(); 38.15 } 38.16 38.17 +static void default_dead_idle(void) 38.18 +{ 38.19 + for ( ; ; ) 38.20 + halt(); 38.21 +} 38.22 + 38.23 static void play_dead(void) 38.24 { 38.25 /* 38.26 @@ -102,8 +110,7 @@ static void play_dead(void) 38.27 38.28 /* With physical CPU hotplug, we should halt the cpu. */ 38.29 local_irq_disable(); 38.30 - for ( ; ; ) 38.31 - halt(); 38.32 + (*dead_idle)(); 38.33 } 38.34 38.35 void idle_loop(void) 38.36 @@ -379,6 +386,8 @@ int arch_domain_create(struct domain *d, 38.37 hvm_funcs.hap_supported && 38.38 (domcr_flags & DOMCRF_hap); 38.39 38.40 + d->arch.s3_integrity = !!(domcr_flags & DOMCRF_s3_integrity); 38.41 + 38.42 INIT_LIST_HEAD(&d->arch.pdev_list); 38.43 38.44 d->arch.relmem = RELMEM_not_started; 38.45 @@ -831,7 +840,7 @@ map_vcpu_info(struct vcpu *v, unsigned l 38.46 * lost. The domain will get a spurious event, but it can cope. 38.47 */ 38.48 vcpu_info(v, evtchn_upcall_pending) = 1; 38.49 - for ( i = 0; i < BITS_PER_GUEST_LONG(d); i++ ) 38.50 + for ( i = 0; i < BITS_PER_EVTCHN_WORD(d); i++ ) 38.51 set_bit(i, &vcpu_info(v, evtchn_pending_sel)); 38.52 38.53 return 0;
39.1 --- a/xen/arch/x86/domctl.c Mon Mar 02 18:26:56 2009 +0900 39.2 +++ b/xen/arch/x86/domctl.c Fri Mar 06 12:22:22 2009 +0900 39.3 @@ -494,7 +494,8 @@ long arch_do_domctl( 39.4 break; 39.5 } 39.6 39.7 - domctl->u.address_size.size = BITS_PER_GUEST_LONG(d); 39.8 + domctl->u.address_size.size = 39.9 + is_pv_32on64_domain(d) ? 32 : BITS_PER_LONG; 39.10 39.11 ret = 0; 39.12 rcu_unlock_domain(d);
40.1 --- a/xen/arch/x86/hvm/hvm.c Mon Mar 02 18:26:56 2009 +0900 40.2 +++ b/xen/arch/x86/hvm/hvm.c Fri Mar 06 12:22:22 2009 +0900 40.3 @@ -308,6 +308,9 @@ int hvm_domain_initialise(struct domain 40.4 spin_lock_init(&d->arch.hvm_domain.irq_lock); 40.5 spin_lock_init(&d->arch.hvm_domain.uc_lock); 40.6 40.7 + INIT_LIST_HEAD(&d->arch.hvm_domain.msixtbl_list); 40.8 + spin_lock_init(&d->arch.hvm_domain.msixtbl_list_lock); 40.9 + 40.10 hvm_init_guest_time(d); 40.11 40.12 d->arch.hvm_domain.params[HVM_PARAM_HPET_ENABLED] = 1; 40.13 @@ -348,11 +351,15 @@ int hvm_domain_initialise(struct domain 40.14 return rc; 40.15 } 40.16 40.17 +extern void msixtbl_pt_cleanup(struct domain *d); 40.18 + 40.19 void hvm_domain_relinquish_resources(struct domain *d) 40.20 { 40.21 hvm_destroy_ioreq_page(d, &d->arch.hvm_domain.ioreq); 40.22 hvm_destroy_ioreq_page(d, &d->arch.hvm_domain.buf_ioreq); 40.23 40.24 + msixtbl_pt_cleanup(d); 40.25 + 40.26 /* Stop all asynchronous timer actions. */ 40.27 rtc_deinit(d); 40.28 if ( d->vcpu[0] != NULL )
41.1 --- a/xen/arch/x86/hvm/intercept.c Mon Mar 02 18:26:56 2009 +0900 41.2 +++ b/xen/arch/x86/hvm/intercept.c Fri Mar 06 12:22:22 2009 +0900 41.3 @@ -35,14 +35,16 @@ 41.4 extern struct hvm_mmio_handler hpet_mmio_handler; 41.5 extern struct hvm_mmio_handler vlapic_mmio_handler; 41.6 extern struct hvm_mmio_handler vioapic_mmio_handler; 41.7 +extern struct hvm_mmio_handler msixtbl_mmio_handler; 41.8 41.9 -#define HVM_MMIO_HANDLER_NR 3 41.10 +#define HVM_MMIO_HANDLER_NR 4 41.11 41.12 static struct hvm_mmio_handler *hvm_mmio_handlers[HVM_MMIO_HANDLER_NR] = 41.13 { 41.14 &hpet_mmio_handler, 41.15 &vlapic_mmio_handler, 41.16 - &vioapic_mmio_handler 41.17 + &vioapic_mmio_handler, 41.18 + &msixtbl_mmio_handler 41.19 }; 41.20 41.21 static int hvm_mmio_access(struct vcpu *v,
42.1 --- a/xen/arch/x86/hvm/vmsi.c Mon Mar 02 18:26:56 2009 +0900 42.2 +++ b/xen/arch/x86/hvm/vmsi.c Fri Mar 06 12:22:22 2009 +0900 42.3 @@ -193,3 +193,286 @@ int vmsi_deliver(struct domain *d, int p 42.4 return 1; 42.5 } 42.6 42.7 +/* MSI-X mask bit hypervisor interception */ 42.8 +struct msixtbl_entry 42.9 +{ 42.10 + struct list_head list; 42.11 + atomic_t refcnt; /* how many bind_pt_irq called for the device */ 42.12 + 42.13 + /* TODO: resolve the potential race by destruction of pdev */ 42.14 + struct pci_dev *pdev; 42.15 + unsigned long gtable; /* gpa of msix table */ 42.16 + unsigned long table_len; 42.17 + unsigned long table_flags[MAX_MSIX_TABLE_ENTRIES / BITS_PER_LONG + 1]; 42.18 + 42.19 + struct rcu_head rcu; 42.20 +}; 42.21 + 42.22 +static struct msixtbl_entry *msixtbl_find_entry( 42.23 + struct vcpu *v, unsigned long addr) 42.24 +{ 42.25 + struct msixtbl_entry *entry; 42.26 + struct domain *d = v->domain; 42.27 + 42.28 + list_for_each_entry( entry, &d->arch.hvm_domain.msixtbl_list, list ) 42.29 + if ( addr >= entry->gtable && 42.30 + addr < entry->gtable + entry->table_len ) 42.31 + return entry; 42.32 + 42.33 + return NULL; 42.34 +} 42.35 + 42.36 +static void __iomem *msixtbl_addr_to_virt( 42.37 + struct msixtbl_entry *entry, unsigned long addr) 42.38 +{ 42.39 + int idx, nr_page; 42.40 + 42.41 + if ( !entry ) 42.42 + return NULL; 42.43 + 42.44 + nr_page = (addr >> PAGE_SHIFT) - 42.45 + (entry->gtable >> PAGE_SHIFT); 42.46 + 42.47 + if ( !entry->pdev ) 42.48 + return NULL; 42.49 + 42.50 + idx = entry->pdev->msix_table_idx[nr_page]; 42.51 + if ( !idx ) 42.52 + return NULL; 42.53 + 42.54 + return (void *)(fix_to_virt(idx) + 42.55 + (addr & ((1UL << PAGE_SHIFT) - 1))); 42.56 +} 42.57 + 42.58 +static int msixtbl_read( 42.59 + struct vcpu *v, unsigned long address, 42.60 + unsigned long len, unsigned long *pval) 42.61 +{ 42.62 + unsigned long offset; 42.63 + struct msixtbl_entry *entry; 42.64 + void *virt; 42.65 + int r = X86EMUL_UNHANDLEABLE; 42.66 + 42.67 + rcu_read_lock(); 42.68 + 42.69 + if ( len != 4 ) 42.70 + goto out; 42.71 + 42.72 + offset = address & (PCI_MSIX_ENTRY_SIZE - 1); 42.73 + if ( offset != PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) 42.74 + goto out; 42.75 + 42.76 + entry = msixtbl_find_entry(v, address); 42.77 + virt = msixtbl_addr_to_virt(entry, address); 42.78 + if ( !virt ) 42.79 + goto out; 42.80 + 42.81 + *pval = readl(virt); 42.82 + r = X86EMUL_OKAY; 42.83 + 42.84 +out: 42.85 + rcu_read_unlock(); 42.86 + return r; 42.87 +} 42.88 + 42.89 +static int msixtbl_write(struct vcpu *v, unsigned long address, 42.90 + unsigned long len, unsigned long val) 42.91 +{ 42.92 + unsigned long offset; 42.93 + struct msixtbl_entry *entry; 42.94 + void *virt; 42.95 + int nr_entry; 42.96 + int r = X86EMUL_UNHANDLEABLE; 42.97 + 42.98 + rcu_read_lock(); 42.99 + 42.100 + if ( len != 4 ) 42.101 + goto out; 42.102 + 42.103 + entry = msixtbl_find_entry(v, address); 42.104 + nr_entry = (address - entry->gtable) % PCI_MSIX_ENTRY_SIZE; 42.105 + 42.106 + offset = address & (PCI_MSIX_ENTRY_SIZE - 1); 42.107 + if ( offset != PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) 42.108 + { 42.109 + set_bit(nr_entry, &entry->table_flags); 42.110 + goto out; 42.111 + } 42.112 + 42.113 + /* exit to device model if address/data has been modified */ 42.114 + if ( test_and_clear_bit(nr_entry, &entry->table_flags) ) 42.115 + goto out; 42.116 + 42.117 + virt = msixtbl_addr_to_virt(entry, address); 42.118 + if ( !virt ) 42.119 + goto out; 42.120 + 42.121 + writel(val, virt); 42.122 + r = X86EMUL_OKAY; 42.123 + 42.124 +out: 42.125 + rcu_read_unlock(); 42.126 + return r; 42.127 +} 42.128 + 42.129 +static int msixtbl_range(struct vcpu *v, unsigned long addr) 42.130 +{ 42.131 + struct msixtbl_entry *entry; 42.132 + void *virt; 42.133 + 42.134 + rcu_read_lock(); 42.135 + 42.136 + entry = msixtbl_find_entry(v, addr); 42.137 + virt = msixtbl_addr_to_virt(entry, addr); 42.138 + 42.139 + rcu_read_unlock(); 42.140 + 42.141 + return !!virt; 42.142 +} 42.143 + 42.144 +struct hvm_mmio_handler msixtbl_mmio_handler = { 42.145 + .check_handler = msixtbl_range, 42.146 + .read_handler = msixtbl_read, 42.147 + .write_handler = msixtbl_write 42.148 +}; 42.149 + 42.150 +static struct msixtbl_entry *add_msixtbl_entry(struct domain *d, 42.151 + struct pci_dev *pdev, 42.152 + uint64_t gtable) 42.153 +{ 42.154 + struct msixtbl_entry *entry; 42.155 + u32 len; 42.156 + 42.157 + entry = xmalloc(struct msixtbl_entry); 42.158 + if ( !entry ) 42.159 + return NULL; 42.160 + 42.161 + memset(entry, 0, sizeof(struct msixtbl_entry)); 42.162 + 42.163 + INIT_LIST_HEAD(&entry->list); 42.164 + INIT_RCU_HEAD(&entry->rcu); 42.165 + atomic_set(&entry->refcnt, 0); 42.166 + 42.167 + len = pci_msix_get_table_len(pdev); 42.168 + entry->table_len = len; 42.169 + entry->pdev = pdev; 42.170 + entry->gtable = (unsigned long) gtable; 42.171 + 42.172 + list_add_rcu(&entry->list, &d->arch.hvm_domain.msixtbl_list); 42.173 + 42.174 + return entry; 42.175 +} 42.176 + 42.177 +static void free_msixtbl_entry(struct rcu_head *rcu) 42.178 +{ 42.179 + struct msixtbl_entry *entry; 42.180 + 42.181 + entry = container_of (rcu, struct msixtbl_entry, rcu); 42.182 + 42.183 + xfree(entry); 42.184 +} 42.185 + 42.186 +static void del_msixtbl_entry(struct msixtbl_entry *entry) 42.187 +{ 42.188 + list_del_rcu(&entry->list); 42.189 + call_rcu(&entry->rcu, free_msixtbl_entry); 42.190 +} 42.191 + 42.192 +int msixtbl_pt_register(struct domain *d, int pirq, uint64_t gtable) 42.193 +{ 42.194 + irq_desc_t *irq_desc; 42.195 + struct msi_desc *msi_desc; 42.196 + struct pci_dev *pdev; 42.197 + struct msixtbl_entry *entry; 42.198 + int r = -EINVAL; 42.199 + 42.200 + ASSERT(spin_is_locked(&pcidevs_lock)); 42.201 + 42.202 + irq_desc = domain_spin_lock_irq_desc(d, pirq, NULL); 42.203 + 42.204 + if ( irq_desc->handler != &pci_msi_type ) 42.205 + goto out; 42.206 + 42.207 + msi_desc = irq_desc->msi_desc; 42.208 + if ( !msi_desc ) 42.209 + goto out; 42.210 + 42.211 + pdev = msi_desc->dev; 42.212 + 42.213 + spin_lock(&d->arch.hvm_domain.msixtbl_list_lock); 42.214 + 42.215 + list_for_each_entry( entry, &d->arch.hvm_domain.msixtbl_list, list ) 42.216 + if ( pdev == entry->pdev ) 42.217 + goto found; 42.218 + 42.219 + entry = add_msixtbl_entry(d, pdev, gtable); 42.220 + if ( !entry ) 42.221 + { 42.222 + spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock); 42.223 + goto out; 42.224 + } 42.225 + 42.226 +found: 42.227 + atomic_inc(&entry->refcnt); 42.228 + spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock); 42.229 + r = 0; 42.230 + 42.231 +out: 42.232 + spin_unlock_irq(&irq_desc->lock); 42.233 + return r; 42.234 + 42.235 +} 42.236 + 42.237 +void msixtbl_pt_unregister(struct domain *d, int pirq) 42.238 +{ 42.239 + irq_desc_t *irq_desc; 42.240 + struct msi_desc *msi_desc; 42.241 + struct pci_dev *pdev; 42.242 + struct msixtbl_entry *entry; 42.243 + 42.244 + ASSERT(spin_is_locked(&pcidevs_lock)); 42.245 + 42.246 + irq_desc = domain_spin_lock_irq_desc(d, pirq, NULL); 42.247 + 42.248 + if ( irq_desc->handler != &pci_msi_type ) 42.249 + goto out; 42.250 + 42.251 + msi_desc = irq_desc->msi_desc; 42.252 + if ( !msi_desc ) 42.253 + goto out; 42.254 + 42.255 + pdev = msi_desc->dev; 42.256 + 42.257 + spin_lock(&d->arch.hvm_domain.msixtbl_list_lock); 42.258 + 42.259 + list_for_each_entry( entry, &d->arch.hvm_domain.msixtbl_list, list ) 42.260 + if ( pdev == entry->pdev ) 42.261 + goto found; 42.262 + 42.263 + spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock); 42.264 + 42.265 + 42.266 +out: 42.267 + spin_unlock(&irq_desc->lock); 42.268 + return; 42.269 + 42.270 +found: 42.271 + if ( !atomic_dec_and_test(&entry->refcnt) ) 42.272 + del_msixtbl_entry(entry); 42.273 + 42.274 + spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock); 42.275 + spin_unlock(&irq_desc->lock); 42.276 +} 42.277 + 42.278 +void msixtbl_pt_cleanup(struct domain *d, int pirq) 42.279 +{ 42.280 + struct msixtbl_entry *entry, *temp; 42.281 + 42.282 + spin_lock(&d->arch.hvm_domain.msixtbl_list_lock); 42.283 + 42.284 + list_for_each_entry_safe( entry, temp, 42.285 + &d->arch.hvm_domain.msixtbl_list, list ) 42.286 + del_msixtbl_entry(entry); 42.287 + 42.288 + spin_unlock(&d->arch.hvm_domain.msixtbl_list_lock); 42.289 +}
43.1 --- a/xen/arch/x86/irq.c Mon Mar 02 18:26:56 2009 +0900 43.2 +++ b/xen/arch/x86/irq.c Fri Mar 06 12:22:22 2009 +0900 43.3 @@ -1104,7 +1104,8 @@ static void dump_irqs(unsigned char key) 43.4 (test_bit(d->pirq_to_evtchn[irq], 43.5 &shared_info(d, evtchn_pending)) ? 43.6 'P' : '-'), 43.7 - (test_bit(d->pirq_to_evtchn[irq]/BITS_PER_GUEST_LONG(d), 43.8 + (test_bit(d->pirq_to_evtchn[irq] / 43.9 + BITS_PER_EVTCHN_WORD(d), 43.10 &vcpu_info(d->vcpu[0], evtchn_pending_sel)) ? 43.11 'S' : '-'), 43.12 (test_bit(d->pirq_to_evtchn[irq],
44.1 --- a/xen/arch/x86/mm.c Mon Mar 02 18:26:56 2009 +0900 44.2 +++ b/xen/arch/x86/mm.c Fri Mar 06 12:22:22 2009 +0900 44.3 @@ -279,15 +279,39 @@ void __init arch_init_memory(void) 44.4 subarch_init_memory(); 44.5 } 44.6 44.7 -int page_is_conventional_ram(unsigned long mfn) 44.8 +int page_is_ram_type(unsigned long mfn, unsigned long mem_type) 44.9 { 44.10 uint64_t maddr = pfn_to_paddr(mfn); 44.11 int i; 44.12 44.13 for ( i = 0; i < e820.nr_map; i++ ) 44.14 { 44.15 - if ( (e820.map[i].type == E820_RAM) && 44.16 - (e820.map[i].addr <= maddr) && 44.17 + switch ( e820.map[i].type ) 44.18 + { 44.19 + case E820_RAM: 44.20 + if ( mem_type & RAM_TYPE_CONVENTIONAL ) 44.21 + break; 44.22 + continue; 44.23 + case E820_RESERVED: 44.24 + if ( mem_type & RAM_TYPE_RESERVED ) 44.25 + break; 44.26 + continue; 44.27 + case E820_UNUSABLE: 44.28 + if ( mem_type & RAM_TYPE_UNUSABLE ) 44.29 + break; 44.30 + continue; 44.31 + case E820_ACPI: 44.32 + case E820_NVS: 44.33 + if ( mem_type & RAM_TYPE_ACPI ) 44.34 + break; 44.35 + continue; 44.36 + default: 44.37 + /* unknown */ 44.38 + continue; 44.39 + } 44.40 + 44.41 + /* Test the range. */ 44.42 + if ( (e820.map[i].addr <= maddr) && 44.43 ((e820.map[i].addr + e820.map[i].size) >= (maddr + PAGE_SIZE)) ) 44.44 return 1; 44.45 } 44.46 @@ -347,14 +371,14 @@ void share_xen_page_with_privileged_gues 44.47 #else 44.48 /* 44.49 * In debug builds we shadow a selection of <4GB PDPTs to exercise code paths. 44.50 - * We cannot safely shadow the idle page table, nor shadow (v1) page tables 44.51 - * (detected by lack of an owning domain). As required for correctness, we 44.52 + * We cannot safely shadow the idle page table, nor shadow page tables 44.53 + * (detected by zero reference count). As required for correctness, we 44.54 * always shadow PDPTs above 4GB. 44.55 */ 44.56 -#define l3tab_needs_shadow(mfn) \ 44.57 - (((((mfn) << PAGE_SHIFT) != __pa(idle_pg_table)) && \ 44.58 - (page_get_owner(mfn_to_page(mfn)) != NULL) && \ 44.59 - ((mfn) & 1)) || /* odd MFNs are shadowed */ \ 44.60 +#define l3tab_needs_shadow(mfn) \ 44.61 + (((((mfn) << PAGE_SHIFT) != __pa(idle_pg_table)) && \ 44.62 + (mfn_to_page(mfn)->count_info & PGC_count_mask) && \ 44.63 + ((mfn) & 1)) || /* odd MFNs are shadowed */ \ 44.64 ((mfn) >= 0x100000)) 44.65 #endif 44.66 44.67 @@ -666,7 +690,16 @@ get_##level##_linear_pagetable( 44.68 44.69 int is_iomem_page(unsigned long mfn) 44.70 { 44.71 - return (!mfn_valid(mfn) || (page_get_owner(mfn_to_page(mfn)) == dom_io)); 44.72 + struct page_info *page; 44.73 + 44.74 + if ( !mfn_valid(mfn) ) 44.75 + return 1; 44.76 + 44.77 + /* Caller must know that it is an iomem page, or a reference is held. */ 44.78 + page = mfn_to_page(mfn); 44.79 + ASSERT((page->count_info & PGC_count_mask) != 0); 44.80 + 44.81 + return (page_get_owner(page) == dom_io); 44.82 } 44.83 44.84 44.85 @@ -679,7 +712,6 @@ get_page_from_l1e( 44.86 uint32_t l1f = l1e_get_flags(l1e); 44.87 struct vcpu *curr = current; 44.88 struct domain *owner; 44.89 - int okay; 44.90 44.91 if ( !(l1f & _PAGE_PRESENT) ) 44.92 return 1; 44.93 @@ -690,8 +722,13 @@ get_page_from_l1e( 44.94 return 0; 44.95 } 44.96 44.97 - if ( is_iomem_page(mfn) ) 44.98 + if ( !mfn_valid(mfn) || 44.99 + (owner = page_get_owner_and_reference(page)) == dom_io ) 44.100 { 44.101 + /* Only needed the reference to confirm dom_io ownership. */ 44.102 + if ( mfn_valid(mfn) ) 44.103 + put_page(page); 44.104 + 44.105 /* DOMID_IO reverts to caller for privilege checks. */ 44.106 if ( d == dom_io ) 44.107 d = curr->domain; 44.108 @@ -707,33 +744,29 @@ get_page_from_l1e( 44.109 return 1; 44.110 } 44.111 44.112 + if ( owner == NULL ) 44.113 + goto could_not_pin; 44.114 + 44.115 /* 44.116 * Let privileged domains transfer the right to map their target 44.117 * domain's pages. This is used to allow stub-domain pvfb export to dom0, 44.118 * until pvfb supports granted mappings. At that time this minor hack 44.119 * can go away. 44.120 */ 44.121 - owner = page_get_owner(page); 44.122 - if ( unlikely(d != owner) && (owner != NULL) && 44.123 - (d != curr->domain) && IS_PRIV_FOR(d, owner) ) 44.124 + if ( unlikely(d != owner) && (d != curr->domain) && IS_PRIV_FOR(d, owner) ) 44.125 d = owner; 44.126 44.127 /* Foreign mappings into guests in shadow external mode don't 44.128 * contribute to writeable mapping refcounts. (This allows the 44.129 * qemu-dm helper process in dom0 to map the domain's memory without 44.130 * messing up the count of "real" writable mappings.) */ 44.131 - okay = get_data_page( 44.132 - page, d, 44.133 - (l1f & _PAGE_RW) && !(paging_mode_external(d) && (d != curr->domain))); 44.134 - if ( !okay ) 44.135 - { 44.136 - MEM_LOG("Error getting mfn %lx (pfn %lx) from L1 entry %" PRIpte 44.137 - " for dom%d", 44.138 - mfn, get_gpfn_from_mfn(mfn), 44.139 - l1e_get_intpte(l1e), d->domain_id); 44.140 - } 44.141 - else if ( pte_flags_to_cacheattr(l1f) != 44.142 - ((page->count_info >> PGC_cacheattr_base) & 7) ) 44.143 + if ( (l1f & _PAGE_RW) && 44.144 + !(paging_mode_external(d) && (d != curr->domain)) && 44.145 + !get_page_type(page, PGT_writable_page) ) 44.146 + goto could_not_pin; 44.147 + 44.148 + if ( pte_flags_to_cacheattr(l1f) != 44.149 + ((page->count_info >> PGC_cacheattr_base) & 7) ) 44.150 { 44.151 unsigned long x, nx, y = page->count_info; 44.152 unsigned long cacheattr = pte_flags_to_cacheattr(l1f); 44.153 @@ -762,7 +795,16 @@ get_page_from_l1e( 44.154 #endif 44.155 } 44.156 44.157 - return okay; 44.158 + return 1; 44.159 + 44.160 + could_not_pin: 44.161 + MEM_LOG("Error getting mfn %lx (pfn %lx) from L1 entry %" PRIpte 44.162 + " for dom%d", 44.163 + mfn, get_gpfn_from_mfn(mfn), 44.164 + l1e_get_intpte(l1e), d->domain_id); 44.165 + if ( owner != NULL ) 44.166 + put_page(page); 44.167 + return 0; 44.168 } 44.169 44.170 44.171 @@ -1150,7 +1192,7 @@ static int create_pae_xen_mappings(struc 44.172 for ( i = 0; i < PDPT_L2_ENTRIES; i++ ) 44.173 { 44.174 l2e = l2e_from_page( 44.175 - virt_to_page(page_get_owner(page)->arch.mm_perdomain_pt) + i, 44.176 + virt_to_page(d->arch.mm_perdomain_pt) + i, 44.177 __PAGE_HYPERVISOR); 44.178 l2e_write(&pl2e[l2_table_offset(PERDOMAIN_VIRT_START) + i], l2e); 44.179 } 44.180 @@ -1900,7 +1942,7 @@ void put_page(struct page_info *page) 44.181 } 44.182 44.183 44.184 -int get_page(struct page_info *page, struct domain *domain) 44.185 +struct domain *page_get_owner_and_reference(struct page_info *page) 44.186 { 44.187 unsigned long x, y = page->count_info; 44.188 44.189 @@ -1909,22 +1951,29 @@ int get_page(struct page_info *page, str 44.190 if ( unlikely((x & PGC_count_mask) == 0) || /* Not allocated? */ 44.191 /* Keep one spare reference to be acquired by get_page_light(). */ 44.192 unlikely(((x + 2) & PGC_count_mask) <= 1) ) /* Overflow? */ 44.193 - goto fail; 44.194 + return NULL; 44.195 } 44.196 while ( (y = cmpxchg(&page->count_info, x, x + 1)) != x ); 44.197 44.198 - if ( likely(page_get_owner(page) == domain) ) 44.199 + return page_get_owner(page); 44.200 +} 44.201 + 44.202 + 44.203 +int get_page(struct page_info *page, struct domain *domain) 44.204 +{ 44.205 + struct domain *owner = page_get_owner_and_reference(page); 44.206 + 44.207 + if ( likely(owner == domain) ) 44.208 return 1; 44.209 44.210 put_page(page); 44.211 44.212 - fail: 44.213 if ( !_shadow_mode_refcounts(domain) && !domain->is_dying ) 44.214 gdprintk(XENLOG_INFO, 44.215 "Error pfn %lx: rd=%p, od=%p, caf=%08lx, taf=%" 44.216 PRtype_info "\n", 44.217 - page_to_mfn(page), domain, page_get_owner(page), 44.218 - y, page->u.inuse.type_info); 44.219 + page_to_mfn(page), domain, owner, 44.220 + page->count_info, page->u.inuse.type_info); 44.221 return 0; 44.222 } 44.223 44.224 @@ -1950,7 +1999,6 @@ static void get_page_light(struct page_i 44.225 while ( unlikely(y != x) ); 44.226 } 44.227 44.228 - 44.229 static int alloc_page_type(struct page_info *page, unsigned long type, 44.230 int preemptible) 44.231 {
45.1 --- a/xen/arch/x86/msi.c Mon Mar 02 18:26:56 2009 +0900 45.2 +++ b/xen/arch/x86/msi.c Fri Mar 06 12:22:22 2009 +0900 45.3 @@ -839,3 +839,23 @@ int pci_restore_msi_state(struct pci_dev 45.4 return 0; 45.5 } 45.6 45.7 +unsigned int pci_msix_get_table_len(struct pci_dev *pdev) 45.8 +{ 45.9 + int pos; 45.10 + u16 control; 45.11 + u8 bus, slot, func; 45.12 + unsigned int len; 45.13 + 45.14 + bus = pdev->bus; 45.15 + slot = PCI_SLOT(pdev->devfn); 45.16 + func = PCI_FUNC(pdev->devfn); 45.17 + 45.18 + pos = pci_find_cap_offset(bus, slot, func, PCI_CAP_ID_MSIX); 45.19 + if ( !pos ) 45.20 + return 0; 45.21 + 45.22 + control = pci_conf_read16(bus, slot, func, msix_control_reg(pos)); 45.23 + len = msix_table_size(control) * PCI_MSIX_ENTRY_SIZE; 45.24 + 45.25 + return len; 45.26 +}
46.1 --- a/xen/arch/x86/setup.c Mon Mar 02 18:26:56 2009 +0900 46.2 +++ b/xen/arch/x86/setup.c Fri Mar 06 12:22:22 2009 +0900 46.3 @@ -97,6 +97,7 @@ int early_boot = 1; 46.4 cpumask_t cpu_present_map; 46.5 46.6 unsigned long xen_phys_start; 46.7 +unsigned long allocator_bitmap_end; 46.8 46.9 #ifdef CONFIG_X86_32 46.10 /* Limits of Xen heap, used to initialise the allocator. */ 46.11 @@ -418,7 +419,6 @@ void __init __start_xen(unsigned long mb 46.12 multiboot_info_t *mbi = __va(mbi_p); 46.13 module_t *mod = (module_t *)__va(mbi->mods_addr); 46.14 unsigned long nr_pages, modules_length, modules_headroom; 46.15 - unsigned long allocator_bitmap_end; 46.16 int i, e820_warn = 0, bytes = 0; 46.17 struct ns16550_defaults ns16550 = { 46.18 .data_bits = 8, 46.19 @@ -985,9 +985,12 @@ void __init __start_xen(unsigned long mb 46.20 46.21 if ( opt_watchdog ) 46.22 watchdog_enable(); 46.23 + 46.24 + if ( !tboot_protect_mem_regions() ) 46.25 + panic("Could not protect TXT memory regions\n"); 46.26 46.27 /* Create initial domain 0. */ 46.28 - dom0 = domain_create(0, 0, DOM0_SSIDREF); 46.29 + dom0 = domain_create(0, DOMCRF_s3_integrity, DOM0_SSIDREF); 46.30 if ( (dom0 == NULL) || (alloc_vcpu(dom0, 0, 0) == NULL) ) 46.31 panic("Error creating domain 0\n"); 46.32 46.33 @@ -1037,9 +1040,6 @@ void __init __start_xen(unsigned long mb 46.34 if ( xen_cpuidle ) 46.35 xen_processor_pmbits |= XEN_PROCESSOR_PM_CX; 46.36 46.37 - if ( !tboot_protect_mem_regions() ) 46.38 - panic("Could not protect TXT memory regions\n"); 46.39 - 46.40 /* 46.41 * We're going to setup domain0 using the module(s) that we stashed safely 46.42 * above our heap. The second module, if present, is an initrd ramdisk.
47.1 --- a/xen/arch/x86/tboot.c Mon Mar 02 18:26:56 2009 +0900 47.2 +++ b/xen/arch/x86/tboot.c Fri Mar 06 12:22:22 2009 +0900 47.3 @@ -3,11 +3,14 @@ 47.4 #include <xen/types.h> 47.5 #include <xen/lib.h> 47.6 #include <xen/sched.h> 47.7 +#include <xen/domain_page.h> 47.8 +#include <xen/iommu.h> 47.9 #include <asm/fixmap.h> 47.10 #include <asm/page.h> 47.11 #include <asm/processor.h> 47.12 #include <asm/e820.h> 47.13 #include <asm/tboot.h> 47.14 +#include <crypto/vmac.h> 47.15 47.16 /* tboot=<physical address of shared page> */ 47.17 static char opt_tboot[20] = ""; 47.18 @@ -16,6 +19,10 @@ string_param("tboot", opt_tboot); 47.19 /* Global pointer to shared data; NULL means no measured launch. */ 47.20 tboot_shared_t *g_tboot_shared; 47.21 47.22 +static vmac_t domain_mac; /* MAC for all domains during S3 */ 47.23 +static vmac_t xenheap_mac; /* MAC for xen heap during S3 */ 47.24 +static vmac_t frametable_mac; /* MAC for frame table during S3 */ 47.25 + 47.26 static const uuid_t tboot_shared_uuid = TBOOT_SHARED_UUID; 47.27 47.28 /* used by tboot_protect_mem_regions() and/or tboot_parse_dmar_table() */ 47.29 @@ -40,6 +47,7 @@ static uint64_t sinit_base, sinit_size; 47.30 #define TXTCR_HEAP_SIZE 0x0308 47.31 47.32 extern char __init_begin[], __per_cpu_start[], __per_cpu_end[], __bss_start[]; 47.33 +extern unsigned long allocator_bitmap_end; 47.34 47.35 #define SHA1_SIZE 20 47.36 typedef uint8_t sha1_hash_t[SHA1_SIZE]; 47.37 @@ -82,8 +90,9 @@ void __init tboot_probe(void) 47.38 if ( memcmp(&tboot_shared_uuid, (uuid_t *)tboot_shared, sizeof(uuid_t)) ) 47.39 return; 47.40 47.41 - /* new tboot_shared (w/ GAS support) is not backwards compatible */ 47.42 - if ( tboot_shared->version < 3 ) { 47.43 + /* new tboot_shared (w/ GAS support, integrity, etc.) is not backwards 47.44 + compatible */ 47.45 + if ( tboot_shared->version < 4 ) { 47.46 printk("unsupported version of tboot (%u)\n", tboot_shared->version); 47.47 return; 47.48 } 47.49 @@ -121,6 +130,143 @@ void __init tboot_probe(void) 47.50 (unsigned long)__va((map_base + map_size) << PAGE_SHIFT)); 47.51 } 47.52 47.53 +/* definitions from xen/drivers/passthrough/vtd/iommu.h 47.54 + * used to walk through vtd page tables */ 47.55 +#define LEVEL_STRIDE (9) 47.56 +#define PTE_NUM (1<<LEVEL_STRIDE) 47.57 +#define dma_pte_present(p) (((p).val & 3) != 0) 47.58 +#define dma_pte_addr(p) ((p).val & PAGE_MASK_4K) 47.59 +#define agaw_to_level(val) ((val)+2) 47.60 +struct dma_pte { 47.61 + u64 val; 47.62 +}; 47.63 + 47.64 +static void update_iommu_mac(vmac_ctx_t *ctx, uint64_t pt_maddr, int level) 47.65 +{ 47.66 + int i; 47.67 + struct dma_pte *pt_vaddr, *pte; 47.68 + int next_level = level - 1; 47.69 + 47.70 + if ( pt_maddr == 0 ) 47.71 + return; 47.72 + 47.73 + pt_vaddr = (struct dma_pte *)map_domain_page(pt_maddr >> PAGE_SHIFT_4K); 47.74 + vmac_update((void *)pt_vaddr, PAGE_SIZE, ctx); 47.75 + 47.76 + for ( i = 0; i < PTE_NUM; i++ ) 47.77 + { 47.78 + pte = &pt_vaddr[i]; 47.79 + if ( !dma_pte_present(*pte) ) 47.80 + continue; 47.81 + 47.82 + if ( next_level >= 1 ) 47.83 + update_iommu_mac(ctx, dma_pte_addr(*pte), next_level); 47.84 + } 47.85 + 47.86 + unmap_domain_page(pt_vaddr); 47.87 +} 47.88 + 47.89 +#define is_page_in_use(page) \ 47.90 + ((page->count_info & PGC_count_mask) != 0 || page->count_info == 0) 47.91 + 47.92 +static void update_pagetable_mac(vmac_ctx_t *ctx) 47.93 +{ 47.94 + unsigned long mfn; 47.95 + 47.96 + for ( mfn = 0; mfn < max_page; mfn++ ) 47.97 + { 47.98 + struct page_info *page = mfn_to_page(mfn); 47.99 + if ( is_page_in_use(page) && !is_xen_heap_page(page) ) { 47.100 + if ( page->count_info & PGC_page_table ) { 47.101 + void *pg = map_domain_page(mfn); 47.102 + vmac_update(pg, PAGE_SIZE, ctx); 47.103 + unmap_domain_page(pg); 47.104 + } 47.105 + } 47.106 + } 47.107 +} 47.108 + 47.109 +static void tboot_gen_domain_integrity(const uint8_t key[TB_KEY_SIZE], 47.110 + vmac_t *mac) 47.111 +{ 47.112 + struct domain *d; 47.113 + struct page_info *page; 47.114 + uint8_t nonce[16] = {}; 47.115 + vmac_ctx_t ctx; 47.116 + 47.117 + vmac_set_key((uint8_t *)key, &ctx); 47.118 + for_each_domain( d ) 47.119 + { 47.120 + if ( !d->arch.s3_integrity ) 47.121 + continue; 47.122 + printk("MACing Domain %u\n", d->domain_id); 47.123 + 47.124 + page_list_for_each(page, &d->page_list) 47.125 + { 47.126 + void *pg; 47.127 + pg = map_domain_page(page_to_mfn(page)); 47.128 + vmac_update(pg, PAGE_SIZE, &ctx); 47.129 + unmap_domain_page(pg); 47.130 + } 47.131 + 47.132 + if ( !is_idle_domain(d) ) 47.133 + { 47.134 + struct hvm_iommu *hd = domain_hvm_iommu(d); 47.135 + update_iommu_mac(&ctx, hd->pgd_maddr, agaw_to_level(hd->agaw)); 47.136 + } 47.137 + } 47.138 + 47.139 + /* MAC all shadow page tables */ 47.140 + update_pagetable_mac(&ctx); 47.141 + 47.142 + *mac = vmac(NULL, 0, nonce, NULL, &ctx); 47.143 + 47.144 + printk("MAC for domains is: 0x%08"PRIx64"\n", *mac); 47.145 + 47.146 + /* wipe ctx to ensure key is not left in memory */ 47.147 + memset(&ctx, 0, sizeof(ctx)); 47.148 +} 47.149 + 47.150 +static void tboot_gen_xenheap_integrity(const uint8_t key[TB_KEY_SIZE], 47.151 + vmac_t *mac) 47.152 +{ 47.153 + unsigned long mfn; 47.154 + uint8_t nonce[16] = {}; 47.155 + vmac_ctx_t ctx; 47.156 + 47.157 + vmac_set_key((uint8_t *)key, &ctx); 47.158 + for ( mfn = 0; mfn < max_page; mfn++ ) 47.159 + { 47.160 + struct page_info *page = __mfn_to_page(mfn); 47.161 + if ( is_page_in_use(page) && is_xen_heap_page(page) ) { 47.162 + void *pg = mfn_to_virt(mfn); 47.163 + vmac_update((uint8_t *)pg, PAGE_SIZE, &ctx); 47.164 + } 47.165 + } 47.166 + *mac = vmac(NULL, 0, nonce, NULL, &ctx); 47.167 + 47.168 + printk("MAC for xenheap is: 0x%08"PRIx64"\n", *mac); 47.169 + 47.170 + /* wipe ctx to ensure key is not left in memory */ 47.171 + memset(&ctx, 0, sizeof(ctx)); 47.172 +} 47.173 + 47.174 +static void tboot_gen_frametable_integrity(const uint8_t key[TB_KEY_SIZE], 47.175 + vmac_t *mac) 47.176 +{ 47.177 + uint8_t nonce[16] = {}; 47.178 + vmac_ctx_t ctx; 47.179 + 47.180 + vmac_set_key((uint8_t *)key, &ctx); 47.181 + *mac = vmac((uint8_t *)frame_table, 47.182 + PFN_UP(max_page * sizeof(*frame_table)), nonce, NULL, &ctx); 47.183 + 47.184 + printk("MAC for frametable is: 0x%08"PRIx64"\n", *mac); 47.185 + 47.186 + /* wipe ctx to ensure key is not left in memory */ 47.187 + memset(&ctx, 0, sizeof(ctx)); 47.188 +} 47.189 + 47.190 void tboot_shutdown(uint32_t shutdown_type) 47.191 { 47.192 uint32_t map_base, map_size; 47.193 @@ -130,38 +276,60 @@ void tboot_shutdown(uint32_t shutdown_ty 47.194 47.195 local_irq_disable(); 47.196 47.197 - /* if this is S3 then set regions to MAC */ 47.198 - if ( shutdown_type == TB_SHUTDOWN_S3 ) { 47.199 - g_tboot_shared->num_mac_regions = 4; 47.200 - /* S3 resume code (and other real mode trampoline code) */ 47.201 - g_tboot_shared->mac_regions[0].start = 47.202 - (uint64_t)bootsym_phys(trampoline_start); 47.203 - g_tboot_shared->mac_regions[0].end = 47.204 - (uint64_t)bootsym_phys(trampoline_end); 47.205 - /* hypervisor code + data */ 47.206 - g_tboot_shared->mac_regions[1].start = (uint64_t)__pa(&_stext); 47.207 - g_tboot_shared->mac_regions[1].end = (uint64_t)__pa(&__init_begin); 47.208 - /* per-cpu data */ 47.209 - g_tboot_shared->mac_regions[2].start = (uint64_t)__pa(&__per_cpu_start); 47.210 - g_tboot_shared->mac_regions[2].end = (uint64_t)__pa(&__per_cpu_end); 47.211 - /* bss */ 47.212 - g_tboot_shared->mac_regions[3].start = (uint64_t)__pa(&__bss_start); 47.213 - g_tboot_shared->mac_regions[3].end = (uint64_t)__pa(&_end); 47.214 - } 47.215 + /* we may be called from an interrupt context, so to prevent */ 47.216 + /* 'ASSERT(!in_irq());' in alloc_domheap_pages(), decrease count */ 47.217 + while ( in_irq() ) 47.218 + irq_exit(); 47.219 47.220 /* Create identity map for tboot shutdown code. */ 47.221 + /* do before S3 integrity because mapping tboot may change xenheap */ 47.222 map_base = PFN_DOWN(g_tboot_shared->tboot_base); 47.223 map_size = PFN_UP(g_tboot_shared->tboot_size); 47.224 47.225 err = map_pages_to_xen(map_base << PAGE_SHIFT, map_base, map_size, 47.226 __PAGE_HYPERVISOR); 47.227 - if ( err != 0 ) 47.228 - { 47.229 + if ( err != 0 ) { 47.230 printk("error (0x%x) mapping tboot pages (mfns) @ 0x%x, 0x%x\n", err, 47.231 map_base, map_size); 47.232 return; 47.233 } 47.234 47.235 + /* if this is S3 then set regions to MAC */ 47.236 + if ( shutdown_type == TB_SHUTDOWN_S3 ) { 47.237 + /* 47.238 + * Xen regions for tboot to MAC 47.239 + */ 47.240 + g_tboot_shared->num_mac_regions = 5; 47.241 + /* S3 resume code (and other real mode trampoline code) */ 47.242 + g_tboot_shared->mac_regions[0].start = bootsym_phys(trampoline_start); 47.243 + g_tboot_shared->mac_regions[0].size = bootsym_phys(trampoline_end) - 47.244 + bootsym_phys(trampoline_start); 47.245 + /* hypervisor code + data */ 47.246 + g_tboot_shared->mac_regions[1].start = (uint64_t)__pa(&_stext); 47.247 + g_tboot_shared->mac_regions[1].size = __pa(&__init_begin) - 47.248 + __pa(&_stext); 47.249 + /* per-cpu data */ 47.250 + g_tboot_shared->mac_regions[2].start = (uint64_t)__pa(&__per_cpu_start); 47.251 + g_tboot_shared->mac_regions[2].size = __pa(&__per_cpu_end) - 47.252 + __pa(&__per_cpu_start); 47.253 + /* bss */ 47.254 + g_tboot_shared->mac_regions[3].start = (uint64_t)__pa(&__bss_start); 47.255 + g_tboot_shared->mac_regions[3].size = __pa(&_end) - __pa(&__bss_start); 47.256 + /* boot allocator bitmap */ 47.257 + g_tboot_shared->mac_regions[4].start = (uint64_t)__pa(&_end); 47.258 + g_tboot_shared->mac_regions[4].size = allocator_bitmap_end - 47.259 + __pa(&_end); 47.260 + 47.261 + /* 47.262 + * MAC domains and other Xen memory 47.263 + */ 47.264 + /* Xen has no better entropy source for MAC key than tboot's */ 47.265 + /* MAC domains first in case it perturbs xenheap */ 47.266 + tboot_gen_domain_integrity(g_tboot_shared->s3_key, &domain_mac); 47.267 + tboot_gen_frametable_integrity(g_tboot_shared->s3_key, &frametable_mac); 47.268 + tboot_gen_xenheap_integrity(g_tboot_shared->s3_key, &xenheap_mac); 47.269 + } 47.270 + 47.271 write_ptbase(idle_vcpu[0]); 47.272 47.273 ((void(*)(void))(unsigned long)g_tboot_shared->shutdown_entry)(); 47.274 @@ -264,6 +432,29 @@ int __init tboot_parse_dmar_table(acpi_t 47.275 return rc; 47.276 } 47.277 47.278 +int tboot_s3_resume(void) 47.279 +{ 47.280 + vmac_t mac; 47.281 + 47.282 + if ( !tboot_in_measured_env() ) 47.283 + return 0; 47.284 + 47.285 + /* need to do these in reverse order of shutdown */ 47.286 + tboot_gen_xenheap_integrity(g_tboot_shared->s3_key, &mac); 47.287 + if ( mac != xenheap_mac ) 47.288 + return -1; 47.289 + 47.290 + tboot_gen_frametable_integrity(g_tboot_shared->s3_key, &mac); 47.291 + if ( mac != frametable_mac ) 47.292 + return -2; 47.293 + 47.294 + tboot_gen_domain_integrity(g_tboot_shared->s3_key, &mac); 47.295 + if ( mac != domain_mac ) 47.296 + return 0; /* -3 */ 47.297 + 47.298 + return 0; 47.299 +} 47.300 + 47.301 /* 47.302 * Local variables: 47.303 * mode: C
48.1 --- a/xen/arch/x86/time.c Mon Mar 02 18:26:56 2009 +0900 48.2 +++ b/xen/arch/x86/time.c Fri Mar 06 12:22:22 2009 +0900 48.3 @@ -1087,38 +1087,69 @@ static void local_time_calibration(void) 48.4 */ 48.5 struct calibration_rendezvous { 48.6 cpumask_t cpu_calibration_map; 48.7 - atomic_t nr_cpus; 48.8 + atomic_t count_start; 48.9 + atomic_t count_end; 48.10 s_time_t master_stime; 48.11 u64 master_tsc_stamp; 48.12 }; 48.13 48.14 +#define NR_LOOPS 5 48.15 + 48.16 static void time_calibration_rendezvous(void *_r) 48.17 { 48.18 + int i; 48.19 struct cpu_calibration *c = &this_cpu(cpu_calibration); 48.20 struct calibration_rendezvous *r = _r; 48.21 unsigned int total_cpus = cpus_weight(r->cpu_calibration_map); 48.22 48.23 - if ( smp_processor_id() == 0 ) 48.24 + /* 48.25 + * Loop is used here to get rid of the cache's side effect to enlarge 48.26 + * the TSC difference among CPUs. 48.27 + */ 48.28 + for ( i = 0; i < NR_LOOPS; i++ ) 48.29 { 48.30 - while ( atomic_read(&r->nr_cpus) != (total_cpus - 1) ) 48.31 - cpu_relax(); 48.32 - r->master_stime = read_platform_stime(); 48.33 - rdtscll(r->master_tsc_stamp); 48.34 - mb(); /* write r->master_* /then/ signal */ 48.35 - atomic_inc(&r->nr_cpus); 48.36 - c->local_tsc_stamp = r->master_tsc_stamp; 48.37 - } 48.38 - else 48.39 - { 48.40 - atomic_inc(&r->nr_cpus); 48.41 - while ( atomic_read(&r->nr_cpus) != total_cpus ) 48.42 - cpu_relax(); 48.43 - mb(); /* receive signal /then/ read r->master_* */ 48.44 - if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) ) 48.45 - wrmsrl(MSR_IA32_TSC, r->master_tsc_stamp); 48.46 - rdtscll(c->local_tsc_stamp); 48.47 + if ( smp_processor_id() == 0 ) 48.48 + { 48.49 + while ( atomic_read(&r->count_start) != (total_cpus - 1) ) 48.50 + mb(); 48.51 + 48.52 + if ( r->master_stime == 0 ) 48.53 + { 48.54 + r->master_stime = read_platform_stime(); 48.55 + if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) ) 48.56 + rdtscll(r->master_tsc_stamp); 48.57 + } 48.58 + atomic_set(&r->count_end, 0); 48.59 + wmb(); 48.60 + atomic_inc(&r->count_start); 48.61 + 48.62 + if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && 48.63 + i == NR_LOOPS - 1 ) 48.64 + write_tsc((u32)r->master_tsc_stamp, (u32)(r->master_tsc_stamp >> 32)); 48.65 + 48.66 + while (atomic_read(&r->count_end) != total_cpus - 1) 48.67 + mb(); 48.68 + atomic_set(&r->count_start, 0); 48.69 + wmb(); 48.70 + atomic_inc(&r->count_end); 48.71 + } 48.72 + else 48.73 + { 48.74 + atomic_inc(&r->count_start); 48.75 + while ( atomic_read(&r->count_start) != total_cpus ) 48.76 + mb(); 48.77 + 48.78 + if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && 48.79 + i == NR_LOOPS - 1 ) 48.80 + write_tsc((u32)r->master_tsc_stamp, (u32)(r->master_tsc_stamp >> 32)); 48.81 + 48.82 + atomic_inc(&r->count_end); 48.83 + while (atomic_read(&r->count_end) != total_cpus) 48.84 + mb(); 48.85 + } 48.86 } 48.87 48.88 + rdtscll(c->local_tsc_stamp); 48.89 c->stime_local_stamp = get_s_time(); 48.90 c->stime_master_stamp = r->master_stime; 48.91 48.92 @@ -1129,7 +1160,9 @@ static void time_calibration(void *unuse 48.93 { 48.94 struct calibration_rendezvous r = { 48.95 .cpu_calibration_map = cpu_online_map, 48.96 - .nr_cpus = ATOMIC_INIT(0) 48.97 + .count_start = ATOMIC_INIT(0), 48.98 + .count_end = ATOMIC_INIT(0), 48.99 + .master_stime = 0 48.100 }; 48.101 48.102 /* @wait=1 because we must wait for all cpus before freeing @r. */
49.1 --- a/xen/arch/x86/traps.c Mon Mar 02 18:26:56 2009 +0900 49.2 +++ b/xen/arch/x86/traps.c Fri Mar 06 12:22:22 2009 +0900 49.3 @@ -2187,10 +2187,17 @@ static int emulate_privileged_op(struct 49.4 case MSR_IA32_MPERF: 49.5 case MSR_IA32_APERF: 49.6 case MSR_IA32_PERF_CTL: 49.7 + if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ) 49.8 + goto fail; 49.9 + if ( !is_cpufreq_controller(v->domain) ) 49.10 + break; 49.11 + if ( wrmsr_safe(regs->ecx, eax, edx) != 0 ) 49.12 + goto fail; 49.13 + break; 49.14 case MSR_IA32_THERM_CONTROL: 49.15 if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ) 49.16 goto fail; 49.17 - if ( !is_cpufreq_controller(v->domain) ) 49.18 + if ( (v->domain->domain_id != 0) || !v->domain->is_pinned ) 49.19 break; 49.20 if ( wrmsr_safe(regs->ecx, eax, edx) != 0 ) 49.21 goto fail; 49.22 @@ -3095,7 +3102,8 @@ long register_guest_nmi_callback(unsigne 49.23 49.24 t->vector = TRAP_nmi; 49.25 t->flags = 0; 49.26 - t->cs = !IS_COMPAT(d) ? FLAT_KERNEL_CS : FLAT_COMPAT_KERNEL_CS; 49.27 + t->cs = (is_pv_32on64_domain(d) ? 49.28 + FLAT_COMPAT_KERNEL_CS : FLAT_KERNEL_CS); 49.29 t->address = address; 49.30 TI_SET_IF(t, 1); 49.31
50.1 --- a/xen/common/domain.c Mon Mar 02 18:26:56 2009 +0900 50.2 +++ b/xen/common/domain.c Fri Mar 06 12:22:22 2009 +0900 50.3 @@ -465,7 +465,9 @@ void domain_shutdown(struct domain *d, u 50.4 50.5 for_each_vcpu ( d, v ) 50.6 { 50.7 - if ( v->defer_shutdown ) 50.8 + if ( reason == SHUTDOWN_crash ) 50.9 + v->defer_shutdown = 0; 50.10 + else if ( v->defer_shutdown ) 50.11 continue; 50.12 vcpu_pause_nosync(v); 50.13 v->paused_for_shutdown = 1;
51.1 --- a/xen/common/domctl.c Mon Mar 02 18:26:56 2009 +0900 51.2 +++ b/xen/common/domctl.c Fri Mar 06 12:22:22 2009 +0900 51.3 @@ -242,13 +242,15 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc 51.4 if ( (c.nat = xmalloc(struct vcpu_guest_context)) == NULL ) 51.5 goto svc_out; 51.6 51.7 - if ( !IS_COMPAT(v->domain) ) 51.8 +#ifdef CONFIG_COMPAT 51.9 + if ( !is_pv_32on64_vcpu(v) ) 51.10 ret = copy_from_guest(c.nat, op->u.vcpucontext.ctxt, 1); 51.11 -#ifdef CONFIG_COMPAT 51.12 else 51.13 ret = copy_from_guest(c.cmp, 51.14 guest_handle_cast(op->u.vcpucontext.ctxt, 51.15 void), 1); 51.16 +#else 51.17 + ret = copy_from_guest(c.nat, op->u.vcpucontext.ctxt, 1); 51.18 #endif 51.19 ret = ret ? -EFAULT : 0; 51.20 51.21 @@ -339,7 +341,8 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc 51.22 ret = -EINVAL; 51.23 if ( supervisor_mode_kernel || 51.24 (op->u.createdomain.flags & 51.25 - ~(XEN_DOMCTL_CDF_hvm_guest | XEN_DOMCTL_CDF_hap)) ) 51.26 + ~(XEN_DOMCTL_CDF_hvm_guest | XEN_DOMCTL_CDF_hap | 51.27 + XEN_DOMCTL_CDF_s3_integrity)) ) 51.28 break; 51.29 51.30 dom = op->domain; 51.31 @@ -371,6 +374,8 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc 51.32 domcr_flags |= DOMCRF_hvm; 51.33 if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_hap ) 51.34 domcr_flags |= DOMCRF_hap; 51.35 + if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_s3_integrity ) 51.36 + domcr_flags |= DOMCRF_s3_integrity; 51.37 51.38 ret = -ENOMEM; 51.39 d = domain_create(dom, domcr_flags, op->u.createdomain.ssidref); 51.40 @@ -593,12 +598,14 @@ long do_domctl(XEN_GUEST_HANDLE(xen_domc 51.41 if ( v != current ) 51.42 vcpu_unpause(v); 51.43 51.44 - if ( !IS_COMPAT(v->domain) ) 51.45 +#ifdef CONFIG_COMPAT 51.46 + if ( !is_pv_32on64_vcpu(v) ) 51.47 ret = copy_to_guest(op->u.vcpucontext.ctxt, c.nat, 1); 51.48 -#ifdef CONFIG_COMPAT 51.49 else 51.50 ret = copy_to_guest(guest_handle_cast(op->u.vcpucontext.ctxt, 51.51 void), c.cmp, 1); 51.52 +#else 51.53 + ret = copy_to_guest(op->u.vcpucontext.ctxt, c.nat, 1); 51.54 #endif 51.55 51.56 if ( copy_to_guest(u_domctl, op, 1) || ret )
52.1 --- a/xen/common/event_channel.c Mon Mar 02 18:26:56 2009 +0900 52.2 +++ b/xen/common/event_channel.c Fri Mar 06 12:22:22 2009 +0900 52.3 @@ -548,7 +548,7 @@ static int evtchn_set_pending(struct vcp 52.4 return 1; 52.5 52.6 if ( !test_bit (port, &shared_info(d, evtchn_mask)) && 52.7 - !test_and_set_bit(port / BITS_PER_GUEST_LONG(d), 52.8 + !test_and_set_bit(port / BITS_PER_EVTCHN_WORD(d), 52.9 &vcpu_info(v, evtchn_pending_sel)) ) 52.10 { 52.11 vcpu_mark_events_pending(v); 52.12 @@ -783,7 +783,7 @@ int evtchn_unmask(unsigned int port) 52.13 */ 52.14 if ( test_and_clear_bit(port, &shared_info(d, evtchn_mask)) && 52.15 test_bit (port, &shared_info(d, evtchn_pending)) && 52.16 - !test_and_set_bit (port / BITS_PER_GUEST_LONG(d), 52.17 + !test_and_set_bit (port / BITS_PER_EVTCHN_WORD(d), 52.18 &vcpu_info(v, evtchn_pending_sel)) ) 52.19 { 52.20 vcpu_mark_events_pending(v);
53.1 --- a/xen/common/grant_table.c Mon Mar 02 18:26:56 2009 +0900 53.2 +++ b/xen/common/grant_table.c Fri Mar 06 12:22:22 2009 +0900 53.3 @@ -195,7 +195,7 @@ static void 53.4 __gnttab_map_grant_ref( 53.5 struct gnttab_map_grant_ref *op) 53.6 { 53.7 - struct domain *ld, *rd; 53.8 + struct domain *ld, *rd, *owner; 53.9 struct vcpu *led; 53.10 int handle; 53.11 unsigned long frame = 0, nr_gets = 0; 53.12 @@ -336,8 +336,13 @@ static void 53.13 53.14 spin_unlock(&rd->grant_table->lock); 53.15 53.16 - if ( is_iomem_page(frame) ) 53.17 + if ( !mfn_valid(frame) || 53.18 + (owner = page_get_owner_and_reference(mfn_to_page(frame))) == dom_io ) 53.19 { 53.20 + /* Only needed the reference to confirm dom_io ownership. */ 53.21 + if ( mfn_valid(frame) ) 53.22 + put_page(mfn_to_page(frame)); 53.23 + 53.24 if ( !iomem_access_permitted(rd, frame, frame) ) 53.25 { 53.26 gdprintk(XENLOG_WARNING, 53.27 @@ -352,20 +357,11 @@ static void 53.28 if ( rc != GNTST_okay ) 53.29 goto undo_out; 53.30 } 53.31 - else 53.32 + else if ( owner == rd ) 53.33 { 53.34 - if ( unlikely(!mfn_valid(frame)) || 53.35 - unlikely(!(gnttab_host_mapping_get_page_type(op, ld, rd) ? 53.36 - get_page_and_type(mfn_to_page(frame), rd, 53.37 - PGT_writable_page) : 53.38 - get_page(mfn_to_page(frame), rd))) ) 53.39 - { 53.40 - if ( !rd->is_dying ) 53.41 - gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n", 53.42 - frame); 53.43 - rc = GNTST_general_error; 53.44 - goto undo_out; 53.45 - } 53.46 + if ( gnttab_host_mapping_get_page_type(op, ld, rd) && 53.47 + !get_page_type(mfn_to_page(frame), PGT_writable_page) ) 53.48 + goto could_not_pin; 53.49 53.50 nr_gets++; 53.51 if ( op->flags & GNTMAP_host_map ) 53.52 @@ -383,6 +379,17 @@ static void 53.53 } 53.54 } 53.55 } 53.56 + else 53.57 + { 53.58 + could_not_pin: 53.59 + if ( !rd->is_dying ) 53.60 + gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n", 53.61 + frame); 53.62 + if ( owner != NULL ) 53.63 + put_page(mfn_to_page(frame)); 53.64 + rc = GNTST_general_error; 53.65 + goto undo_out; 53.66 + } 53.67 53.68 if ( need_iommu(ld) && 53.69 !(old_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) &&
54.1 --- a/xen/common/keyhandler.c Mon Mar 02 18:26:56 2009 +0900 54.2 +++ b/xen/common/keyhandler.c Fri Mar 06 12:22:22 2009 +0900 54.3 @@ -225,7 +225,7 @@ static void dump_domains(unsigned char k 54.4 test_bit(v->virq_to_evtchn[VIRQ_DEBUG], 54.5 &shared_info(d, evtchn_mask)), 54.6 test_bit(v->virq_to_evtchn[VIRQ_DEBUG] / 54.7 - BITS_PER_GUEST_LONG(d), 54.8 + BITS_PER_EVTCHN_WORD(d), 54.9 &vcpu_info(v, evtchn_pending_sel))); 54.10 send_guest_vcpu_virq(v, VIRQ_DEBUG); 54.11 }
55.1 --- a/xen/common/trace.c Mon Mar 02 18:26:56 2009 +0900 55.2 +++ b/xen/common/trace.c Fri Mar 06 12:22:22 2009 +0900 55.3 @@ -37,10 +37,8 @@ 55.4 #define xen_t_buf t_buf 55.5 CHECK_t_buf; 55.6 #undef xen_t_buf 55.7 -#define TB_COMPAT IS_COMPAT(dom0) 55.8 #else 55.9 #define compat_t_rec t_rec 55.10 -#define TB_COMPAT 0 55.11 #endif 55.12 55.13 /* opt_tbuf_size: trace buffer size (in pages) */
56.1 --- a/xen/common/xencomm.c Mon Mar 02 18:26:56 2009 +0900 56.2 +++ b/xen/common/xencomm.c Fri Mar 06 12:22:22 2009 +0900 56.3 @@ -51,24 +51,16 @@ xencomm_get_page(unsigned long paddr, st 56.4 return -EFAULT; 56.5 56.6 *page = maddr_to_page(maddr); 56.7 - if ( get_page(*page, current->domain) == 0 ) 56.8 + if ( !get_page(*page, current->domain) ) 56.9 { 56.10 - if ( page_get_owner(*page) != current->domain ) 56.11 - { 56.12 - /* 56.13 - * This page might be a page granted by another domain, or 56.14 - * this page is freed with decrease reservation hypercall at 56.15 - * the same time. 56.16 - */ 56.17 - gdprintk(XENLOG_WARNING, 56.18 - "bad page is passed. paddr 0x%lx maddr 0x%lx\n", 56.19 - paddr, maddr); 56.20 - return -EFAULT; 56.21 - } 56.22 - 56.23 - /* Try again. */ 56.24 - cpu_relax(); 56.25 - return -EAGAIN; 56.26 + /* 56.27 + * This page might be a page granted by another domain, or this page 56.28 + * is freed with decrease reservation hypercall at the same time. 56.29 + */ 56.30 + gdprintk(XENLOG_WARNING, 56.31 + "bad page is passed. paddr 0x%lx maddr 0x%lx\n", 56.32 + paddr, maddr); 56.33 + return -EFAULT; 56.34 } 56.35 56.36 return 0;
57.1 --- a/xen/common/xenoprof.c Mon Mar 02 18:26:56 2009 +0900 57.2 +++ b/xen/common/xenoprof.c Fri Mar 06 12:22:22 2009 +0900 57.3 @@ -208,7 +208,7 @@ static int alloc_xenoprof_struct( 57.4 bufsize = sizeof(struct xenoprof_buf); 57.5 i = sizeof(struct event_log); 57.6 #ifdef CONFIG_COMPAT 57.7 - d->xenoprof->is_compat = IS_COMPAT(is_passive ? dom0 : d); 57.8 + d->xenoprof->is_compat = is_pv_32on64_domain(is_passive ? dom0 : d); 57.9 if ( XENOPROF_COMPAT(d->xenoprof) ) 57.10 { 57.11 bufsize = sizeof(struct compat_oprof_buf);
58.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 58.2 +++ b/xen/crypto/Makefile Fri Mar 06 12:22:22 2009 +0900 58.3 @@ -0,0 +1,2 @@ 58.4 +obj-y += rijndael.o 58.5 +obj-y += vmac.o
59.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 59.2 +++ b/xen/crypto/rijndael.c Fri Mar 06 12:22:22 2009 +0900 59.3 @@ -0,0 +1,1269 @@ 59.4 +/* $OpenBSD: rijndael.c,v 1.19 2008/06/09 07:49:45 djm Exp $ */ 59.5 + 59.6 +/** 59.7 + * rijndael-alg-fst.c 59.8 + * 59.9 + * @version 3.0 (December 2000) 59.10 + * 59.11 + * Optimised ANSI C code for the Rijndael cipher (now AES) 59.12 + * 59.13 + * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be> 59.14 + * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be> 59.15 + * @author Paulo Barreto <paulo.barreto@terra.com.br> 59.16 + * 59.17 + * This code is hereby placed in the public domain. 59.18 + * 59.19 + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS 59.20 + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 59.21 + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59.22 + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE 59.23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59.24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59.25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 59.26 + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 59.27 + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 59.28 + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 59.29 + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 59.30 + */ 59.31 + 59.32 +/* start for Xen */ 59.33 +#include <xen/config.h> 59.34 +#include <xen/init.h> 59.35 +#include <xen/types.h> 59.36 +#include <xen/lib.h> 59.37 +#include <crypto/rijndael.h> 59.38 +/* end for Xen */ 59.39 + 59.40 +#undef FULL_UNROLL 59.41 + 59.42 +/* 59.43 +Te0[x] = S [x].[02, 01, 01, 03]; 59.44 +Te1[x] = S [x].[03, 02, 01, 01]; 59.45 +Te2[x] = S [x].[01, 03, 02, 01]; 59.46 +Te3[x] = S [x].[01, 01, 03, 02]; 59.47 +Te4[x] = S [x].[01, 01, 01, 01]; 59.48 + 59.49 +Td0[x] = Si[x].[0e, 09, 0d, 0b]; 59.50 +Td1[x] = Si[x].[0b, 0e, 09, 0d]; 59.51 +Td2[x] = Si[x].[0d, 0b, 0e, 09]; 59.52 +Td3[x] = Si[x].[09, 0d, 0b, 0e]; 59.53 +Td4[x] = Si[x].[01, 01, 01, 01]; 59.54 +*/ 59.55 + 59.56 +static const u32 Te0[256] = { 59.57 + 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, 59.58 + 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, 59.59 + 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU, 59.60 + 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU, 59.61 + 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, 59.62 + 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, 59.63 + 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, 59.64 + 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU, 59.65 + 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU, 59.66 + 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, 59.67 + 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, 59.68 + 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU, 59.69 + 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU, 59.70 + 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U, 59.71 + 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, 59.72 + 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, 59.73 + 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, 59.74 + 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU, 59.75 + 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU, 59.76 + 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, 59.77 + 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, 59.78 + 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, 59.79 + 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU, 59.80 + 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU, 59.81 + 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, 59.82 + 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, 59.83 + 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, 59.84 + 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U, 59.85 + 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU, 59.86 + 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, 59.87 + 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, 59.88 + 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, 59.89 + 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU, 59.90 + 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U, 59.91 + 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, 59.92 + 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, 59.93 + 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, 59.94 + 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U, 59.95 + 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU, 59.96 + 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, 59.97 + 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, 59.98 + 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, 59.99 + 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U, 59.100 + 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU, 59.101 + 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, 59.102 + 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, 59.103 + 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, 59.104 + 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U, 59.105 + 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U, 59.106 + 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, 59.107 + 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, 59.108 + 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, 59.109 + 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU, 59.110 + 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U, 59.111 + 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, 59.112 + 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, 59.113 + 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U, 59.114 + 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U, 59.115 + 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U, 59.116 + 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, 59.117 + 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, 59.118 + 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, 59.119 + 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U, 59.120 + 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU, 59.121 +}; 59.122 +static const u32 Te1[256] = { 59.123 + 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, 59.124 + 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, 59.125 + 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU, 59.126 + 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U, 59.127 + 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, 59.128 + 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, 59.129 + 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, 59.130 + 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U, 59.131 + 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U, 59.132 + 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, 59.133 + 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, 59.134 + 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, 59.135 + 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U, 59.136 + 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU, 59.137 + 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, 59.138 + 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, 59.139 + 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, 59.140 + 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U, 59.141 + 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U, 59.142 + 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, 59.143 + 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, 59.144 + 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, 59.145 + 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U, 59.146 + 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU, 59.147 + 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, 59.148 + 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, 59.149 + 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, 59.150 + 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U, 59.151 + 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU, 59.152 + 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, 59.153 + 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, 59.154 + 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, 59.155 + 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU, 59.156 + 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U, 59.157 + 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, 59.158 + 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, 59.159 + 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, 59.160 + 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U, 59.161 + 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U, 59.162 + 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, 59.163 + 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, 59.164 + 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, 59.165 + 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U, 59.166 + 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U, 59.167 + 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, 59.168 + 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, 59.169 + 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, 59.170 + 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U, 59.171 + 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU, 59.172 + 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, 59.173 + 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, 59.174 + 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, 59.175 + 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U, 59.176 + 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU, 59.177 + 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, 59.178 + 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, 59.179 + 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, 59.180 + 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U, 59.181 + 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U, 59.182 + 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, 59.183 + 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, 59.184 + 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, 59.185 + 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU, 59.186 + 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U, 59.187 +}; 59.188 +static const u32 Te2[256] = { 59.189 + 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, 59.190 + 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, 59.191 + 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU, 59.192 + 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U, 59.193 + 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, 59.194 + 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, 59.195 + 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, 59.196 + 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U, 59.197 + 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U, 59.198 + 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, 59.199 + 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, 59.200 + 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U, 59.201 + 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U, 59.202 + 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU, 59.203 + 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, 59.204 + 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, 59.205 + 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, 59.206 + 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U, 59.207 + 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U, 59.208 + 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, 59.209 + 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, 59.210 + 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, 59.211 + 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U, 59.212 + 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU, 59.213 + 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, 59.214 + 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, 59.215 + 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, 59.216 + 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U, 59.217 + 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU, 59.218 + 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, 59.219 + 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, 59.220 + 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, 59.221 + 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU, 59.222 + 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U, 59.223 + 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, 59.224 + 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, 59.225 + 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, 59.226 + 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U, 59.227 + 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U, 59.228 + 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, 59.229 + 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, 59.230 + 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, 59.231 + 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U, 59.232 + 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U, 59.233 + 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, 59.234 + 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, 59.235 + 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, 59.236 + 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U, 59.237 + 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU, 59.238 + 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, 59.239 + 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, 59.240 + 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, 59.241 + 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U, 59.242 + 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU, 59.243 + 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, 59.244 + 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, 59.245 + 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U, 59.246 + 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U, 59.247 + 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U, 59.248 + 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, 59.249 + 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, 59.250 + 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, 59.251 + 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU, 59.252 + 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U, 59.253 +}; 59.254 +static const u32 Te3[256] = { 59.255 + 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, 59.256 + 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, 59.257 + 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U, 59.258 + 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU, 59.259 + 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, 59.260 + 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, 59.261 + 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, 59.262 + 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU, 59.263 + 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU, 59.264 + 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, 59.265 + 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, 59.266 + 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU, 59.267 + 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU, 59.268 + 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU, 59.269 + 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, 59.270 + 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, 59.271 + 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, 59.272 + 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU, 59.273 + 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU, 59.274 + 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, 59.275 + 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, 59.276 + 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, 59.277 + 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U, 59.278 + 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U, 59.279 + 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, 59.280 + 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, 59.281 + 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, 59.282 + 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU, 59.283 + 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U, 59.284 + 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, 59.285 + 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, 59.286 + 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, 59.287 + 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U, 59.288 + 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU, 59.289 + 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, 59.290 + 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, 59.291 + 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, 59.292 + 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU, 59.293 + 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U, 59.294 + 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, 59.295 + 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, 59.296 + 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, 59.297 + 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U, 59.298 + 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U, 59.299 + 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, 59.300 + 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, 59.301 + 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, 59.302 + 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U, 59.303 + 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU, 59.304 + 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, 59.305 + 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, 59.306 + 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, 59.307 + 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU, 59.308 + 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU, 59.309 + 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, 59.310 + 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, 59.311 + 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U, 59.312 + 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U, 59.313 + 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U, 59.314 + 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, 59.315 + 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, 59.316 + 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, 59.317 + 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, 59.318 + 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU, 59.319 +}; 59.320 +static const u32 Te4[256] = { 59.321 + 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, 59.322 + 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, 59.323 + 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU, 59.324 + 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U, 59.325 + 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, 59.326 + 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, 59.327 + 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, 59.328 + 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U, 59.329 + 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U, 59.330 + 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, 59.331 + 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, 59.332 + 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U, 59.333 + 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U, 59.334 + 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU, 59.335 + 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, 59.336 + 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, 59.337 + 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, 59.338 + 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U, 59.339 + 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U, 59.340 + 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, 59.341 + 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, 59.342 + 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, 59.343 + 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U, 59.344 + 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU, 59.345 + 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, 59.346 + 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, 59.347 + 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, 59.348 + 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U, 59.349 + 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU, 59.350 + 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, 59.351 + 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, 59.352 + 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, 59.353 + 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU, 59.354 + 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U, 59.355 + 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, 59.356 + 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, 59.357 + 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, 59.358 + 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U, 59.359 + 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U, 59.360 + 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, 59.361 + 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, 59.362 + 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU, 59.363 + 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U, 59.364 + 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U, 59.365 + 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, 59.366 + 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, 59.367 + 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, 59.368 + 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U, 59.369 + 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU, 59.370 + 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, 59.371 + 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, 59.372 + 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, 59.373 + 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U, 59.374 + 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU, 59.375 + 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, 59.376 + 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, 59.377 + 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U, 59.378 + 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U, 59.379 + 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U, 59.380 + 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, 59.381 + 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, 59.382 + 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U, 59.383 + 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU, 59.384 + 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U, 59.385 +}; 59.386 +static const u32 Td0[256] = { 59.387 + 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U, 59.388 + 0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U, 59.389 + 0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U, 59.390 + 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU, 59.391 + 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U, 59.392 + 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U, 59.393 + 0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU, 59.394 + 0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U, 59.395 + 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU, 59.396 + 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U, 59.397 + 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U, 59.398 + 0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U, 59.399 + 0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U, 59.400 + 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU, 59.401 + 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U, 59.402 + 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU, 59.403 + 0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U, 59.404 + 0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU, 59.405 + 0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U, 59.406 + 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U, 59.407 + 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U, 59.408 + 0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU, 59.409 + 0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U, 59.410 + 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU, 59.411 + 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U, 59.412 + 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU, 59.413 + 0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U, 59.414 + 0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU, 59.415 + 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU, 59.416 + 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U, 59.417 + 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU, 59.418 + 0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U, 59.419 + 0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU, 59.420 + 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U, 59.421 + 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U, 59.422 + 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U, 59.423 + 0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU, 59.424 + 0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U, 59.425 + 0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U, 59.426 + 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU, 59.427 + 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U, 59.428 + 0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U, 59.429 + 0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U, 59.430 + 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U, 59.431 + 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U, 59.432 + 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU, 59.433 + 0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U, 59.434 + 0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U, 59.435 + 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U, 59.436 + 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U, 59.437 + 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U, 59.438 + 0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU, 59.439 + 0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU, 59.440 + 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU, 59.441 + 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU, 59.442 + 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U, 59.443 + 0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U, 59.444 + 0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU, 59.445 + 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU, 59.446 + 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U, 59.447 + 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU, 59.448 + 0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U, 59.449 + 0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U, 59.450 + 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U, 59.451 +}; 59.452 +static const u32 Td1[256] = { 59.453 + 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU, 59.454 + 0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U, 59.455 + 0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU, 59.456 + 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U, 59.457 + 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U, 59.458 + 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U, 59.459 + 0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U, 59.460 + 0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U, 59.461 + 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U, 59.462 + 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU, 59.463 + 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU, 59.464 + 0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU, 59.465 + 0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U, 59.466 + 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU, 59.467 + 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U, 59.468 + 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U, 59.469 + 0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U, 59.470 + 0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU, 59.471 + 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU, 59.472 + 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U, 59.473 + 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU, 59.474 + 0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U, 59.475 + 0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU, 59.476 + 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU, 59.477 + 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U, 59.478 + 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U, 59.479 + 0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U, 59.480 + 0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU, 59.481 + 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U, 59.482 + 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU, 59.483 + 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U, 59.484 + 0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U, 59.485 + 0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U, 59.486 + 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU, 59.487 + 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U, 59.488 + 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U, 59.489 + 0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U, 59.490 + 0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U, 59.491 + 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U, 59.492 + 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U, 59.493 + 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU, 59.494 + 0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU, 59.495 + 0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U, 59.496 + 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU, 59.497 + 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U, 59.498 + 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU, 59.499 + 0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU, 59.500 + 0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U, 59.501 + 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU, 59.502 + 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U, 59.503 + 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U, 59.504 + 0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U, 59.505 + 0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U, 59.506 + 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U, 59.507 + 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U, 59.508 + 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U, 59.509 + 0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU, 59.510 + 0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U, 59.511 + 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U, 59.512 + 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU, 59.513 + 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U, 59.514 + 0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U, 59.515 + 0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U, 59.516 + 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U, 59.517 +}; 59.518 +static const u32 Td2[256] = { 59.519 + 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U, 59.520 + 0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U, 59.521 + 0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U, 59.522 + 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U, 59.523 + 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU, 59.524 + 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U, 59.525 + 0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U, 59.526 + 0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U, 59.527 + 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U, 59.528 + 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU, 59.529 + 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U, 59.530 + 0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U, 59.531 + 0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU, 59.532 + 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U, 59.533 + 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U, 59.534 + 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U, 59.535 + 0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U, 59.536 + 0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U, 59.537 + 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U, 59.538 + 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU, 59.539 + 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U, 59.540 + 0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U, 59.541 + 0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U, 59.542 + 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U, 59.543 + 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U, 59.544 + 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU, 59.545 + 0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU, 59.546 + 0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U, 59.547 + 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU, 59.548 + 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U, 59.549 + 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU, 59.550 + 0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU, 59.551 + 0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU, 59.552 + 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU, 59.553 + 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U, 59.554 + 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U, 59.555 + 0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U, 59.556 + 0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U, 59.557 + 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U, 59.558 + 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U, 59.559 + 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U, 59.560 + 0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU, 59.561 + 0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU, 59.562 + 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U, 59.563 + 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U, 59.564 + 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU, 59.565 + 0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU, 59.566 + 0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U, 59.567 + 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U, 59.568 + 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U, 59.569 + 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U, 59.570 + 0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U, 59.571 + 0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U, 59.572 + 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U, 59.573 + 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU, 59.574 + 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U, 59.575 + 0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U, 59.576 + 0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U, 59.577 + 0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U, 59.578 + 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U, 59.579 + 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U, 59.580 + 0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU, 59.581 + 0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U, 59.582 + 0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U, 59.583 +}; 59.584 +static const u32 Td3[256] = { 59.585 + 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU, 59.586 + 0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU, 59.587 + 0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U, 59.588 + 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U, 59.589 + 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU, 59.590 + 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU, 59.591 + 0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U, 59.592 + 0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU, 59.593 + 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U, 59.594 + 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU, 59.595 + 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U, 59.596 + 0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U, 59.597 + 0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U, 59.598 + 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U, 59.599 + 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U, 59.600 + 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU, 59.601 + 0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU, 59.602 + 0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U, 59.603 + 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U, 59.604 + 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU, 59.605 + 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU, 59.606 + 0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U, 59.607 + 0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U, 59.608 + 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U, 59.609 + 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U, 59.610 + 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU, 59.611 + 0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U, 59.612 + 0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U, 59.613 + 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU, 59.614 + 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU, 59.615 + 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U, 59.616 + 0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U, 59.617 + 0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U, 59.618 + 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU, 59.619 + 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U, 59.620 + 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U, 59.621 + 0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U, 59.622 + 0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U, 59.623 + 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U, 59.624 + 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U, 59.625 + 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U, 59.626 + 0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU, 59.627 + 0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U, 59.628 + 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U, 59.629 + 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU, 59.630 + 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU, 59.631 + 0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U, 59.632 + 0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU, 59.633 + 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U, 59.634 + 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U, 59.635 + 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U, 59.636 + 0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U, 59.637 + 0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U, 59.638 + 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U, 59.639 + 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU, 59.640 + 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU, 59.641 + 0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU, 59.642 + 0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU, 59.643 + 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U, 59.644 + 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U, 59.645 + 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U, 59.646 + 0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU, 59.647 + 0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U, 59.648 + 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U, 59.649 +}; 59.650 +static const u32 Td4[256] = { 59.651 + 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U, 59.652 + 0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U, 59.653 + 0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU, 59.654 + 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU, 59.655 + 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U, 59.656 + 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U, 59.657 + 0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U, 59.658 + 0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU, 59.659 + 0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U, 59.660 + 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU, 59.661 + 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU, 59.662 + 0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU, 59.663 + 0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U, 59.664 + 0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U, 59.665 + 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U, 59.666 + 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U, 59.667 + 0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U, 59.668 + 0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U, 59.669 + 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU, 59.670 + 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U, 59.671 + 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U, 59.672 + 0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU, 59.673 + 0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U, 59.674 + 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U, 59.675 + 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U, 59.676 + 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU, 59.677 + 0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U, 59.678 + 0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U, 59.679 + 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU, 59.680 + 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U, 59.681 + 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U, 59.682 + 0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU, 59.683 + 0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U, 59.684 + 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU, 59.685 + 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU, 59.686 + 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U, 59.687 + 0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U, 59.688 + 0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U, 59.689 + 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U, 59.690 + 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU, 59.691 + 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U, 59.692 + 0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U, 59.693 + 0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU, 59.694 + 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU, 59.695 + 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU, 59.696 + 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U, 59.697 + 0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU, 59.698 + 0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U, 59.699 + 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U, 59.700 + 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U, 59.701 + 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U, 59.702 + 0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU, 59.703 + 0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U, 59.704 + 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU, 59.705 + 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU, 59.706 + 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU, 59.707 + 0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU, 59.708 + 0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U, 59.709 + 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU, 59.710 + 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U, 59.711 + 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU, 59.712 + 0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U, 59.713 + 0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U, 59.714 + 0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU, 59.715 +}; 59.716 +static const u32 rcon[] = { 59.717 + 0x01000000, 0x02000000, 0x04000000, 0x08000000, 59.718 + 0x10000000, 0x20000000, 0x40000000, 0x80000000, 59.719 + 0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ 59.720 +}; 59.721 + 59.722 +#define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] << 8) ^ ((u32)(pt)[3])) 59.723 +#define PUTU32(ct, st) { (ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); (ct)[2] = (u8)((st) >> 8); (ct)[3] = (u8)(st); } 59.724 + 59.725 +/** 59.726 + * Expand the cipher key into the encryption key schedule. 59.727 + * 59.728 + * @return the number of rounds for the given cipher key size. 59.729 + */ 59.730 +int 59.731 +rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], int keyBits) 59.732 +{ 59.733 + int i = 0; 59.734 + u32 temp; 59.735 + 59.736 + rk[0] = GETU32(cipherKey ); 59.737 + rk[1] = GETU32(cipherKey + 4); 59.738 + rk[2] = GETU32(cipherKey + 8); 59.739 + rk[3] = GETU32(cipherKey + 12); 59.740 + if (keyBits == 128) { 59.741 + for (;;) { 59.742 + temp = rk[3]; 59.743 + rk[4] = rk[0] ^ 59.744 + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ 59.745 + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ 59.746 + (Te4[(temp ) & 0xff] & 0x0000ff00) ^ 59.747 + (Te4[(temp >> 24) ] & 0x000000ff) ^ 59.748 + rcon[i]; 59.749 + rk[5] = rk[1] ^ rk[4]; 59.750 + rk[6] = rk[2] ^ rk[5]; 59.751 + rk[7] = rk[3] ^ rk[6]; 59.752 + if (++i == 10) { 59.753 + return 10; 59.754 + } 59.755 + rk += 4; 59.756 + } 59.757 + } 59.758 + rk[4] = GETU32(cipherKey + 16); 59.759 + rk[5] = GETU32(cipherKey + 20); 59.760 + if (keyBits == 192) { 59.761 + for (;;) { 59.762 + temp = rk[ 5]; 59.763 + rk[ 6] = rk[ 0] ^ 59.764 + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ 59.765 + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ 59.766 + (Te4[(temp ) & 0xff] & 0x0000ff00) ^ 59.767 + (Te4[(temp >> 24) ] & 0x000000ff) ^ 59.768 + rcon[i]; 59.769 + rk[ 7] = rk[ 1] ^ rk[ 6]; 59.770 + rk[ 8] = rk[ 2] ^ rk[ 7]; 59.771 + rk[ 9] = rk[ 3] ^ rk[ 8]; 59.772 + if (++i == 8) { 59.773 + return 12; 59.774 + } 59.775 + rk[10] = rk[ 4] ^ rk[ 9]; 59.776 + rk[11] = rk[ 5] ^ rk[10]; 59.777 + rk += 6; 59.778 + } 59.779 + } 59.780 + rk[6] = GETU32(cipherKey + 24); 59.781 + rk[7] = GETU32(cipherKey + 28); 59.782 + if (keyBits == 256) { 59.783 + for (;;) { 59.784 + temp = rk[ 7]; 59.785 + rk[ 8] = rk[ 0] ^ 59.786 + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ 59.787 + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ 59.788 + (Te4[(temp ) & 0xff] & 0x0000ff00) ^ 59.789 + (Te4[(temp >> 24) ] & 0x000000ff) ^ 59.790 + rcon[i]; 59.791 + rk[ 9] = rk[ 1] ^ rk[ 8]; 59.792 + rk[10] = rk[ 2] ^ rk[ 9]; 59.793 + rk[11] = rk[ 3] ^ rk[10]; 59.794 + if (++i == 7) { 59.795 + return 14; 59.796 + } 59.797 + temp = rk[11]; 59.798 + rk[12] = rk[ 4] ^ 59.799 + (Te4[(temp >> 24) ] & 0xff000000) ^ 59.800 + (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^ 59.801 + (Te4[(temp >> 8) & 0xff] & 0x0000ff00) ^ 59.802 + (Te4[(temp ) & 0xff] & 0x000000ff); 59.803 + rk[13] = rk[ 5] ^ rk[12]; 59.804 + rk[14] = rk[ 6] ^ rk[13]; 59.805 + rk[15] = rk[ 7] ^ rk[14]; 59.806 + rk += 8; 59.807 + } 59.808 + } 59.809 + return 0; 59.810 +} 59.811 + 59.812 +/** 59.813 + * Expand the cipher key into the decryption key schedule. 59.814 + * 59.815 + * @return the number of rounds for the given cipher key size. 59.816 + */ 59.817 +int 59.818 +rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], int keyBits) 59.819 +{ 59.820 + int Nr, i, j; 59.821 + u32 temp; 59.822 + 59.823 + /* expand the cipher key: */ 59.824 + Nr = rijndaelKeySetupEnc(rk, cipherKey, keyBits); 59.825 + 59.826 + /* invert the order of the round keys: */ 59.827 + for (i = 0, j = 4*Nr; i < j; i += 4, j -= 4) { 59.828 + temp = rk[i ]; rk[i ] = rk[j ]; rk[j ] = temp; 59.829 + temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp; 59.830 + temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp; 59.831 + temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp; 59.832 + } 59.833 + /* apply the inverse MixColumn transform to all round keys but the first and the last: */ 59.834 + for (i = 1; i < Nr; i++) { 59.835 + rk += 4; 59.836 + rk[0] = 59.837 + Td0[Te4[(rk[0] >> 24) ] & 0xff] ^ 59.838 + Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^ 59.839 + Td2[Te4[(rk[0] >> 8) & 0xff] & 0xff] ^ 59.840 + Td3[Te4[(rk[0] ) & 0xff] & 0xff]; 59.841 + rk[1] = 59.842 + Td0[Te4[(rk[1] >> 24) ] & 0xff] ^ 59.843 + Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^ 59.844 + Td2[Te4[(rk[1] >> 8) & 0xff] & 0xff] ^ 59.845 + Td3[Te4[(rk[1] ) & 0xff] & 0xff]; 59.846 + rk[2] = 59.847 + Td0[Te4[(rk[2] >> 24) ] & 0xff] ^ 59.848 + Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^ 59.849 + Td2[Te4[(rk[2] >> 8) & 0xff] & 0xff] ^ 59.850 + Td3[Te4[(rk[2] ) & 0xff] & 0xff]; 59.851 + rk[3] = 59.852 + Td0[Te4[(rk[3] >> 24) ] & 0xff] ^ 59.853 + Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^ 59.854 + Td2[Te4[(rk[3] >> 8) & 0xff] & 0xff] ^ 59.855 + Td3[Te4[(rk[3] ) & 0xff] & 0xff]; 59.856 + } 59.857 + return Nr; 59.858 +} 59.859 + 59.860 +void 59.861 +rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], 59.862 + u8 ct[16]) 59.863 +{ 59.864 + u32 s0, s1, s2, s3, t0, t1, t2, t3; 59.865 +#ifndef FULL_UNROLL 59.866 + int r; 59.867 +#endif /* ?FULL_UNROLL */ 59.868 + 59.869 + /* 59.870 + * map byte array block to cipher state 59.871 + * and add initial round key: 59.872 + */ 59.873 + s0 = GETU32(pt ) ^ rk[0]; 59.874 + s1 = GETU32(pt + 4) ^ rk[1]; 59.875 + s2 = GETU32(pt + 8) ^ rk[2]; 59.876 + s3 = GETU32(pt + 12) ^ rk[3]; 59.877 +#ifdef FULL_UNROLL 59.878 + /* round 1: */ 59.879 + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[ 4]; 59.880 + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[ 5]; 59.881 + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[ 6]; 59.882 + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[ 7]; 59.883 + /* round 2: */ 59.884 + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[ 8]; 59.885 + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[ 9]; 59.886 + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[10]; 59.887 + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[11]; 59.888 + /* round 3: */ 59.889 + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[12]; 59.890 + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[13]; 59.891 + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[14]; 59.892 + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[15]; 59.893 + /* round 4: */ 59.894 + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[16]; 59.895 + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[17]; 59.896 + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[18]; 59.897 + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[19]; 59.898 + /* round 5: */ 59.899 + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[20]; 59.900 + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[21]; 59.901 + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[22]; 59.902 + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[23]; 59.903 + /* round 6: */ 59.904 + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[24]; 59.905 + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[25]; 59.906 + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[26]; 59.907 + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[27]; 59.908 + /* round 7: */ 59.909 + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[28]; 59.910 + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[29]; 59.911 + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[30]; 59.912 + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[31]; 59.913 + /* round 8: */ 59.914 + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[32]; 59.915 + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[33]; 59.916 + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[34]; 59.917 + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[35]; 59.918 + /* round 9: */ 59.919 + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[36]; 59.920 + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[37]; 59.921 + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[38]; 59.922 + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[39]; 59.923 + if (Nr > 10) { 59.924 + /* round 10: */ 59.925 + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[40]; 59.926 + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[41]; 59.927 + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[42]; 59.928 + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[43]; 59.929 + /* round 11: */ 59.930 + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[44]; 59.931 + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[45]; 59.932 + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[46]; 59.933 + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[47]; 59.934 + if (Nr > 12) { 59.935 + /* round 12: */ 59.936 + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[48]; 59.937 + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[49]; 59.938 + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[50]; 59.939 + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[51]; 59.940 + /* round 13: */ 59.941 + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[52]; 59.942 + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[53]; 59.943 + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[54]; 59.944 + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[55]; 59.945 + } 59.946 + } 59.947 + rk += Nr << 2; 59.948 +#else /* !FULL_UNROLL */ 59.949 + /* 59.950 + * Nr - 1 full rounds: 59.951 + */ 59.952 + r = Nr >> 1; 59.953 + for (;;) { 59.954 + t0 = 59.955 + Te0[(s0 >> 24) ] ^ 59.956 + Te1[(s1 >> 16) & 0xff] ^ 59.957 + Te2[(s2 >> 8) & 0xff] ^ 59.958 + Te3[(s3 ) & 0xff] ^ 59.959 + rk[4]; 59.960 + t1 = 59.961 + Te0[(s1 >> 24) ] ^ 59.962 + Te1[(s2 >> 16) & 0xff] ^ 59.963 + Te2[(s3 >> 8) & 0xff] ^ 59.964 + Te3[(s0 ) & 0xff] ^ 59.965 + rk[5]; 59.966 + t2 = 59.967 + Te0[(s2 >> 24) ] ^ 59.968 + Te1[(s3 >> 16) & 0xff] ^ 59.969 + Te2[(s0 >> 8) & 0xff] ^ 59.970 + Te3[(s1 ) & 0xff] ^ 59.971 + rk[6]; 59.972 + t3 = 59.973 + Te0[(s3 >> 24) ] ^ 59.974 + Te1[(s0 >> 16) & 0xff] ^ 59.975 + Te2[(s1 >> 8) & 0xff] ^ 59.976 + Te3[(s2 ) & 0xff] ^ 59.977 + rk[7]; 59.978 + 59.979 + rk += 8; 59.980 + if (--r == 0) { 59.981 + break; 59.982 + } 59.983 + 59.984 + s0 = 59.985 + Te0[(t0 >> 24) ] ^ 59.986 + Te1[(t1 >> 16) & 0xff] ^ 59.987 + Te2[(t2 >> 8) & 0xff] ^ 59.988 + Te3[(t3 ) & 0xff] ^ 59.989 + rk[0]; 59.990 + s1 = 59.991 + Te0[(t1 >> 24) ] ^ 59.992 + Te1[(t2 >> 16) & 0xff] ^ 59.993 + Te2[(t3 >> 8) & 0xff] ^ 59.994 + Te3[(t0 ) & 0xff] ^ 59.995 + rk[1]; 59.996 + s2 = 59.997 + Te0[(t2 >> 24) ] ^ 59.998 + Te1[(t3 >> 16) & 0xff] ^ 59.999 + Te2[(t0 >> 8) & 0xff] ^ 59.1000 + Te3[(t1 ) & 0xff] ^ 59.1001 + rk[2]; 59.1002 + s3 = 59.1003 + Te0[(t3 >> 24) ] ^ 59.1004 + Te1[(t0 >> 16) & 0xff] ^ 59.1005 + Te2[(t1 >> 8) & 0xff] ^ 59.1006 + Te3[(t2 ) & 0xff] ^ 59.1007 + rk[3]; 59.1008 + } 59.1009 +#endif /* ?FULL_UNROLL */ 59.1010 + /* 59.1011 + * apply last round and 59.1012 + * map cipher state to byte array block: 59.1013 + */ 59.1014 + s0 = 59.1015 + (Te4[(t0 >> 24) ] & 0xff000000) ^ 59.1016 + (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ 59.1017 + (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ 59.1018 + (Te4[(t3 ) & 0xff] & 0x000000ff) ^ 59.1019 + rk[0]; 59.1020 + PUTU32(ct , s0); 59.1021 + s1 = 59.1022 + (Te4[(t1 >> 24) ] & 0xff000000) ^ 59.1023 + (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ 59.1024 + (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ 59.1025 + (Te4[(t0 ) & 0xff] & 0x000000ff) ^ 59.1026 + rk[1]; 59.1027 + PUTU32(ct + 4, s1); 59.1028 + s2 = 59.1029 + (Te4[(t2 >> 24) ] & 0xff000000) ^ 59.1030 + (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ 59.1031 + (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ 59.1032 + (Te4[(t1 ) & 0xff] & 0x000000ff) ^ 59.1033 + rk[2]; 59.1034 + PUTU32(ct + 8, s2); 59.1035 + s3 = 59.1036 + (Te4[(t3 >> 24) ] & 0xff000000) ^ 59.1037 + (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ 59.1038 + (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ 59.1039 + (Te4[(t2 ) & 0xff] & 0x000000ff) ^ 59.1040 + rk[3]; 59.1041 + PUTU32(ct + 12, s3); 59.1042 +} 59.1043 + 59.1044 +static void 59.1045 +rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], 59.1046 + u8 pt[16]) 59.1047 +{ 59.1048 + u32 s0, s1, s2, s3, t0, t1, t2, t3; 59.1049 +#ifndef FULL_UNROLL 59.1050 + int r; 59.1051 +#endif /* ?FULL_UNROLL */ 59.1052 + 59.1053 + /* 59.1054 + * map byte array block to cipher state 59.1055 + * and add initial round key: 59.1056 + */ 59.1057 + s0 = GETU32(ct ) ^ rk[0]; 59.1058 + s1 = GETU32(ct + 4) ^ rk[1]; 59.1059 + s2 = GETU32(ct + 8) ^ rk[2]; 59.1060 + s3 = GETU32(ct + 12) ^ rk[3]; 59.1061 +#ifdef FULL_UNROLL 59.1062 + /* round 1: */ 59.1063 + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[ 4]; 59.1064 + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[ 5]; 59.1065 + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[ 6]; 59.1066 + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[ 7]; 59.1067 + /* round 2: */ 59.1068 + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[ 8]; 59.1069 + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[ 9]; 59.1070 + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[10]; 59.1071 + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[11]; 59.1072 + /* round 3: */ 59.1073 + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[12]; 59.1074 + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[13]; 59.1075 + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[14]; 59.1076 + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[15]; 59.1077 + /* round 4: */ 59.1078 + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[16]; 59.1079 + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[17]; 59.1080 + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[18]; 59.1081 + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[19]; 59.1082 + /* round 5: */ 59.1083 + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[20]; 59.1084 + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[21]; 59.1085 + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[22]; 59.1086 + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[23]; 59.1087 + /* round 6: */ 59.1088 + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[24]; 59.1089 + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[25]; 59.1090 + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[26]; 59.1091 + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[27]; 59.1092 + /* round 7: */ 59.1093 + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[28]; 59.1094 + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[29]; 59.1095 + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[30]; 59.1096 + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[31]; 59.1097 + /* round 8: */ 59.1098 + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[32]; 59.1099 + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[33]; 59.1100 + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[34]; 59.1101 + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[35]; 59.1102 + /* round 9: */ 59.1103 + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[36]; 59.1104 + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[37]; 59.1105 + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[38]; 59.1106 + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[39]; 59.1107 + if (Nr > 10) { 59.1108 + /* round 10: */ 59.1109 + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[40]; 59.1110 + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[41]; 59.1111 + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[42]; 59.1112 + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[43]; 59.1113 + /* round 11: */ 59.1114 + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[44]; 59.1115 + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[45]; 59.1116 + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[46]; 59.1117 + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[47]; 59.1118 + if (Nr > 12) { 59.1119 + /* round 12: */ 59.1120 + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[48]; 59.1121 + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[49]; 59.1122 + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[50]; 59.1123 + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[51]; 59.1124 + /* round 13: */ 59.1125 + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[52]; 59.1126 + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[53]; 59.1127 + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[54]; 59.1128 + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[55]; 59.1129 + } 59.1130 + } 59.1131 + rk += Nr << 2; 59.1132 +#else /* !FULL_UNROLL */ 59.1133 + /* 59.1134 + * Nr - 1 full rounds: 59.1135 + */ 59.1136 + r = Nr >> 1; 59.1137 + for (;;) { 59.1138 + t0 = 59.1139 + Td0[(s0 >> 24) ] ^ 59.1140 + Td1[(s3 >> 16) & 0xff] ^ 59.1141 + Td2[(s2 >> 8) & 0xff] ^ 59.1142 + Td3[(s1 ) & 0xff] ^ 59.1143 + rk[4]; 59.1144 + t1 = 59.1145 + Td0[(s1 >> 24) ] ^ 59.1146 + Td1[(s0 >> 16) & 0xff] ^ 59.1147 + Td2[(s3 >> 8) & 0xff] ^ 59.1148 + Td3[(s2 ) & 0xff] ^ 59.1149 + rk[5]; 59.1150 + t2 = 59.1151 + Td0[(s2 >> 24) ] ^ 59.1152 + Td1[(s1 >> 16) & 0xff] ^ 59.1153 + Td2[(s0 >> 8) & 0xff] ^ 59.1154 + Td3[(s3 ) & 0xff] ^ 59.1155 + rk[6]; 59.1156 + t3 = 59.1157 + Td0[(s3 >> 24) ] ^ 59.1158 + Td1[(s2 >> 16) & 0xff] ^ 59.1159 + Td2[(s1 >> 8) & 0xff] ^ 59.1160 + Td3[(s0 ) & 0xff] ^ 59.1161 + rk[7]; 59.1162 + 59.1163 + rk += 8; 59.1164 + if (--r == 0) { 59.1165 + break; 59.1166 + } 59.1167 + 59.1168 + s0 = 59.1169 + Td0[(t0 >> 24) ] ^ 59.1170 + Td1[(t3 >> 16) & 0xff] ^ 59.1171 + Td2[(t2 >> 8) & 0xff] ^ 59.1172 + Td3[(t1 ) & 0xff] ^ 59.1173 + rk[0]; 59.1174 + s1 = 59.1175 + Td0[(t1 >> 24) ] ^ 59.1176 + Td1[(t0 >> 16) & 0xff] ^ 59.1177 + Td2[(t3 >> 8) & 0xff] ^ 59.1178 + Td3[(t2 ) & 0xff] ^ 59.1179 + rk[1]; 59.1180 + s2 = 59.1181 + Td0[(t2 >> 24) ] ^ 59.1182 + Td1[(t1 >> 16) & 0xff] ^ 59.1183 + Td2[(t0 >> 8) & 0xff] ^ 59.1184 + Td3[(t3 ) & 0xff] ^ 59.1185 + rk[2]; 59.1186 + s3 = 59.1187 + Td0[(t3 >> 24) ] ^ 59.1188 + Td1[(t2 >> 16) & 0xff] ^ 59.1189 + Td2[(t1 >> 8) & 0xff] ^ 59.1190 + Td3[(t0 ) & 0xff] ^ 59.1191 + rk[3]; 59.1192 + } 59.1193 +#endif /* ?FULL_UNROLL */ 59.1194 + /* 59.1195 + * apply last round and 59.1196 + * map cipher state to byte array block: 59.1197 + */ 59.1198 + s0 = 59.1199 + (Td4[(t0 >> 24) ] & 0xff000000) ^ 59.1200 + (Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ 59.1201 + (Td4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ 59.1202 + (Td4[(t1 ) & 0xff] & 0x000000ff) ^ 59.1203 + rk[0]; 59.1204 + PUTU32(pt , s0); 59.1205 + s1 = 59.1206 + (Td4[(t1 >> 24) ] & 0xff000000) ^ 59.1207 + (Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ 59.1208 + (Td4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ 59.1209 + (Td4[(t2 ) & 0xff] & 0x000000ff) ^ 59.1210 + rk[1]; 59.1211 + PUTU32(pt + 4, s1); 59.1212 + s2 = 59.1213 + (Td4[(t2 >> 24) ] & 0xff000000) ^ 59.1214 + (Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ 59.1215 + (Td4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ 59.1216 + (Td4[(t3 ) & 0xff] & 0x000000ff) ^ 59.1217 + rk[2]; 59.1218 + PUTU32(pt + 8, s2); 59.1219 + s3 = 59.1220 + (Td4[(t3 >> 24) ] & 0xff000000) ^ 59.1221 + (Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ 59.1222 + (Td4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ 59.1223 + (Td4[(t0 ) & 0xff] & 0x000000ff) ^ 59.1224 + rk[3]; 59.1225 + PUTU32(pt + 12, s3); 59.1226 +} 59.1227 + 59.1228 +/* setup key context for encryption only */ 59.1229 +int 59.1230 +rijndael_set_key_enc_only(rijndael_ctx *ctx, const u_char *key, int bits) 59.1231 +{ 59.1232 + int rounds; 59.1233 + 59.1234 + rounds = rijndaelKeySetupEnc(ctx->ek, key, bits); 59.1235 + if (rounds == 0) 59.1236 + return -1; 59.1237 + 59.1238 + ctx->Nr = rounds; 59.1239 + ctx->enc_only = 1; 59.1240 + 59.1241 + return 0; 59.1242 +} 59.1243 + 59.1244 +/* setup key context for both encryption and decryption */ 59.1245 +int 59.1246 +rijndael_set_key(rijndael_ctx *ctx, const u_char *key, int bits) 59.1247 +{ 59.1248 + int rounds; 59.1249 + 59.1250 + rounds = rijndaelKeySetupEnc(ctx->ek, key, bits); 59.1251 + if (rounds == 0) 59.1252 + return -1; 59.1253 + if (rijndaelKeySetupDec(ctx->dk, key, bits) != rounds) 59.1254 + return -1; 59.1255 + 59.1256 + ctx->Nr = rounds; 59.1257 + ctx->enc_only = 0; 59.1258 + 59.1259 + return 0; 59.1260 +} 59.1261 + 59.1262 +void 59.1263 +rijndael_decrypt(rijndael_ctx *ctx, const u_char *src, u_char *dst) 59.1264 +{ 59.1265 + rijndaelDecrypt(ctx->dk, ctx->Nr, src, dst); 59.1266 +} 59.1267 + 59.1268 +void 59.1269 +rijndael_encrypt(rijndael_ctx *ctx, const u_char *src, u_char *dst) 59.1270 +{ 59.1271 + rijndaelEncrypt(ctx->ek, ctx->Nr, src, dst); 59.1272 +}
60.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 60.2 +++ b/xen/crypto/vmac.c Fri Mar 06 12:22:22 2009 +0900 60.3 @@ -0,0 +1,1220 @@ 60.4 +/* -------------------------------------------------------------------------- 60.5 + * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai. 60.6 + * This implementation is herby placed in the public domain. 60.7 + * The authors offers no warranty. Use at your own risk. 60.8 + * Please send bug reports to the authors. 60.9 + * Last modified: 17 APR 08, 1700 PDT 60.10 + * ----------------------------------------------------------------------- */ 60.11 + 60.12 +/* start for Xen */ 60.13 +#include <xen/config.h> 60.14 +#include <xen/init.h> 60.15 +#include <xen/types.h> 60.16 +#include <xen/lib.h> 60.17 +#include <crypto/vmac.h> 60.18 +#define UINT64_C(x) x##ULL 60.19 +/* end for Xen */ 60.20 + 60.21 +/* Enable code tuned for 64-bit registers; otherwise tuned for 32-bit */ 60.22 +#ifndef VMAC_ARCH_64 60.23 +#define VMAC_ARCH_64 (__x86_64__ || __ppc64__ || _M_X64) 60.24 +#endif 60.25 + 60.26 +/* Enable code tuned for Intel SSE2 instruction set */ 60.27 +#if ((__SSE2__ || (_M_IX86_FP >= 2)) && ( ! VMAC_ARCH_64)) 60.28 +#define VMAC_USE_SSE2 1 60.29 +#include <emmintrin.h> 60.30 +#endif 60.31 + 60.32 +/* Native word reads. Update (or define via compiler) if incorrect */ 60.33 +#ifndef VMAC_ARCH_BIG_ENDIAN /* Assume big-endian unless on the list */ 60.34 +#define VMAC_ARCH_BIG_ENDIAN \ 60.35 + (!(__x86_64__ || __i386__ || _M_IX86 || \ 60.36 + _M_X64 || __ARMEL__ || __MIPSEL__)) 60.37 +#endif 60.38 + 60.39 +/* ----------------------------------------------------------------------- */ 60.40 +/* Constants and masks */ 60.41 + 60.42 +const uint64_t p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */ 60.43 +const uint64_t m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */ 60.44 +const uint64_t m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */ 60.45 +const uint64_t m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */ 60.46 +const uint64_t mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */ 60.47 + 60.48 +/* ----------------------------------------------------------------------- * 60.49 + * The following routines are used in this implementation. They are 60.50 + * written via macros to simulate zero-overhead call-by-reference. 60.51 + * All have default implemantations for when they are not defined in an 60.52 + * architecture-specific manner. 60.53 + * 60.54 + * MUL64: 64x64->128-bit multiplication 60.55 + * PMUL64: assumes top bits cleared on inputs 60.56 + * ADD128: 128x128->128-bit addition 60.57 + * GET_REVERSED_64: load and byte-reverse 64-bit word 60.58 + * ----------------------------------------------------------------------- */ 60.59 + 60.60 +/* ----------------------------------------------------------------------- */ 60.61 +#if (__GNUC__ && (__x86_64__ || __amd64__)) 60.62 +/* ----------------------------------------------------------------------- */ 60.63 + 60.64 +#define ADD128(rh,rl,ih,il) \ 60.65 + asm ("addq %3, %1 \n\t" \ 60.66 + "adcq %2, %0" \ 60.67 + : "+r"(rh),"+r"(rl) \ 60.68 + : "r"(ih),"r"(il) : "cc"); 60.69 + 60.70 +#define MUL64(rh,rl,i1,i2) \ 60.71 + asm ("mulq %3" : "=a"(rl), "=d"(rh) : "a"(i1), "r"(i2) : "cc") 60.72 + 60.73 +#define PMUL64 MUL64 60.74 + 60.75 +#define GET_REVERSED_64(p) \ 60.76 + ({uint64_t x; \ 60.77 + asm ("bswapq %0" : "=r" (x) : "0"(*(uint64_t *)(p))); x;}) 60.78 + 60.79 +/* ----------------------------------------------------------------------- */ 60.80 +#elif (__GNUC__ && __i386__) 60.81 +/* ----------------------------------------------------------------------- */ 60.82 + 60.83 +#define GET_REVERSED_64(p) \ 60.84 + ({ uint64_t x; \ 60.85 + uint32_t *tp = (uint32_t *)(p); \ 60.86 + asm ("bswap %%edx\n\t" \ 60.87 + "bswap %%eax" \ 60.88 + : "=A"(x) \ 60.89 + : "a"(tp[1]), "d"(tp[0])); \ 60.90 + x; }) 60.91 + 60.92 +/* ----------------------------------------------------------------------- */ 60.93 +#elif (__GNUC__ && __ppc64__) 60.94 +/* ----------------------------------------------------------------------- */ 60.95 + 60.96 +#define ADD128(rh,rl,ih,il) \ 60.97 + asm volatile ( "addc %1, %1, %3 \n\t" \ 60.98 + "adde %0, %0, %2" \ 60.99 + : "+r"(rh),"+r"(rl) \ 60.100 + : "r"(ih),"r"(il)); 60.101 + 60.102 +#define MUL64(rh,rl,i1,i2) \ 60.103 +{ uint64_t _i1 = (i1), _i2 = (i2); \ 60.104 + rl = _i1 * _i2; \ 60.105 + asm volatile ("mulhdu %0, %1, %2" : "=r" (rh) : "r" (_i1), "r" (_i2));\ 60.106 +} 60.107 + 60.108 +#define PMUL64 MUL64 60.109 + 60.110 +#define GET_REVERSED_64(p) \ 60.111 + ({ uint32_t hi, lo, *_p = (uint32_t *)(p); \ 60.112 + asm volatile ("lwbrx %0, %1, %2" : "=r"(lo) : "b%"(0), "r"(_p) ); \ 60.113 + asm volatile ("lwbrx %0, %1, %2" : "=r"(hi) : "b%"(4), "r"(_p) ); \ 60.114 + ((uint64_t)hi << 32) | (uint64_t)lo; } ) 60.115 + 60.116 +/* ----------------------------------------------------------------------- */ 60.117 +#elif (__GNUC__ && (__ppc__ || __PPC__)) 60.118 +/* ----------------------------------------------------------------------- */ 60.119 + 60.120 +#define GET_REVERSED_64(p) \ 60.121 + ({ uint32_t hi, lo, *_p = (uint32_t *)(p); \ 60.122 + asm volatile ("lwbrx %0, %1, %2" : "=r"(lo) : "b%"(0), "r"(_p) ); \ 60.123 + asm volatile ("lwbrx %0, %1, %2" : "=r"(hi) : "b%"(4), "r"(_p) ); \ 60.124 + ((uint64_t)hi << 32) | (uint64_t)lo; } ) 60.125 + 60.126 +/* ----------------------------------------------------------------------- */ 60.127 +#elif (__GNUC__ && (__ARMEL__ || __ARM__)) 60.128 +/* ----------------------------------------------------------------------- */ 60.129 + 60.130 +#define bswap32(v) \ 60.131 +({ uint32_t tmp,out; \ 60.132 + asm volatile( \ 60.133 + "eor %1, %2, %2, ror #16\n" \ 60.134 + "bic %1, %1, #0x00ff0000\n" \ 60.135 + "mov %0, %2, ror #8\n" \ 60.136 + "eor %0, %0, %1, lsr #8" \ 60.137 + : "=r" (out), "=&r" (tmp) \ 60.138 + : "r" (v)); \ 60.139 + out;}) 60.140 + 60.141 +/* ----------------------------------------------------------------------- */ 60.142 +#elif _MSC_VER 60.143 +/* ----------------------------------------------------------------------- */ 60.144 + 60.145 +#include <intrin.h> 60.146 + 60.147 +#if (_M_IA64 || _M_X64) && \ 60.148 + (!defined(__INTEL_COMPILER) || __INTEL_COMPILER >= 1000) 60.149 +#define MUL64(rh,rl,i1,i2) (rl) = _umul128(i1,i2,&(rh)); 60.150 +#pragma intrinsic(_umul128) 60.151 +#define PMUL64 MUL64 60.152 +#endif 60.153 + 60.154 +/* MSVC uses add, adc in this version */ 60.155 +#define ADD128(rh,rl,ih,il) \ 60.156 + { uint64_t _il = (il); \ 60.157 + (rl) += (_il); \ 60.158 + (rh) += (ih) + ((rl) < (_il)); \ 60.159 + } 60.160 + 60.161 +#if _MSC_VER >= 1300 60.162 +#define GET_REVERSED_64(p) _byteswap_uint64(*(uint64_t *)(p)) 60.163 +#pragma intrinsic(_byteswap_uint64) 60.164 +#endif 60.165 + 60.166 +#if _MSC_VER >= 1400 && \ 60.167 + (!defined(__INTEL_COMPILER) || __INTEL_COMPILER >= 1000) 60.168 +#define MUL32(i1,i2) (__emulu((uint32_t)(i1),(uint32_t)(i2))) 60.169 +#pragma intrinsic(__emulu) 60.170 +#endif 60.171 + 60.172 +/* ----------------------------------------------------------------------- */ 60.173 +#endif 60.174 +/* ----------------------------------------------------------------------- */ 60.175 + 60.176 +#if __GNUC__ 60.177 +#define ALIGN(n) __attribute__ ((aligned(n))) 60.178 +#define NOINLINE __attribute__ ((noinline)) 60.179 +#define FASTCALL 60.180 +#elif _MSC_VER 60.181 +#define ALIGN(n) __declspec(align(n)) 60.182 +#define NOINLINE __declspec(noinline) 60.183 +#define FASTCALL __fastcall 60.184 +#else 60.185 +#define ALIGN(n) 60.186 +#define NOINLINE 60.187 +#define FASTCALL 60.188 +#endif 60.189 + 60.190 +/* ----------------------------------------------------------------------- */ 60.191 +/* Default implementations, if not defined above */ 60.192 +/* ----------------------------------------------------------------------- */ 60.193 + 60.194 +#ifndef ADD128 60.195 +#define ADD128(rh,rl,ih,il) \ 60.196 + { uint64_t _il = (il); \ 60.197 + (rl) += (_il); \ 60.198 + if ((rl) < (_il)) (rh)++; \ 60.199 + (rh) += (ih); \ 60.200 + } 60.201 +#endif 60.202 + 60.203 +#ifndef MUL32 60.204 +#define MUL32(i1,i2) ((uint64_t)(uint32_t)(i1)*(uint32_t)(i2)) 60.205 +#endif 60.206 + 60.207 +#ifndef PMUL64 /* rh may not be same as i1 or i2 */ 60.208 +#define PMUL64(rh,rl,i1,i2) /* Assumes m doesn't overflow */ \ 60.209 + { uint64_t _i1 = (i1), _i2 = (i2); \ 60.210 + uint64_t m = MUL32(_i1,_i2>>32) + MUL32(_i1>>32,_i2); \ 60.211 + rh = MUL32(_i1>>32,_i2>>32); \ 60.212 + rl = MUL32(_i1,_i2); \ 60.213 + ADD128(rh,rl,(m >> 32),(m << 32)); \ 60.214 + } 60.215 +#endif 60.216 + 60.217 +#ifndef MUL64 60.218 +#define MUL64(rh,rl,i1,i2) \ 60.219 + { uint64_t _i1 = (i1), _i2 = (i2); \ 60.220 + uint64_t m1= MUL32(_i1,_i2>>32); \ 60.221 + uint64_t m2= MUL32(_i1>>32,_i2); \ 60.222 + rh = MUL32(_i1>>32,_i2>>32); \ 60.223 + rl = MUL32(_i1,_i2); \ 60.224 + ADD128(rh,rl,(m1 >> 32),(m1 << 32)); \ 60.225 + ADD128(rh,rl,(m2 >> 32),(m2 << 32)); \ 60.226 + } 60.227 +#endif 60.228 + 60.229 +#ifndef GET_REVERSED_64 60.230 +#ifndef bswap64 60.231 +#ifndef bswap32 60.232 +#define bswap32(x) \ 60.233 + ({ uint32_t bsx = (x); \ 60.234 + ((((bsx) & 0xff000000u) >> 24) | (((bsx) & 0x00ff0000u) >> 8) | \ 60.235 + (((bsx) & 0x0000ff00u) << 8) | (((bsx) & 0x000000ffu) << 24)); }) 60.236 +#endif 60.237 +#define bswap64(x) \ 60.238 + ({ union { uint64_t ll; uint32_t l[2]; } w, r; \ 60.239 + w.ll = (x); \ 60.240 + r.l[0] = bswap32 (w.l[1]); \ 60.241 + r.l[1] = bswap32 (w.l[0]); \ 60.242 + r.ll; }) 60.243 +#endif 60.244 +#define GET_REVERSED_64(p) bswap64(*(uint64_t *)(p)) 60.245 +#endif 60.246 + 60.247 +/* ----------------------------------------------------------------------- */ 60.248 + 60.249 +#if (VMAC_PREFER_BIG_ENDIAN) 60.250 +# define get64PE get64BE 60.251 +#else 60.252 +# define get64PE get64LE 60.253 +#endif 60.254 + 60.255 +#if (VMAC_ARCH_BIG_ENDIAN) 60.256 +# define get64BE(ptr) (*(uint64_t *)(ptr)) 60.257 +# define get64LE(ptr) GET_REVERSED_64(ptr) 60.258 +#else /* assume little-endian */ 60.259 +# define get64BE(ptr) GET_REVERSED_64(ptr) 60.260 +# define get64LE(ptr) (*(uint64_t *)(ptr)) 60.261 +#endif 60.262 + 60.263 + 60.264 +/* --------------------------------------------------------------------- * 60.265 + * For highest performance the L1 NH and L2 polynomial hashes should be 60.266 + * carefully implemented to take advantage of one's target architechture. 60.267 + * Here these two hash functions are defined multiple time; once for 60.268 + * 64-bit architectures, once for 32-bit SSE2 architectures, and once 60.269 + * for the rest (32-bit) architectures. 60.270 + * For each, nh_16 *must* be defined (works on multiples of 16 bytes). 60.271 + * Optionally, nh_vmac_nhbytes can be defined (for multiples of 60.272 + * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two 60.273 + * NH computations at once). 60.274 + * --------------------------------------------------------------------- */ 60.275 + 60.276 +/* ----------------------------------------------------------------------- */ 60.277 +#if VMAC_ARCH_64 60.278 +/* ----------------------------------------------------------------------- */ 60.279 + 60.280 +#define nh_16(mp, kp, nw, rh, rl) \ 60.281 +{ int i; uint64_t th, tl; \ 60.282 + rh = rl = 0; \ 60.283 + for (i = 0; i < nw; i+= 2) { \ 60.284 + MUL64(th,tl,get64PE((mp)+i )+(kp)[i ],get64PE((mp)+i+1)+(kp)[i+1]);\ 60.285 + ADD128(rh,rl,th,tl); \ 60.286 + } \ 60.287 +} 60.288 +#define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \ 60.289 +{ int i; uint64_t th, tl; \ 60.290 + rh1 = rl1 = rh = rl = 0; \ 60.291 + for (i = 0; i < nw; i+= 2) { \ 60.292 + MUL64(th,tl,get64PE((mp)+i )+(kp)[i ],get64PE((mp)+i+1)+(kp)[i+1]);\ 60.293 + ADD128(rh,rl,th,tl); \ 60.294 + MUL64(th,tl,get64PE((mp)+i )+(kp)[i+2],get64PE((mp)+i+1)+(kp)[i+3]);\ 60.295 + ADD128(rh1,rl1,th,tl); \ 60.296 + } \ 60.297 +} 60.298 + 60.299 +#if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */ 60.300 +#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ 60.301 +{ int i; uint64_t th, tl; \ 60.302 + rh = rl = 0; \ 60.303 + for (i = 0; i < nw; i+= 8) { \ 60.304 + MUL64(th,tl,get64PE((mp)+i )+(kp)[i ],get64PE((mp)+i+1)+(kp)[i+1]);\ 60.305 + ADD128(rh,rl,th,tl); \ 60.306 + MUL64(th,tl,get64PE((mp)+i+2)+(kp)[i+2],get64PE((mp)+i+3)+(kp)[i+3]);\ 60.307 + ADD128(rh,rl,th,tl); \ 60.308 + MUL64(th,tl,get64PE((mp)+i+4)+(kp)[i+4],get64PE((mp)+i+5)+(kp)[i+5]);\ 60.309 + ADD128(rh,rl,th,tl); \ 60.310 + MUL64(th,tl,get64PE((mp)+i+6)+(kp)[i+6],get64PE((mp)+i+7)+(kp)[i+7]);\ 60.311 + ADD128(rh,rl,th,tl); \ 60.312 + } \ 60.313 +} 60.314 +#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \ 60.315 +{ int i; uint64_t th, tl; \ 60.316 + rh1 = rl1 = rh = rl = 0; \ 60.317 + for (i = 0; i < nw; i+= 8) { \ 60.318 + MUL64(th,tl,get64PE((mp)+i )+(kp)[i ],get64PE((mp)+i+1)+(kp)[i+1]);\ 60.319 + ADD128(rh,rl,th,tl); \ 60.320 + MUL64(th,tl,get64PE((mp)+i )+(kp)[i+2],get64PE((mp)+i+1)+(kp)[i+3]);\ 60.321 + ADD128(rh1,rl1,th,tl); \ 60.322 + MUL64(th,tl,get64PE((mp)+i+2)+(kp)[i+2],get64PE((mp)+i+3)+(kp)[i+3]);\ 60.323 + ADD128(rh,rl,th,tl); \ 60.324 + MUL64(th,tl,get64PE((mp)+i+2)+(kp)[i+4],get64PE((mp)+i+3)+(kp)[i+5]);\ 60.325 + ADD128(rh1,rl1,th,tl); \ 60.326 + MUL64(th,tl,get64PE((mp)+i+4)+(kp)[i+4],get64PE((mp)+i+5)+(kp)[i+5]);\ 60.327 + ADD128(rh,rl,th,tl); \ 60.328 + MUL64(th,tl,get64PE((mp)+i+4)+(kp)[i+6],get64PE((mp)+i+5)+(kp)[i+7]);\ 60.329 + ADD128(rh1,rl1,th,tl); \ 60.330 + MUL64(th,tl,get64PE((mp)+i+6)+(kp)[i+6],get64PE((mp)+i+7)+(kp)[i+7]);\ 60.331 + ADD128(rh,rl,th,tl); \ 60.332 + MUL64(th,tl,get64PE((mp)+i+6)+(kp)[i+8],get64PE((mp)+i+7)+(kp)[i+9]);\ 60.333 + ADD128(rh1,rl1,th,tl); \ 60.334 + } \ 60.335 +} 60.336 +#endif 60.337 + 60.338 +#define poly_step(ah, al, kh, kl, mh, ml) \ 60.339 +{ uint64_t t1h, t1l, t2h, t2l, t3h, t3l, z=0; \ 60.340 + /* compute ab*cd, put bd into result registers */ \ 60.341 + PMUL64(t3h,t3l,al,kh); \ 60.342 + PMUL64(t2h,t2l,ah,kl); \ 60.343 + PMUL64(t1h,t1l,ah,2*kh); \ 60.344 + PMUL64(ah,al,al,kl); \ 60.345 + /* add 2 * ac to result */ \ 60.346 + ADD128(ah,al,t1h,t1l); \ 60.347 + /* add together ad + bc */ \ 60.348 + ADD128(t2h,t2l,t3h,t3l); \ 60.349 + /* now (ah,al), (t2l,2*t2h) need summing */ \ 60.350 + /* first add the high registers, carrying into t2h */ \ 60.351 + ADD128(t2h,ah,z,t2l); \ 60.352 + /* double t2h and add top bit of ah */ \ 60.353 + t2h = 2 * t2h + (ah >> 63); \ 60.354 + ah &= m63; \ 60.355 + /* now add the low registers */ \ 60.356 + ADD128(ah,al,mh,ml); \ 60.357 + ADD128(ah,al,z,t2h); \ 60.358 +} 60.359 + 60.360 +/* ----------------------------------------------------------------------- */ 60.361 +#elif VMAC_USE_SSE2 60.362 +/* ----------------------------------------------------------------------- */ 60.363 + 60.364 +// macros from Crypto++ for sharing inline assembly code between MSVC and GNU C 60.365 +#if defined(__GNUC__) 60.366 + // define these in two steps to allow arguments to be expanded 60.367 + #define GNU_AS2(x, y) #x ", " #y ";" 60.368 + #define GNU_AS3(x, y, z) #x ", " #y ", " #z ";" 60.369 + #define GNU_ASL(x) "\n" #x ":" 60.370 + #define GNU_ASJ(x, y, z) #x " " #y #z ";" 60.371 + #define AS2(x, y) GNU_AS2(x, y) 60.372 + #define AS3(x, y, z) GNU_AS3(x, y, z) 60.373 + #define ASS(x, y, a, b, c, d) #x ", " #y ", " #a "*64+" #b "*16+" #c "*4+" #d ";" 60.374 + #define ASL(x) GNU_ASL(x) 60.375 + #define ASJ(x, y, z) GNU_ASJ(x, y, z) 60.376 +#else 60.377 + #define AS2(x, y) __asm {x, y} 60.378 + #define AS3(x, y, z) __asm {x, y, z} 60.379 + #define ASS(x, y, a, b, c, d) __asm {x, y, _MM_SHUFFLE(a, b, c, d)} 60.380 + #define ASL(x) __asm {label##x:} 60.381 + #define ASJ(x, y, z) __asm {x label##y} 60.382 +#endif 60.383 + 60.384 +static void NOINLINE nh_16_func(const uint64_t *mp, const uint64_t *kp, size_t nw, uint64_t *rh, uint64_t *rl) 60.385 +{ 60.386 + // This assembly version, using MMX registers, is just as fast as the 60.387 + // intrinsics version (which uses XMM registers) on the Intel Core 2, 60.388 + // but is much faster on the Pentium 4. In order to schedule multiplies 60.389 + // as early as possible, the loop interleaves operations for the current 60.390 + // block and the next block. To mask out high 32-bits, we use "movd" 60.391 + // to move the lower 32-bits to the stack and then back. Surprisingly, 60.392 + // this is faster than any other method. 60.393 +#ifdef __GNUC__ 60.394 + __asm__ __volatile__ 60.395 + ( 60.396 + ".intel_syntax noprefix;" 60.397 +#else 60.398 + AS2( mov esi, mp) 60.399 + AS2( mov edi, kp) 60.400 + AS2( mov ecx, nw) 60.401 + AS2( mov eax, rl) 60.402 + AS2( mov edx, rh) 60.403 +#endif 60.404 + AS2( sub esp, 12) 60.405 + AS2( movq mm6, [esi]) 60.406 + AS2( paddq mm6, [edi]) 60.407 + AS2( movq mm5, [esi+8]) 60.408 + AS2( paddq mm5, [edi+8]) 60.409 + AS2( add esi, 16) 60.410 + AS2( add edi, 16) 60.411 + AS2( movq mm4, mm6) 60.412 + ASS( pshufw mm2, mm6, 1, 0, 3, 2) 60.413 + AS2( pmuludq mm6, mm5) 60.414 + ASS( pshufw mm3, mm5, 1, 0, 3, 2) 60.415 + AS2( pmuludq mm5, mm2) 60.416 + AS2( pmuludq mm2, mm3) 60.417 + AS2( pmuludq mm3, mm4) 60.418 + AS2( pxor mm7, mm7) 60.419 + AS2( movd [esp], mm6) 60.420 + AS2( psrlq mm6, 32) 60.421 + AS2( movd [esp+4], mm5) 60.422 + AS2( psrlq mm5, 32) 60.423 + AS2( sub ecx, 2) 60.424 + ASJ( jz, 1, f) 60.425 + ASL(0) 60.426 + AS2( movq mm0, [esi]) 60.427 + AS2( paddq mm0, [edi]) 60.428 + AS2( movq mm1, [esi+8]) 60.429 + AS2( paddq mm1, [edi+8]) 60.430 + AS2( add esi, 16) 60.431 + AS2( add edi, 16) 60.432 + AS2( movq mm4, mm0) 60.433 + AS2( paddq mm5, mm2) 60.434 + ASS( pshufw mm2, mm0, 1, 0, 3, 2) 60.435 + AS2( pmuludq mm0, mm1) 60.436 + AS2( movd [esp+8], mm3) 60.437 + AS2( psrlq mm3, 32) 60.438 + AS2( paddq mm5, mm3) 60.439 + ASS( pshufw mm3, mm1, 1, 0, 3, 2) 60.440 + AS2( pmuludq mm1, mm2) 60.441 + AS2( pmuludq mm2, mm3) 60.442 + AS2( pmuludq mm3, mm4) 60.443 + AS2( movd mm4, [esp]) 60.444 + AS2( paddq mm7, mm4) 60.445 + AS2( movd mm4, [esp+4]) 60.446 + AS2( paddq mm6, mm4) 60.447 + AS2( movd mm4, [esp+8]) 60.448 + AS2( paddq mm6, mm4) 60.449 + AS2( movd [esp], mm0) 60.450 + AS2( psrlq mm0, 32) 60.451 + AS2( paddq mm6, mm0) 60.452 + AS2( movd [esp+4], mm1) 60.453 + AS2( psrlq mm1, 32) 60.454 + AS2( paddq mm5, mm1) 60.455 + AS2( sub ecx, 2) 60.456 + ASJ( jnz, 0, b) 60.457 + ASL(1) 60.458 + AS2( paddq mm5, mm2) 60.459 + AS2( movd [esp+8], mm3) 60.460 + AS2( psrlq mm3, 32) 60.461 + AS2( paddq mm5, mm3) 60.462 + AS2( movd mm4, [esp]) 60.463 + AS2( paddq mm7, mm4) 60.464 + AS2( movd mm4, [esp+4]) 60.465 + AS2( paddq mm6, mm4) 60.466 + AS2( movd mm4, [esp+8]) 60.467 + AS2( paddq mm6, mm4) 60.468 + 60.469 + ASS( pshufw mm0, mm7, 3, 2, 1, 0) 60.470 + AS2( psrlq mm7, 32) 60.471 + AS2( paddq mm6, mm7) 60.472 + AS2( punpckldq mm0, mm6) 60.473 + AS2( psrlq mm6, 32) 60.474 + AS2( paddq mm5, mm6) 60.475 + AS2( movq [eax], mm0) 60.476 + AS2( movq [edx], mm5) 60.477 + AS2( add esp, 12) 60.478 +#ifdef __GNUC__ 60.479 + ".att_syntax prefix;" 60.480 + : 60.481 + : "S" (mp), "D" (kp), "c" (nw), "a" (rl), "d" (rh) 60.482 + : "memory", "cc" 60.483 + ); 60.484 +#endif 60.485 +} 60.486 +#define nh_16(mp, kp, nw, rh, rl) nh_16_func(mp, kp, nw, &(rh), &(rl)); 60.487 + 60.488 +static void poly_step_func(uint64_t *ahi, uint64_t *alo, const uint64_t *kh, 60.489 + const uint64_t *kl, const uint64_t *mh, const uint64_t *ml) 60.490 +{ 60.491 + // This code tries to schedule the multiplies as early as possible to overcome 60.492 + // the long latencies on the Pentium 4. It also minimizes "movq" instructions 60.493 + // which are very expensive on the P4. 60.494 + 60.495 +#define a0 [eax+0] 60.496 +#define a1 [eax+4] 60.497 +#define a2 [ebx+0] 60.498 +#define a3 [ebx+4] 60.499 +#define k0 [ecx+0] 60.500 +#define k1 [ecx+4] 60.501 +#define k2 [edx+0] 60.502 +#define k3 [edx+4] 60.503 + 60.504 +#ifdef __GNUC__ 60.505 + uint32_t temp; 60.506 + __asm__ __volatile__ 60.507 + ( 60.508 + "mov %%ebx, %0;" 60.509 + "mov %1, %%ebx;" 60.510 + ".intel_syntax noprefix;" 60.511 +#else 60.512 + AS2( mov ebx, ahi) 60.513 + AS2( mov edx, kh) 60.514 + AS2( mov eax, alo) 60.515 + AS2( mov ecx, kl) 60.516 + AS2( mov esi, mh) 60.517 + AS2( mov edi, ml) 60.518 +#endif 60.519 + 60.520 + AS2( movd mm0, a3) 60.521 + AS2( movq mm4, mm0) 60.522 + AS2( pmuludq mm0, k3) // a3*k3 60.523 + AS2( movd mm1, a0) 60.524 + AS2( pmuludq mm1, k2) // a0*k2 60.525 + AS2( movd mm2, a1) 60.526 + AS2( movd mm6, k1) 60.527 + AS2( pmuludq mm2, mm6) // a1*k1 60.528 + AS2( movd mm3, a2) 60.529 + AS2( movq mm5, mm3) 60.530 + AS2( movd mm7, k0) 60.531 + AS2( pmuludq mm3, mm7) // a2*k0 60.532 + AS2( pmuludq mm4, mm7) // a3*k0 60.533 + AS2( pmuludq mm5, mm6) // a2*k1 60.534 + AS2( psllq mm0, 1) 60.535 + AS2( paddq mm0, [esi]) 60.536 + AS2( paddq mm0, mm1) 60.537 + AS2( movd mm1, a1) 60.538 + AS2( paddq mm4, mm5) 60.539 + AS2( movq mm5, mm1) 60.540 + AS2( pmuludq mm1, k2) // a1*k2 60.541 + AS2( paddq mm0, mm2) 60.542 + AS2( movd mm2, a0) 60.543 + AS2( paddq mm0, mm3) 60.544 + AS2( movq mm3, mm2) 60.545 + AS2( pmuludq mm2, k3) // a0*k3 60.546 + AS2( pmuludq mm3, mm7) // a0*k0 60.547 + AS2( movd esi, mm0) 60.548 + AS2( psrlq mm0, 32) 60.549 + AS2( pmuludq mm7, mm5) // a1*k0 60.550 + AS2( pmuludq mm5, k3) // a1*k3 60.551 + AS2( paddq mm0, mm1) 60.552 + AS2( movd mm1, a2) 60.553 + AS2( pmuludq mm1, k2) // a2*k2 60.554 + AS2( paddq mm0, mm2) 60.555 + AS2( paddq mm0, mm4) 60.556 + AS2( movq mm4, mm0) 60.557 + AS2( movd mm2, a3) 60.558 + AS2( pmuludq mm2, mm6) // a3*k1 60.559 + AS2( pmuludq mm6, a0) // a0*k1 60.560 + AS2( psrlq mm0, 31) 60.561 + AS2( paddq mm0, mm3) 60.562 + AS2( movd mm3, [edi]) 60.563 + AS2( paddq mm0, mm3) 60.564 + AS2( movd mm3, a2) 60.565 + AS2( pmuludq mm3, k3) // a2*k3 60.566 + AS2( paddq mm5, mm1) 60.567 + AS2( movd mm1, a3) 60.568 + AS2( pmuludq mm1, k2) // a3*k2 60.569 + AS2( paddq mm5, mm2) 60.570 + AS2( movd mm2, [edi+4]) 60.571 + AS2( psllq mm5, 1) 60.572 + AS2( paddq mm0, mm5) 60.573 + AS2( movq mm5, mm0) 60.574 + AS2( psllq mm4, 33) 60.575 + AS2( psrlq mm0, 32) 60.576 + AS2( paddq mm6, mm7) 60.577 + AS2( movd mm7, esi) 60.578 + AS2( paddq mm0, mm6) 60.579 + AS2( paddq mm0, mm2) 60.580 + AS2( paddq mm3, mm1) 60.581 + AS2( psllq mm3, 1) 60.582 + AS2( paddq mm0, mm3) 60.583 + AS2( psrlq mm4, 1) 60.584 + AS2( punpckldq mm5, mm0) 60.585 + AS2( psrlq mm0, 32) 60.586 + AS2( por mm4, mm7) 60.587 + AS2( paddq mm0, mm4) 60.588 + AS2( movq a0, mm5) 60.589 + AS2( movq a2, mm0) 60.590 +#ifdef __GNUC__ 60.591 + ".att_syntax prefix;" 60.592 + "mov %0, %%ebx;" 60.593 + : "=m" (temp) 60.594 + : "m" (ahi), "D" (ml), "d" (kh), "a" (alo), "S" (mh), "c" (kl) 60.595 + : "memory", "cc" 60.596 + ); 60.597 +#endif 60.598 + 60.599 + 60.600 +#undef a0 60.601 +#undef a1 60.602 +#undef a2 60.603 +#undef a3 60.604 +#undef k0 60.605 +#undef k1 60.606 +#undef k2 60.607 +#undef k3 60.608 +} 60.609 + 60.610 +#define poly_step(ah, al, kh, kl, mh, ml) \ 60.611 + poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml)) 60.612 + 60.613 +/* ----------------------------------------------------------------------- */ 60.614 +#else /* not VMAC_ARCH_64 and not SSE2 */ 60.615 +/* ----------------------------------------------------------------------- */ 60.616 + 60.617 +#ifndef nh_16 60.618 +#define nh_16(mp, kp, nw, rh, rl) \ 60.619 +{ uint64_t t1,t2,m1,m2,t; \ 60.620 + int i; \ 60.621 + rh = rl = t = 0; \ 60.622 + for (i = 0; i < nw; i+=2) { \ 60.623 + t1 = get64PE(mp+i) + kp[i]; \ 60.624 + t2 = get64PE(mp+i+1) + kp[i+1]; \ 60.625 + m2 = MUL32(t1 >> 32, t2); \ 60.626 + m1 = MUL32(t1, t2 >> 32); \ 60.627 + ADD128(rh,rl,MUL32(t1 >> 32,t2 >> 32),MUL32(t1,t2)); \ 60.628 + rh += (uint64_t)(uint32_t)(m1 >> 32) + (uint32_t)(m2 >> 32); \ 60.629 + t += (uint64_t)(uint32_t)m1 + (uint32_t)m2; \ 60.630 + } \ 60.631 + ADD128(rh,rl,(t >> 32),(t << 32)); \ 60.632 +} 60.633 +#endif 60.634 + 60.635 +static void poly_step_func(uint64_t *ahi, uint64_t *alo, const uint64_t *kh, 60.636 + const uint64_t *kl, const uint64_t *mh, const uint64_t *ml) 60.637 +{ 60.638 + 60.639 +#if VMAC_ARCH_BIG_ENDIAN 60.640 +#define INDEX_HIGH 0 60.641 +#define INDEX_LOW 1 60.642 +#else 60.643 +#define INDEX_HIGH 1 60.644 +#define INDEX_LOW 0 60.645 +#endif 60.646 + 60.647 +#define a0 *(((uint32_t*)alo)+INDEX_LOW) 60.648 +#define a1 *(((uint32_t*)alo)+INDEX_HIGH) 60.649 +#define a2 *(((uint32_t*)ahi)+INDEX_LOW) 60.650 +#define a3 *(((uint32_t*)ahi)+INDEX_HIGH) 60.651 +#define k0 *(((uint32_t*)kl)+INDEX_LOW) 60.652 +#define k1 *(((uint32_t*)kl)+INDEX_HIGH) 60.653 +#define k2 *(((uint32_t*)kh)+INDEX_LOW) 60.654 +#define k3 *(((uint32_t*)kh)+INDEX_HIGH) 60.655 + 60.656 + uint64_t p, q, t; 60.657 + uint32_t t2; 60.658 + 60.659 + p = MUL32(a3, k3); 60.660 + p += p; 60.661 + p += *(uint64_t *)mh; 60.662 + p += MUL32(a0, k2); 60.663 + p += MUL32(a1, k1); 60.664 + p += MUL32(a2, k0); 60.665 + t = (uint32_t)(p); 60.666 + p >>= 32; 60.667 + p += MUL32(a0, k3); 60.668 + p += MUL32(a1, k2); 60.669 + p += MUL32(a2, k1); 60.670 + p += MUL32(a3, k0); 60.671 + t |= ((uint64_t)((uint32_t)p & 0x7fffffff)) << 32; 60.672 + p >>= 31; 60.673 + p += (uint64_t)(((uint32_t*)ml)[INDEX_LOW]); 60.674 + p += MUL32(a0, k0); 60.675 + q = MUL32(a1, k3); 60.676 + q += MUL32(a2, k2); 60.677 + q += MUL32(a3, k1); 60.678 + q += q; 60.679 + p += q; 60.680 + t2 = (uint32_t)(p); 60.681 + p >>= 32; 60.682 + p += (uint64_t)(((uint32_t*)ml)[INDEX_HIGH]); 60.683 + p += MUL32(a0, k1); 60.684 + p += MUL32(a1, k0); 60.685 + q = MUL32(a2, k3); 60.686 + q += MUL32(a3, k2); 60.687 + q += q; 60.688 + p += q; 60.689 + *(uint64_t *)(alo) = (p << 32) | t2; 60.690 + p >>= 32; 60.691 + *(uint64_t *)(ahi) = p + t; 60.692 + 60.693 +#undef a0 60.694 +#undef a1 60.695 +#undef a2 60.696 +#undef a3 60.697 +#undef k0 60.698 +#undef k1 60.699 +#undef k2 60.700 +#undef k3 60.701 +} 60.702 + 60.703 +#define poly_step(ah, al, kh, kl, mh, ml) \ 60.704 + poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml)) 60.705 + 60.706 +/* ----------------------------------------------------------------------- */ 60.707 +#endif /* end of specialized NH and poly definitions */ 60.708 +/* ----------------------------------------------------------------------- */ 60.709 + 60.710 +/* At least nh_16 is defined. Defined others as needed here */ 60.711 +#ifndef nh_16_2 60.712 +#define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \ 60.713 + nh_16(mp, kp, nw, rh, rl); \ 60.714 + nh_16(mp, ((kp)+2), nw, rh2, rl2); 60.715 +#endif 60.716 +#ifndef nh_vmac_nhbytes 60.717 +#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ 60.718 + nh_16(mp, kp, nw, rh, rl) 60.719 +#endif 60.720 +#ifndef nh_vmac_nhbytes_2 60.721 +#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \ 60.722 + nh_vmac_nhbytes(mp, kp, nw, rh, rl); \ 60.723 + nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); 60.724 +#endif 60.725 + 60.726 +/* ----------------------------------------------------------------------- */ 60.727 + 60.728 +void vhash_abort(vmac_ctx_t *ctx) 60.729 +{ 60.730 + ctx->polytmp[0] = ctx->polykey[0] ; 60.731 + ctx->polytmp[1] = ctx->polykey[1] ; 60.732 + #if (VMAC_TAG_LEN == 128) 60.733 + ctx->polytmp[2] = ctx->polykey[2] ; 60.734 + ctx->polytmp[3] = ctx->polykey[3] ; 60.735 + #endif 60.736 + ctx->first_block_processed = 0; 60.737 +} 60.738 + 60.739 +/* ----------------------------------------------------------------------- */ 60.740 +static uint64_t l3hash(uint64_t p1, uint64_t p2, 60.741 + uint64_t k1, uint64_t k2, uint64_t len) 60.742 +{ 60.743 + uint64_t rh, rl, t, z=0; 60.744 + 60.745 + /* fully reduce (p1,p2)+(len,0) mod p127 */ 60.746 + t = p1 >> 63; 60.747 + p1 &= m63; 60.748 + ADD128(p1, p2, len, t); 60.749 + /* At this point, (p1,p2) is at most 2^127+(len<<64) */ 60.750 + t = (p1 > m63) + ((p1 == m63) && (p2 == m64)); 60.751 + ADD128(p1, p2, z, t); 60.752 + p1 &= m63; 60.753 + 60.754 + /* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */ 60.755 + t = p1 + (p2 >> 32); 60.756 + t += (t >> 32); 60.757 + t += (uint32_t)t > 0xfffffffeu; 60.758 + p1 += (t >> 32); 60.759 + p2 += (p1 << 32); 60.760 + 60.761 + /* compute (p1+k1)%p64 and (p2+k2)%p64 */ 60.762 + p1 += k1; 60.763 + p1 += (0 - (p1 < k1)) & 257; 60.764 + p2 += k2; 60.765 + p2 += (0 - (p2 < k2)) & 257; 60.766 + 60.767 + /* compute (p1+k1)*(p2+k2)%p64 */ 60.768 + MUL64(rh, rl, p1, p2); 60.769 + t = rh >> 56; 60.770 + ADD128(t, rl, z, rh); 60.771 + rh <<= 8; 60.772 + ADD128(t, rl, z, rh); 60.773 + t += t << 8; 60.774 + rl += t; 60.775 + rl += (0 - (rl < t)) & 257; 60.776 + rl += (0 - (rl > p64-1)) & 257; 60.777 + return rl; 60.778 +} 60.779 + 60.780 +/* ----------------------------------------------------------------------- */ 60.781 + 60.782 +void vhash_update(unsigned char *m, 60.783 + unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */ 60.784 + vmac_ctx_t *ctx) 60.785 +{ 60.786 + uint64_t rh, rl, *mptr; 60.787 + const uint64_t *kptr = (uint64_t *)ctx->nhkey; 60.788 + int i; 60.789 + uint64_t ch, cl; 60.790 + uint64_t pkh = ctx->polykey[0]; 60.791 + uint64_t pkl = ctx->polykey[1]; 60.792 + #if (VMAC_TAG_LEN == 128) 60.793 + uint64_t ch2, cl2, rh2, rl2; 60.794 + uint64_t pkh2 = ctx->polykey[2]; 60.795 + uint64_t pkl2 = ctx->polykey[3]; 60.796 + #endif 60.797 + 60.798 + mptr = (uint64_t *)m; 60.799 + i = mbytes / VMAC_NHBYTES; /* Must be non-zero */ 60.800 + 60.801 + ch = ctx->polytmp[0]; 60.802 + cl = ctx->polytmp[1]; 60.803 + #if (VMAC_TAG_LEN == 128) 60.804 + ch2 = ctx->polytmp[2]; 60.805 + cl2 = ctx->polytmp[3]; 60.806 + #endif 60.807 + 60.808 + if ( ! ctx->first_block_processed) { 60.809 + ctx->first_block_processed = 1; 60.810 + #if (VMAC_TAG_LEN == 64) 60.811 + nh_vmac_nhbytes(mptr,kptr,VMAC_NHBYTES/8,rh,rl); 60.812 + #else 60.813 + nh_vmac_nhbytes_2(mptr,kptr,VMAC_NHBYTES/8,rh,rl,rh2,rl2); 60.814 + rh2 &= m62; 60.815 + ADD128(ch2,cl2,rh2,rl2); 60.816 + #endif 60.817 + rh &= m62; 60.818 + ADD128(ch,cl,rh,rl); 60.819 + mptr += (VMAC_NHBYTES/sizeof(uint64_t)); 60.820 + i--; 60.821 + } 60.822 + 60.823 + while (i--) { 60.824 + #if (VMAC_TAG_LEN == 64) 60.825 + nh_vmac_nhbytes(mptr,kptr,VMAC_NHBYTES/8,rh,rl); 60.826 + #else 60.827 + nh_vmac_nhbytes_2(mptr,kptr,VMAC_NHBYTES/8,rh,rl,rh2,rl2); 60.828 + rh2 &= m62; 60.829 + poly_step(ch2,cl2,pkh2,pkl2,rh2,rl2); 60.830 + #endif 60.831 + rh &= m62; 60.832 + poly_step(ch,cl,pkh,pkl,rh,rl); 60.833 + mptr += (VMAC_NHBYTES/sizeof(uint64_t)); 60.834 + } 60.835 + 60.836 + ctx->polytmp[0] = ch; 60.837 + ctx->polytmp[1] = cl; 60.838 + #if (VMAC_TAG_LEN == 128) 60.839 + ctx->polytmp[2] = ch2; 60.840 + ctx->polytmp[3] = cl2; 60.841 + #endif 60.842 + #if VMAC_USE_SSE2 60.843 + _mm_empty(); /* SSE2 version of poly_step uses mmx instructions */ 60.844 + #endif 60.845 +} 60.846 + 60.847 +/* ----------------------------------------------------------------------- */ 60.848 + 60.849 +uint64_t xvhash(unsigned char m[], 60.850 + unsigned int mbytes, 60.851 + uint64_t *tagl, 60.852 + vmac_ctx_t *ctx) 60.853 +{ 60.854 + uint64_t ch, cl, rh, rl, *mptr; 60.855 + #if (VMAC_TAG_LEN == 128) 60.856 + uint64_t ch2, cl2, rh2, rl2; 60.857 + #endif 60.858 + const uint64_t *kptr = (uint64_t *)ctx->nhkey; 60.859 + int i, remaining; 60.860 + 60.861 + remaining = mbytes % VMAC_NHBYTES; 60.862 + i = mbytes-remaining; 60.863 + mptr = (uint64_t *)(m+i); 60.864 + if (i) vhash_update(m,i,ctx); 60.865 + 60.866 + ch = ctx->polytmp[0]; 60.867 + cl = ctx->polytmp[1]; 60.868 + #if (VMAC_TAG_LEN == 128) 60.869 + ch2 = ctx->polytmp[2]; 60.870 + cl2 = ctx->polytmp[3]; 60.871 + #endif 60.872 + 60.873 + if (remaining) { 60.874 + #if (VMAC_TAG_LEN == 128) 60.875 + nh_16_2(mptr,kptr,2*((remaining+15)/16),rh,rl,rh2,rl2); 60.876 + rh2 &= m62; 60.877 + #else 60.878 + nh_16(mptr,kptr,2*((remaining+15)/16),rh,rl); 60.879 + #endif 60.880 + rh &= m62; 60.881 + if (i) { 60.882 + poly_step(ch,cl,ctx->polykey[0],ctx->polykey[1],rh,rl); 60.883 + #if (VMAC_TAG_LEN == 128) 60.884 + poly_step(ch2,cl2,ctx->polykey[2],ctx->polykey[3],rh2,rl2); 60.885 + #endif 60.886 + } else { 60.887 + ADD128(ch,cl,rh,rl); 60.888 + #if (VMAC_TAG_LEN == 128) 60.889 + ADD128(ch2,cl2,rh2,rl2); 60.890 + #endif 60.891 + } 60.892 + } 60.893 + 60.894 + #if VMAC_USE_SSE2 60.895 + _mm_empty(); /* SSE2 version of poly_step uses mmx instructions */ 60.896 + #endif 60.897 + vhash_abort(ctx); 60.898 + remaining *= 8; 60.899 +#if (VMAC_TAG_LEN == 128) 60.900 + *tagl = l3hash(ch2, cl2, ctx->l3key[2], ctx->l3key[3],remaining); 60.901 +#endif 60.902 + return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1],remaining); 60.903 +} 60.904 + 60.905 +uint64_t vhash(unsigned char m[], 60.906 + unsigned int mbytes, 60.907 + uint64_t *tagl, 60.908 + vmac_ctx_t *ctx) 60.909 +{ 60.910 + uint64_t rh, rl, *mptr; 60.911 + const uint64_t *kptr = (uint64_t *)ctx->nhkey; 60.912 + int i, remaining; 60.913 + uint64_t ch, cl; 60.914 + uint64_t pkh = ctx->polykey[0]; 60.915 + uint64_t pkl = ctx->polykey[1]; 60.916 + #if (VMAC_TAG_LEN == 128) 60.917 + uint64_t ch2, cl2, rh2, rl2; 60.918 + uint64_t pkh2 = ctx->polykey[2]; 60.919 + uint64_t pkl2 = ctx->polykey[3]; 60.920 + #endif 60.921 + 60.922 + mptr = (uint64_t *)m; 60.923 + i = mbytes / VMAC_NHBYTES; 60.924 + remaining = mbytes % VMAC_NHBYTES; 60.925 + 60.926 + if (ctx->first_block_processed) 60.927 + { 60.928 + ch = ctx->polytmp[0]; 60.929 + cl = ctx->polytmp[1]; 60.930 + #if (VMAC_TAG_LEN == 128) 60.931 + ch2 = ctx->polytmp[2]; 60.932 + cl2 = ctx->polytmp[3]; 60.933 + #endif 60.934 + } 60.935 + else if (i) 60.936 + { 60.937 + #if (VMAC_TAG_LEN == 64) 60.938 + nh_vmac_nhbytes(mptr,kptr,VMAC_NHBYTES/8,ch,cl); 60.939 + #else 60.940 + nh_vmac_nhbytes_2(mptr,kptr,VMAC_NHBYTES/8,ch,cl,ch2,cl2); 60.941 + ch2 &= m62; 60.942 + ADD128(ch2,cl2,pkh2,pkl2); 60.943 + #endif 60.944 + ch &= m62; 60.945 + ADD128(ch,cl,pkh,pkl); 60.946 + mptr += (VMAC_NHBYTES/sizeof(uint64_t)); 60.947 + i--; 60.948 + } 60.949 + else if (remaining) 60.950 + { 60.951 + #if (VMAC_TAG_LEN == 64) 60.952 + nh_16(mptr,kptr,2*((remaining+15)/16),ch,cl); 60.953 + #else 60.954 + nh_16_2(mptr,kptr,2*((remaining+15)/16),ch,cl,ch2,cl2); 60.955 + ch2 &= m62; 60.956 + ADD128(ch2,cl2,pkh2,pkl2); 60.957 + #endif 60.958 + ch &= m62; 60.959 + ADD128(ch,cl,pkh,pkl); 60.960 + mptr += (VMAC_NHBYTES/sizeof(uint64_t)); 60.961 + goto do_l3; 60.962 + } 60.963 + else /* Empty String */ 60.964 + { 60.965 + ch = pkh; cl = pkl; 60.966 + #if (VMAC_TAG_LEN == 128) 60.967 + ch2 = pkh2; cl2 = pkl2; 60.968 + #endif 60.969 + goto do_l3; 60.970 + } 60.971 + 60.972 + while (i--) { 60.973 + #if (VMAC_TAG_LEN == 64) 60.974 + nh_vmac_nhbytes(mptr,kptr,VMAC_NHBYTES/8,rh,rl); 60.975 + #else 60.976 + nh_vmac_nhbytes_2(mptr,kptr,VMAC_NHBYTES/8,rh,rl,rh2,rl2); 60.977 + rh2 &= m62; 60.978 + poly_step(ch2,cl2,pkh2,pkl2,rh2,rl2); 60.979 + #endif 60.980 + rh &= m62; 60.981 + poly_step(ch,cl,pkh,pkl,rh,rl); 60.982 + mptr += (VMAC_NHBYTES/sizeof(uint64_t)); 60.983 + } 60.984 + if (remaining) { 60.985 + #if (VMAC_TAG_LEN == 64) 60.986 + nh_16(mptr,kptr,2*((remaining+15)/16),rh,rl); 60.987 + #else 60.988 + nh_16_2(mptr,kptr,2*((remaining+15)/16),rh,rl,rh2,rl2); 60.989 + rh2 &= m62; 60.990 + poly_step(ch2,cl2,pkh2,pkl2,rh2,rl2); 60.991 + #endif 60.992 + rh &= m62; 60.993 + poly_step(ch,cl,pkh,pkl,rh,rl); 60.994 + } 60.995 + 60.996 +do_l3: 60.997 + #if VMAC_USE_SSE2 60.998 + _mm_empty(); /* SSE2 version of poly_step uses mmx instructions */ 60.999 + #endif 60.1000 + vhash_abort(ctx); 60.1001 + remaining *= 8; 60.1002 +#if (VMAC_TAG_LEN == 128) 60.1003 + *tagl = l3hash(ch2, cl2, ctx->l3key[2], ctx->l3key[3],remaining); 60.1004 +#endif 60.1005 + return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1],remaining); 60.1006 +} 60.1007 + 60.1008 +/* ----------------------------------------------------------------------- */ 60.1009 + 60.1010 +uint64_t vmac(unsigned char m[], 60.1011 + unsigned int mbytes, 60.1012 + unsigned char n[16], 60.1013 + uint64_t *tagl, 60.1014 + vmac_ctx_t *ctx) 60.1015 +{ 60.1016 +#if (VMAC_TAG_LEN == 64) 60.1017 + uint64_t *in_n, *out_p; 60.1018 + uint64_t p, h; 60.1019 + int i; 60.1020 + 60.1021 + #if VMAC_CACHE_NONCES 60.1022 + in_n = ctx->cached_nonce; 60.1023 + out_p = ctx->cached_aes; 60.1024 + #else 60.1025 + uint64_t tmp[2]; 60.1026 + in_n = out_p = tmp; 60.1027 + #endif 60.1028 + 60.1029 + i = n[15] & 1; 60.1030 + #if VMAC_CACHE_NONCES 60.1031 + if ((*(uint64_t *)(n+8) != in_n[1]) || 60.1032 + (*(uint64_t *)(n ) != in_n[0])) { 60.1033 + #endif 60.1034 + 60.1035 + in_n[0] = *(uint64_t *)(n ); 60.1036 + in_n[1] = *(uint64_t *)(n+8); 60.1037 + ((unsigned char *)in_n)[15] &= 0xFE; 60.1038 + aes_encryption(in_n, out_p, &ctx->cipher_key); 60.1039 + 60.1040 + #if VMAC_CACHE_NONCES 60.1041 + ((unsigned char *)in_n)[15] |= (unsigned char)(1-i); 60.1042 + } 60.1043 + #endif 60.1044 + p = get64BE(out_p + i); 60.1045 + h = vhash(m, mbytes, (uint64_t *)0, ctx); 60.1046 + return p + h; 60.1047 +#else 60.1048 + uint64_t tmp[2]; 60.1049 + uint64_t th,tl; 60.1050 + aes_encryption(n, (unsigned char *)tmp, &ctx->cipher_key); 60.1051 + th = vhash(m, mbytes, &tl, ctx); 60.1052 + th += get64BE(tmp); 60.1053 + *tagl = tl + get64BE(tmp+1); 60.1054 + return th; 60.1055 +#endif 60.1056 +} 60.1057 + 60.1058 +/* ----------------------------------------------------------------------- */ 60.1059 + 60.1060 +void vmac_set_key(unsigned char user_key[], vmac_ctx_t *ctx) 60.1061 +{ 60.1062 + uint64_t in[2] = {0}, out[2]; 60.1063 + unsigned i; 60.1064 + aes_key_setup(user_key, &ctx->cipher_key); 60.1065 + 60.1066 + /* Fill nh key */ 60.1067 + ((unsigned char *)in)[0] = 0x80; 60.1068 + for (i = 0; i < sizeof(ctx->nhkey)/8; i+=2) { 60.1069 + aes_encryption((unsigned char *)in, (unsigned char *)out, 60.1070 + &ctx->cipher_key); 60.1071 + ctx->nhkey[i ] = get64BE(out); 60.1072 + ctx->nhkey[i+1] = get64BE(out+1); 60.1073 + ((unsigned char *)in)[15] += 1; 60.1074 + } 60.1075 + 60.1076 + /* Fill poly key */ 60.1077 + ((unsigned char *)in)[0] = 0xC0; 60.1078 + in[1] = 0; 60.1079 + for (i = 0; i < sizeof(ctx->polykey)/8; i+=2) { 60.1080 + aes_encryption((unsigned char *)in, (unsigned char *)out, 60.1081 + &ctx->cipher_key); 60.1082 + ctx->polytmp[i ] = ctx->polykey[i ] = get64BE(out) & mpoly; 60.1083 + ctx->polytmp[i+1] = ctx->polykey[i+1] = get64BE(out+1) & mpoly; 60.1084 + ((unsigned char *)in)[15] += 1; 60.1085 + } 60.1086 + 60.1087 + /* Fill ip key */ 60.1088 + ((unsigned char *)in)[0] = 0xE0; 60.1089 + in[1] = 0; 60.1090 + for (i = 0; i < sizeof(ctx->l3key)/8; i+=2) { 60.1091 + do { 60.1092 + aes_encryption((unsigned char *)in, (unsigned char *)out, 60.1093 + &ctx->cipher_key); 60.1094 + ctx->l3key[i ] = get64BE(out); 60.1095 + ctx->l3key[i+1] = get64BE(out+1); 60.1096 + ((unsigned char *)in)[15] += 1; 60.1097 + } while (ctx->l3key[i] >= p64 || ctx->l3key[i+1] >= p64); 60.1098 + } 60.1099 + 60.1100 + /* Invalidate nonce/aes cache and reset other elements */ 60.1101 + #if (VMAC_TAG_LEN == 64) && (VMAC_CACHE_NONCES) 60.1102 + ctx->cached_nonce[0] = (uint64_t)-1; /* Ensure illegal nonce */ 60.1103 + ctx->cached_nonce[1] = (uint64_t)0; /* Ensure illegal nonce */ 60.1104 + #endif 60.1105 + ctx->first_block_processed = 0; 60.1106 +} 60.1107 + 60.1108 +/* ----------------------------------------------------------------------- */ 60.1109 + 60.1110 + 60.1111 +#if VMAC_RUN_TESTS 60.1112 + 60.1113 +#include <stdlib.h> 60.1114 +#include <stdio.h> 60.1115 +#include <time.h> 60.1116 +#include <string.h> 60.1117 + 60.1118 +unsigned prime(void) /* Wake variable speed cpu, get rough speed estimate */ 60.1119 +{ 60.1120 + volatile uint64_t i; 60.1121 + volatile uint64_t j=1; 60.1122 + unsigned cnt=0; 60.1123 + volatile clock_t ticks = clock(); 60.1124 + do { 60.1125 + for (i = 0; i < 500000; i++) { 60.1126 + uint64_t x = get64PE(&j); 60.1127 + j = x * x + (uint64_t)ticks; 60.1128 + } 60.1129 + cnt++; 60.1130 + } while (clock() - ticks < (CLOCKS_PER_SEC/2)); 60.1131 + return cnt; /* cnt is millions of iterations per second */ 60.1132 +} 60.1133 + 60.1134 +int main(void) 60.1135 +{ 60.1136 + ALIGN(16) vmac_ctx_t ctx, ctx_aio, ctx_inc1, ctx_inc2; 60.1137 + uint64_t res, tagl; 60.1138 + void *p; 60.1139 + unsigned char *m; 60.1140 + ALIGN(4) unsigned char key[] = "abcdefghijklmnop"; 60.1141 + ALIGN(4) unsigned char nonce[] = "\0\0\0\0\0\0\0\0bcdefghi"; 60.1142 + unsigned int vector_lengths[] = {0,3,48,300,3000000}; 60.1143 + #if (VMAC_TAG_LEN == 64) 60.1144 + ALIGN(4) char *should_be[] = {"2576BE1C56D8B81B","2D376CF5B1813CE5", 60.1145 + "E8421F61D573D298","4492DF6C5CAC1BBE", 60.1146 + "09BA597DD7601113"}; 60.1147 + #else 60.1148 + ALIGN(4) char *should_be[] = {"472766C70F74ED23481D6D7DE4E80DAC", 60.1149 + "4EE815A06A1D71EDD36FC75D51188A42", 60.1150 + "09F2C80C8E1007A0C12FAE19FE4504AE", 60.1151 + "66438817154850C61D8A412164803BCB", 60.1152 + "2B6B02288FFC461B75485DE893C629DC"}; 60.1153 + #endif 60.1154 + unsigned speed_lengths[] = {16, 32, 64, 128, 256, 512, 1024, 2048, 4096}; 60.1155 + unsigned i, j, *speed_iters; 60.1156 + clock_t ticks; 60.1157 + double cpb; 60.1158 + const unsigned int buf_len = 3 * (1 << 20); 60.1159 + 60.1160 + j = prime(); 60.1161 + i = sizeof(speed_lengths)/sizeof(speed_lengths[0]); 60.1162 + speed_iters = (unsigned *)malloc(i*sizeof(speed_iters[0])); 60.1163 + speed_iters[i-1] = j * (1 << 12); 60.1164 + while (--i) speed_iters[i-1] = (unsigned)(1.3 * speed_iters[i]); 60.1165 + 60.1166 + /* Initialize context and message buffer, all 16-byte aligned */ 60.1167 + p = malloc(buf_len + 32); 60.1168 + m = (unsigned char *)(((size_t)p + 16) & ~((size_t)15)); 60.1169 + memset(m, 0, buf_len + 16); 60.1170 + vmac_set_key(key, &ctx); 60.1171 + 60.1172 + /* Test incremental and all-in-one interfaces for correctness */ 60.1173 + vmac_set_key(key, &ctx_aio); 60.1174 + vmac_set_key(key, &ctx_inc1); 60.1175 + vmac_set_key(key, &ctx_inc2); 60.1176 + 60.1177 + 60.1178 + /* 60.1179 + for (i = 0; i <= 512; i++) { 60.1180 + vhash_update(m,(i/VMAC_NHBYTES)*VMAC_NHBYTES,&ctx_inc1); 60.1181 + tagh = vmac(m+(i/VMAC_NHBYTES)*VMAC_NHBYTES, i%VMAC_NHBYTES, 60.1182 + nonce, &tagl, &ctx); 60.1183 + vhash_update(m,(i/VMAC_NHBYTES)*VMAC_NHBYTES,&ctx_inc1); 60.1184 + for (j = 0; j < vector_lengths[i]; j++) 60.1185 + m[j] = (unsigned char)('a'+j%3); 60.1186 + 60.1187 + } 60.1188 + */ 60.1189 + 60.1190 + /* Generate vectors */ 60.1191 + for (i = 0; i < sizeof(vector_lengths)/sizeof(unsigned int); i++) { 60.1192 + for (j = 0; j < vector_lengths[i]; j++) 60.1193 + m[j] = (unsigned char)('a'+j%3); 60.1194 + res = vmac(m, vector_lengths[i], nonce, &tagl, &ctx); 60.1195 + #if (VMAC_TAG_LEN == 64) 60.1196 + printf("\'abc\' * %7u: %016llX Should be: %s\n", 60.1197 + vector_lengths[i]/3,res,should_be[i]); 60.1198 + #else 60.1199 + printf("\'abc\' * %7u: %016llX%016llX\nShould be : %s\n", 60.1200 + vector_lengths[i]/3,res,tagl,should_be[i]); 60.1201 + #endif 60.1202 + } 60.1203 + 60.1204 + /* Speed test */ 60.1205 + for (i = 0; i < sizeof(speed_lengths)/sizeof(unsigned int); i++) { 60.1206 + ticks = clock(); 60.1207 + for (j = 0; j < speed_iters[i]; j++) { 60.1208 + #if HASH_ONLY 60.1209 + res = vhash(m, speed_lengths[i], &tagl, &ctx); 60.1210 + #else 60.1211 + res = vmac(m, speed_lengths[i], nonce, &tagl, &ctx); 60.1212 + nonce[7]++; 60.1213 + #endif 60.1214 + } 60.1215 + ticks = clock() - ticks; 60.1216 + cpb = ((ticks*VMAC_HZ)/ 60.1217 + ((double)CLOCKS_PER_SEC*speed_lengths[i]*speed_iters[i])); 60.1218 + printf("%4u bytes, %2.2f cpb\n", speed_lengths[i], cpb); 60.1219 + } 60.1220 + return 1; 60.1221 +} 60.1222 + 60.1223 +#endif
61.1 --- a/xen/drivers/passthrough/io.c Mon Mar 02 18:26:56 2009 +0900 61.2 +++ b/xen/drivers/passthrough/io.c Fri Mar 06 12:22:22 2009 +0900 61.3 @@ -105,6 +105,12 @@ int pt_irq_create_bind_vtd( 61.4 hvm_irq_dpci->msi_gvec_pirq[pt_irq_bind->u.msi.gvec] = pirq; 61.5 /* bind after hvm_irq_dpci is setup to avoid race with irq handler*/ 61.6 rc = pirq_guest_bind(d->vcpu[0], pirq, 0); 61.7 + if ( rc == 0 && pt_irq_bind->u.msi.gtable ) 61.8 + { 61.9 + rc = msixtbl_pt_register(d, pirq, pt_irq_bind->u.msi.gtable); 61.10 + if ( unlikely(rc) ) 61.11 + pirq_guest_unbind(d, pirq); 61.12 + } 61.13 if ( unlikely(rc) ) 61.14 { 61.15 hvm_irq_dpci->msi_gvec_pirq[pt_irq_bind->u.msi.gvec] = 0; 61.16 @@ -259,6 +265,7 @@ int pt_irq_destroy_bind_vtd( 61.17 if ( list_empty(&hvm_irq_dpci->mirq[machine_gsi].digl_list) ) 61.18 { 61.19 pirq_guest_unbind(d, machine_gsi); 61.20 + msixtbl_pt_unregister(d, machine_gsi); 61.21 if ( pt_irq_need_timer(hvm_irq_dpci->mirq[machine_gsi].flags) ) 61.22 kill_timer(&hvm_irq_dpci->hvm_timer[domain_irq_to_vector(d, machine_gsi)]); 61.23 hvm_irq_dpci->mirq[machine_gsi].dom = NULL;
62.1 --- a/xen/drivers/passthrough/vtd/dmar.c Mon Mar 02 18:26:56 2009 +0900 62.2 +++ b/xen/drivers/passthrough/vtd/dmar.c Fri Mar 06 12:22:22 2009 +0900 62.3 @@ -377,6 +377,19 @@ acpi_parse_one_rmrr(struct acpi_dmar_ent 62.4 return -EFAULT; 62.5 } 62.6 62.7 +#ifdef CONFIG_X86 62.8 + /* This check is here simply to detect when RMRR values are not properly represented in the 62.9 + system memory map and inform the user */ 62.10 + if ( (!page_is_ram_type(paddr_to_pfn(rmrr->base_address), RAM_TYPE_RESERVED))|| 62.11 + (!page_is_ram_type(paddr_to_pfn(rmrr->end_address) - 1, RAM_TYPE_RESERVED)) ) 62.12 + { 62.13 + dprintk(XENLOG_WARNING VTDPREFIX, 62.14 + "RMRR address range not in reserved memory base = %"PRIx64" end = %"PRIx64"; " \ 62.15 + "iommu_inclusive_mapping=1 parameter may be needed.\n", 62.16 + rmrr->base_address, rmrr->end_address); 62.17 + } 62.18 +#endif 62.19 + 62.20 rmrru = xmalloc(struct acpi_rmrr_unit); 62.21 if ( !rmrru ) 62.22 return -ENOMEM;
63.1 --- a/xen/drivers/passthrough/vtd/iommu.c Mon Mar 02 18:26:56 2009 +0900 63.2 +++ b/xen/drivers/passthrough/vtd/iommu.c Fri Mar 06 12:22:22 2009 +0900 63.3 @@ -992,8 +992,6 @@ static int intel_iommu_domain_init(struc 63.4 63.5 if ( d->domain_id == 0 ) 63.6 { 63.7 - extern int xen_in_range(paddr_t start, paddr_t end); 63.8 - 63.9 /* Set up 1:1 page table for dom0 */ 63.10 iommu_set_dom0_mapping(d); 63.11
64.1 --- a/xen/drivers/passthrough/vtd/x86/vtd.c Mon Mar 02 18:26:56 2009 +0900 64.2 +++ b/xen/drivers/passthrough/vtd/x86/vtd.c Fri Mar 06 12:22:22 2009 +0900 64.3 @@ -27,6 +27,13 @@ 64.4 #include "../dmar.h" 64.5 #include "../vtd.h" 64.6 64.7 +/* 64.8 + * iommu_inclusive_mapping: when set, all memory below 4GB is included in dom0 64.9 + * 1:1 iommu mappings except xen and unusable regions. 64.10 + */ 64.11 +static int iommu_inclusive_mapping; 64.12 +boolean_param("iommu_inclusive_mapping", iommu_inclusive_mapping); 64.13 + 64.14 void *map_vtd_domain_page(u64 maddr) 64.15 { 64.16 return map_domain_page(maddr >> PAGE_SHIFT_4K); 64.17 @@ -147,15 +154,26 @@ void hvm_dpci_isairq_eoi(struct domain * 64.18 void iommu_set_dom0_mapping(struct domain *d) 64.19 { 64.20 u64 i, j, tmp; 64.21 + unsigned long max_pfn = max(max_page, 0x100000000ul >> PAGE_SHIFT); 64.22 extern int xen_in_range(paddr_t start, paddr_t end); 64.23 64.24 BUG_ON(d->domain_id != 0); 64.25 64.26 - for ( i = 0; i < max_page; i++ ) 64.27 + for ( i = 0; i < max_pfn; i++ ) 64.28 { 64.29 - /* Set up 1:1 mapping for dom0 for all RAM except Xen bits. */ 64.30 - if ( !page_is_conventional_ram(i) || 64.31 - xen_in_range(i << PAGE_SHIFT, (i + 1) << PAGE_SHIFT) ) 64.32 + /* 64.33 + * Set up 1:1 mapping for dom0. Default to use only conventional RAM 64.34 + * areas and let RMRRs include needed reserved regions. When set, the 64.35 + * inclusive mapping maps in everything below 4GB except unusable 64.36 + * ranges. 64.37 + */ 64.38 + if ( !page_is_ram_type(i, RAM_TYPE_CONVENTIONAL) && 64.39 + (!iommu_inclusive_mapping || 64.40 + page_is_ram_type(i, RAM_TYPE_UNUSABLE)) ) 64.41 + continue; 64.42 + 64.43 + /* Exclude Xen bits */ 64.44 + if ( xen_in_range(i << PAGE_SHIFT, (i + 1) << PAGE_SHIFT) ) 64.45 continue; 64.46 64.47 tmp = 1 << (PAGE_SHIFT - PAGE_SHIFT_4K);
65.1 --- a/xen/drivers/video/vga.c Mon Mar 02 18:26:56 2009 +0900 65.2 +++ b/xen/drivers/video/vga.c Fri Mar 06 12:22:22 2009 +0900 65.3 @@ -79,7 +79,7 @@ void __init vga_init(void) 65.4 switch ( vga_console_info.video_type ) 65.5 { 65.6 case XEN_VGATYPE_TEXT_MODE_3: 65.7 - if ( page_is_conventional_ram(paddr_to_pfn(0xB8000)) || 65.8 + if ( page_is_ram_type(paddr_to_pfn(0xB8000), RAM_TYPE_CONVENTIONAL) || 65.9 ((video = ioremap(0xB8000, 0x8000)) == NULL) ) 65.10 return; 65.11 outw(0x200a, 0x3d4); /* disable cursor */
66.1 --- a/xen/include/asm-ia64/linux/pci_regs.h Mon Mar 02 18:26:56 2009 +0900 66.2 +++ b/xen/include/asm-ia64/linux/pci_regs.h Fri Mar 06 12:22:22 2009 +0900 66.3 @@ -229,7 +229,7 @@ 66.4 #define PCI_PM_CAP_PME_D3cold 0x8000 /* PME# from D3 (cold) */ 66.5 #define PCI_PM_CTRL 4 /* PM control and status register */ 66.6 #define PCI_PM_CTRL_STATE_MASK 0x0003 /* Current power state (D0 to D3) */ 66.7 -#define PCI_PM_CTRL_NO_SOFT_RESET 0x0004 /* No reset for D3hot->D0 */ 66.8 +#define PCI_PM_CTRL_NO_SOFT_RESET 0x0008 /* No reset for D3hot->D0 */ 66.9 #define PCI_PM_CTRL_PME_ENABLE 0x0100 /* PME pin enable */ 66.10 #define PCI_PM_CTRL_DATA_SEL_MASK 0x1e00 /* Data select (??) */ 66.11 #define PCI_PM_CTRL_DATA_SCALE_MASK 0x6000 /* Data scale (??) */
67.1 --- a/xen/include/asm-ia64/mm.h Mon Mar 02 18:26:56 2009 +0900 67.2 +++ b/xen/include/asm-ia64/mm.h Fri Mar 06 12:22:22 2009 +0900 67.3 @@ -200,9 +200,8 @@ static inline void put_page(struct page_ 67.4 free_domheap_page(page); 67.5 } 67.6 67.7 -/* count_info and ownership are checked atomically. */ 67.8 -static inline int get_page(struct page_info *page, 67.9 - struct domain *domain) 67.10 +static inline struct domain *page_get_owner_and_reference( 67.11 + struct page_info *page) 67.12 { 67.13 unsigned long x, y = page->count_info; 67.14 67.15 @@ -210,21 +209,30 @@ static inline int get_page(struct page_i 67.16 x = y; 67.17 if (unlikely((x & PGC_count_mask) == 0) || /* Not allocated? */ 67.18 unlikely(((x + 1) & PGC_count_mask) == 0) ) {/* Count overflow? */ 67.19 - goto fail; 67.20 + return NULL; 67.21 } 67.22 y = cmpxchg_acq(&page->count_info, x, x + 1); 67.23 } while (unlikely(y != x)); 67.24 67.25 - if (likely(page_get_owner(page) == domain)) 67.26 + return page_get_owner(page); 67.27 +} 67.28 + 67.29 +/* count_info and ownership are checked atomically. */ 67.30 +static inline int get_page(struct page_info *page, 67.31 + struct domain *domain) 67.32 +{ 67.33 + struct domain *owner = page_get_owner_and_reference(page); 67.34 + 67.35 + if (likely(owner == domain)) 67.36 return 1; 67.37 67.38 put_page(page); 67.39 -fail: 67.40 + 67.41 /* if (!domain->is_dying) */ /* XXX: header inclusion hell */ 67.42 gdprintk(XENLOG_INFO, 67.43 "Error pfn %lx: rd=%p, od=%p, caf=%016lx, taf=%" PRtype_info "\n", 67.44 page_to_mfn(page), domain, 67.45 - page_get_owner(page), y, page->u.inuse.type_info); 67.46 + owner, page->count_info, page->u.inuse.type_info); 67.47 return 0; 67.48 } 67.49
68.1 --- a/xen/include/asm-x86/domain.h Mon Mar 02 18:26:56 2009 +0900 68.2 +++ b/xen/include/asm-x86/domain.h Fri Mar 06 12:22:22 2009 +0900 68.3 @@ -16,7 +16,6 @@ 68.4 #define is_pv_32on64_domain(d) (0) 68.5 #endif 68.6 #define is_pv_32on64_vcpu(v) (is_pv_32on64_domain((v)->domain)) 68.7 -#define IS_COMPAT(d) (is_pv_32on64_domain(d)) 68.8 68.9 struct trap_bounce { 68.10 uint32_t error_code; 68.11 @@ -221,6 +220,8 @@ struct arch_domain 68.12 unsigned int hv_compat_vstart; 68.13 #endif 68.14 68.15 + bool_t s3_integrity; 68.16 + 68.17 /* I/O-port admin-specified access capabilities. */ 68.18 struct rangeset *ioport_caps; 68.19 uint32_t pci_cf8;
69.1 --- a/xen/include/asm-x86/hvm/domain.h Mon Mar 02 18:26:56 2009 +0900 69.2 +++ b/xen/include/asm-x86/hvm/domain.h Fri Mar 06 12:22:22 2009 +0900 69.3 @@ -75,6 +75,10 @@ struct hvm_domain { 69.4 /* Pass-through */ 69.5 struct hvm_iommu hvm_iommu; 69.6 69.7 + /* hypervisor intercepted msix table */ 69.8 + struct list_head msixtbl_list; 69.9 + spinlock_t msixtbl_list_lock; 69.10 + 69.11 struct viridian_domain viridian; 69.12 69.13 bool_t hap_enabled;
70.1 --- a/xen/include/asm-x86/mm.h Mon Mar 02 18:26:56 2009 +0900 70.2 +++ b/xen/include/asm-x86/mm.h Fri Mar 06 12:22:22 2009 +0900 70.3 @@ -258,6 +258,7 @@ void cleanup_page_cacheattr(struct page_ 70.4 70.5 int is_iomem_page(unsigned long mfn); 70.6 70.7 +struct domain *page_get_owner_and_reference(struct page_info *page); 70.8 void put_page(struct page_info *page); 70.9 int get_page(struct page_info *page, struct domain *domain); 70.10 void put_page_type(struct page_info *page);
71.1 --- a/xen/include/asm-x86/msi.h Mon Mar 02 18:26:56 2009 +0900 71.2 +++ b/xen/include/asm-x86/msi.h Fri Mar 06 12:22:22 2009 +0900 71.3 @@ -81,6 +81,8 @@ extern void teardown_msi_vector(int vect 71.4 extern int msi_free_vector(struct msi_desc *entry); 71.5 extern int pci_restore_msi_state(struct pci_dev *pdev); 71.6 71.7 +extern unsigned int pci_msix_get_table_len(struct pci_dev *pdev); 71.8 + 71.9 struct msi_desc { 71.10 struct { 71.11 __u8 type : 5; /* {0: unused, 5h:MSI, 11h:MSI-X} */
72.1 --- a/xen/include/asm-x86/tboot.h Mon Mar 02 18:26:56 2009 +0900 72.2 +++ b/xen/include/asm-x86/tboot.h Fri Mar 06 12:22:22 2009 +0900 72.3 @@ -53,10 +53,12 @@ typedef struct __packed { 72.4 72.5 /* used to communicate between tboot and the launched kernel (i.e. Xen) */ 72.6 72.7 +#define TB_KEY_SIZE 64 /* 512 bits */ 72.8 + 72.9 #define MAX_TB_MAC_REGIONS 32 72.10 typedef struct __packed { 72.11 - uint64_t start; 72.12 - uint64_t end; 72.13 + uint64_t start; /* must be 64 byte -aligned */ 72.14 + uint32_t size; /* must be 64 byte -granular */ 72.15 } tboot_mac_region_t; 72.16 72.17 /* GAS - Generic Address Structure (ACPI 2.0+) */ 72.18 @@ -83,7 +85,7 @@ typedef struct __packed { 72.19 typedef struct __packed { 72.20 /* version 3+ fields: */ 72.21 uuid_t uuid; /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ 72.22 - uint32_t version; /* Version number; currently supports 0.3 */ 72.23 + uint32_t version; /* Version number; currently supports 0.4 */ 72.24 uint32_t log_addr; /* physical addr of tb_log_t log */ 72.25 uint32_t shutdown_entry; /* entry point for tboot shutdown */ 72.26 uint32_t shutdown_type; /* type of shutdown (TB_SHUTDOWN_*) */ 72.27 @@ -94,6 +96,9 @@ typedef struct __packed { 72.28 uint8_t num_mac_regions; /* number mem regions to MAC on S3 */ 72.29 /* contig regions memory to MAC on S3 */ 72.30 tboot_mac_region_t mac_regions[MAX_TB_MAC_REGIONS]; 72.31 + /* version 4+ fields: */ 72.32 + /* populated by tboot; will be encrypted */ 72.33 + uint8_t s3_key[TB_KEY_SIZE]; 72.34 } tboot_shared_t; 72.35 72.36 #define TB_SHUTDOWN_REBOOT 0 72.37 @@ -113,6 +118,7 @@ void tboot_shutdown(uint32_t shutdown_ty 72.38 int tboot_in_measured_env(void); 72.39 int tboot_protect_mem_regions(void); 72.40 int tboot_parse_dmar_table(acpi_table_handler dmar_handler); 72.41 +int tboot_s3_resume(void); 72.42 72.43 #endif /* __TBOOT_H__ */ 72.44
73.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 73.2 +++ b/xen/include/crypto/rijndael.h Fri Mar 06 12:22:22 2009 +0900 73.3 @@ -0,0 +1,58 @@ 73.4 +/* $OpenBSD: rijndael.h,v 1.13 2008/06/09 07:49:45 djm Exp $ */ 73.5 + 73.6 +/** 73.7 + * rijndael-alg-fst.h 73.8 + * 73.9 + * @version 3.0 (December 2000) 73.10 + * 73.11 + * Optimised ANSI C code for the Rijndael cipher (now AES) 73.12 + * 73.13 + * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be> 73.14 + * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be> 73.15 + * @author Paulo Barreto <paulo.barreto@terra.com.br> 73.16 + * 73.17 + * This code is hereby placed in the public domain. 73.18 + * 73.19 + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS 73.20 + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 73.21 + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 73.22 + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE 73.23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 73.24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 73.25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 73.26 + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 73.27 + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 73.28 + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 73.29 + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 73.30 + */ 73.31 +#ifndef __RIJNDAEL_H 73.32 +#define __RIJNDAEL_H 73.33 + 73.34 +#define AES_MAXKEYBITS (256) 73.35 +#define AES_MAXKEYBYTES (AES_MAXKEYBITS/8) 73.36 +/* for 256-bit keys, fewer for less */ 73.37 +#define AES_MAXROUNDS 14 73.38 + 73.39 +//typedef unsigned char u8; 73.40 +//typedef unsigned short u16; 73.41 +//typedef unsigned int u32; 73.42 + 73.43 +/* The structure for key information */ 73.44 +typedef struct { 73.45 + int enc_only; /* context contains only encrypt schedule */ 73.46 + int Nr; /* key-length-dependent number of rounds */ 73.47 + u32 ek[4*(AES_MAXROUNDS + 1)]; /* encrypt key schedule */ 73.48 + u32 dk[4*(AES_MAXROUNDS + 1)]; /* decrypt key schedule */ 73.49 +} rijndael_ctx; 73.50 + 73.51 +int rijndael_set_key(rijndael_ctx *, const u_char *, int); 73.52 +int rijndael_set_key_enc_only(rijndael_ctx *, const u_char *, int); 73.53 +void rijndael_decrypt(rijndael_ctx *, const u_char *, u_char *); 73.54 +void rijndael_encrypt(rijndael_ctx *, const u_char *, u_char *); 73.55 + 73.56 +int rijndaelKeySetupEnc(unsigned int [], const unsigned char [], int); 73.57 +int rijndaelKeySetupDec(unsigned int [], const unsigned char [], int); 73.58 +void rijndaelEncrypt(const unsigned int [], int, const unsigned char [], 73.59 + unsigned char []); 73.60 + 73.61 +#endif /* __RIJNDAEL_H */
74.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 74.2 +++ b/xen/include/crypto/vmac.h Fri Mar 06 12:22:22 2009 +0900 74.3 @@ -0,0 +1,178 @@ 74.4 +#ifndef HEADER_VMAC_H 74.5 +#define HEADER_VMAC_H 74.6 + 74.7 +/* -------------------------------------------------------------------------- 74.8 + * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai. 74.9 + * This implementation is herby placed in the public domain. 74.10 + * The authors offers no warranty. Use at your own risk. 74.11 + * Please send bug reports to the authors. 74.12 + * Last modified: 17 APR 08, 1700 PDT 74.13 + * ----------------------------------------------------------------------- */ 74.14 + 74.15 +/* -------------------------------------------------------------------------- 74.16 + * User definable settings. 74.17 + * ----------------------------------------------------------------------- */ 74.18 +#define VMAC_TAG_LEN 64 /* Must be 64 or 128 - 64 sufficient for most */ 74.19 +#define VMAC_KEY_LEN 128 /* Must be 128, 192 or 256 */ 74.20 +#define VMAC_NHBYTES 128 /* Must 2^i for any 3 < i < 13. Standard = 128 */ 74.21 +#define VMAC_PREFER_BIG_ENDIAN 0 /* Prefer non-x86 */ 74.22 + 74.23 +#define VMAC_USE_OPENSSL 0 /* Set to non-zero to use OpenSSL's AES */ 74.24 +#define VMAC_CACHE_NONCES 1 /* Set to non-zero to cause caching */ 74.25 + /* of consecutive nonces on 64-bit tags */ 74.26 + 74.27 +#define VMAC_RUN_TESTS 0 /* Set to non-zero to check vectors and speed */ 74.28 +#define VMAC_HZ (448e6) /* Set to hz of host machine to get speed */ 74.29 +#define VMAC_HASH_ONLY 0 /* Set to non-zero to time hash only (not-mac) */ 74.30 +/* Speeds of cpus I have access to 74.31 +#define hz (2400e6) glyme Core 2 "Conroe" 74.32 +#define hz (2000e6) jupiter G5 74.33 +#define hz (1592e6) titan 74.34 +#define hz (2793e6) athena/gaia 74.35 +#define hz (1250e6) isis G4 74.36 +#define hz (2160e6) imac Core 2 "Merom" 74.37 +#define hz (266e6) ppc/arm 74.38 +#define hz (400e6) mips 74.39 +*/ 74.40 + 74.41 +/* -------------------------------------------------------------------------- 74.42 + * This implementation uses uint32_t and uint64_t as names for unsigned 32- 74.43 + * and 64-bit integer types. These are defined in C99 stdint.h. The 74.44 + * following may need adaptation if you are not running a C99 or 74.45 + * Microsoft C environment. 74.46 + * ----------------------------------------------------------------------- */ 74.47 +#define VMAC_USE_STDINT 1 /* Set to zero if system has no stdint.h */ 74.48 + 74.49 +#if VMAC_USE_STDINT && !_MSC_VER /* Try stdint.h if non-Microsoft */ 74.50 +#ifdef __cplusplus 74.51 +#define __STDC_CONSTANT_MACROS 74.52 +#endif 74.53 +//#include <stdint.h> 74.54 +#elif (_MSC_VER) /* Microsoft C does not have stdint.h */ 74.55 +typedef unsigned __int32 uint32_t; 74.56 +typedef unsigned __int64 uint64_t; 74.57 +#define UINT64_C(v) v ## UI64 74.58 +#else /* Guess sensibly - may need adaptation */ 74.59 +typedef unsigned int uint32_t; 74.60 +typedef unsigned long long uint64_t; 74.61 +#define UINT64_C(v) v ## ULL 74.62 +#endif 74.63 + 74.64 +/* -------------------------------------------------------------------------- 74.65 + * This implementation supports two free AES implementations: OpenSSL's and 74.66 + * Paulo Barreto's. To use OpenSSL's, you will need to include the OpenSSL 74.67 + * crypto library (eg, gcc -lcrypto foo.c). For Barreto's, you will need 74.68 + * to compile rijndael-alg-fst.c, last seen at http://www.iaik.tu-graz.ac.at/ 74.69 + * research/krypto/AES/old/~rijmen/rijndael/rijndael-fst-3.0.zip and 74.70 + * http://homes.esat.kuleuven.be/~rijmen/rijndael/rijndael-fst-3.0.zip. 74.71 + * To use a different implementation, use these definitions as a model. 74.72 + * ----------------------------------------------------------------------- */ 74.73 +#if VMAC_USE_OPENSSL 74.74 + 74.75 +#include <openssl/aes.h> 74.76 +typedef AES_KEY aes_int_key; 74.77 + 74.78 +#define aes_encryption(in,out,int_key) \ 74.79 + AES_encrypt((unsigned char *)(in),(unsigned char *)(out),(int_key)) 74.80 +#define aes_key_setup(key,int_key) \ 74.81 + AES_set_encrypt_key((key),VMAC_KEY_LEN,(int_key)) 74.82 + 74.83 +#else 74.84 + 74.85 +//#include "rijndael-alg-fst.h" 74.86 +typedef uint64_t vmac_t; 74.87 +#include "rijndael.h" 74.88 +typedef u32 aes_int_key[4*(VMAC_KEY_LEN/32+7)]; 74.89 + 74.90 +#define aes_encryption(in,out,int_key) \ 74.91 + rijndaelEncrypt((u32 *)(int_key), \ 74.92 + ((VMAC_KEY_LEN/32)+6), \ 74.93 + (u8 *)(in), (u8 *)(out)) 74.94 +#define aes_key_setup(user_key,int_key) \ 74.95 + rijndaelKeySetupEnc((u32 *)(int_key), \ 74.96 + (u8 *)(user_key), \ 74.97 + VMAC_KEY_LEN) 74.98 +#endif 74.99 + 74.100 +/* --------------------------------------------------------------------- */ 74.101 + 74.102 +typedef struct { 74.103 + uint64_t nhkey [(VMAC_NHBYTES/8)+2*(VMAC_TAG_LEN/64-1)]; 74.104 + uint64_t polykey[2*VMAC_TAG_LEN/64]; 74.105 + uint64_t l3key [2*VMAC_TAG_LEN/64]; 74.106 + uint64_t polytmp[2*VMAC_TAG_LEN/64]; 74.107 + aes_int_key cipher_key; 74.108 + #if (VMAC_TAG_LEN == 64) && (VMAC_CACHE_NONCES) 74.109 + uint64_t cached_nonce[2]; 74.110 + uint64_t cached_aes[2]; 74.111 + #endif 74.112 + int first_block_processed; 74.113 +} vmac_ctx_t; 74.114 + 74.115 +/* --------------------------------------------------------------------- */ 74.116 +#ifdef __cplusplus 74.117 +extern "C" { 74.118 +#endif 74.119 +/* -------------------------------------------------------------------------- 74.120 + * <<<<< USAGE NOTES >>>>> 74.121 + * 74.122 + * Given msg m (mbytes in length) and nonce buffer n 74.123 + * this function returns a tag as its output. The tag is returned as 74.124 + * a number. When VMAC_TAG_LEN == 64, the 'return'ed integer is the tag, 74.125 + * and *tagl is meaningless. When VMAC_TAG_LEN == 128 the tag is the 74.126 + * number y * 2^64 + *tagl where y is the function's return value. 74.127 + * If you want to consider tags to be strings, then you must do so with 74.128 + * an agreed upon endian orientation for interoperability, and convert 74.129 + * the results appropriately. VHASH hashes m without creating any tag. 74.130 + * Consecutive substrings forming a prefix of a message may be passed 74.131 + * to vhash_update, with vhash or vmac being called with the remainder 74.132 + * to produce the output. 74.133 + * 74.134 + * Requirements: 74.135 + * - On 32-bit architectures with SSE2 instructions, ctx and m MUST be 74.136 + * begin on 16-byte memory boundaries. 74.137 + * - m MUST be your message followed by zeroes to the nearest 16-byte 74.138 + * boundary. If m is a length multiple of 16 bytes, then it is already 74.139 + * at a 16-byte boundary and needs no padding. mbytes should be your 74.140 + * message length without any padding. 74.141 + * - The first bit of the nonce buffer n must be 0. An i byte nonce, is made 74.142 + * as the first 16-i bytes of n being zero, and the final i the nonce. 74.143 + * - vhash_update MUST have mbytes be a positive multiple of VMAC_NHBYTES 74.144 + * ----------------------------------------------------------------------- */ 74.145 + 74.146 +#define vmac_update vhash_update 74.147 + 74.148 +void vhash_update(unsigned char m[], 74.149 + unsigned int mbytes, 74.150 + vmac_ctx_t *ctx); 74.151 + 74.152 +uint64_t vmac(unsigned char m[], 74.153 + unsigned int mbytes, 74.154 + unsigned char n[16], 74.155 + uint64_t *tagl, 74.156 + vmac_ctx_t *ctx); 74.157 + 74.158 +uint64_t vhash(unsigned char m[], 74.159 + unsigned int mbytes, 74.160 + uint64_t *tagl, 74.161 + vmac_ctx_t *ctx); 74.162 + 74.163 +/* -------------------------------------------------------------------------- 74.164 + * When passed a VMAC_KEY_LEN bit user_key, this function initialazies ctx. 74.165 + * ----------------------------------------------------------------------- */ 74.166 + 74.167 +void vmac_set_key(unsigned char user_key[], vmac_ctx_t *ctx); 74.168 + 74.169 +/* -------------------------------------------------------------------------- 74.170 + * This function aborts current hash and resets ctx, ready for a new message. 74.171 + * ----------------------------------------------------------------------- */ 74.172 + 74.173 +void vhash_abort(vmac_ctx_t *ctx); 74.174 + 74.175 +/* --------------------------------------------------------------------- */ 74.176 + 74.177 +#ifdef __cplusplus 74.178 +} 74.179 +#endif 74.180 + 74.181 +#endif /* HEADER_AES_H */
75.1 --- a/xen/include/public/domctl.h Mon Mar 02 18:26:56 2009 +0900 75.2 +++ b/xen/include/public/domctl.h Fri Mar 06 12:22:22 2009 +0900 75.3 @@ -51,11 +51,14 @@ struct xen_domctl_createdomain { 75.4 uint32_t ssidref; 75.5 xen_domain_handle_t handle; 75.6 /* Is this an HVM guest (as opposed to a PV guest)? */ 75.7 -#define _XEN_DOMCTL_CDF_hvm_guest 0 75.8 -#define XEN_DOMCTL_CDF_hvm_guest (1U<<_XEN_DOMCTL_CDF_hvm_guest) 75.9 +#define _XEN_DOMCTL_CDF_hvm_guest 0 75.10 +#define XEN_DOMCTL_CDF_hvm_guest (1U<<_XEN_DOMCTL_CDF_hvm_guest) 75.11 /* Use hardware-assisted paging if available? */ 75.12 -#define _XEN_DOMCTL_CDF_hap 1 75.13 -#define XEN_DOMCTL_CDF_hap (1U<<_XEN_DOMCTL_CDF_hap) 75.14 +#define _XEN_DOMCTL_CDF_hap 1 75.15 +#define XEN_DOMCTL_CDF_hap (1U<<_XEN_DOMCTL_CDF_hap) 75.16 + /* Should domain memory integrity be verifed by tboot during Sx? */ 75.17 +#define _XEN_DOMCTL_CDF_s3_integrity 2 75.18 +#define XEN_DOMCTL_CDF_s3_integrity (1U<<_XEN_DOMCTL_CDF_s3_integrity) 75.19 uint32_t flags; 75.20 }; 75.21 typedef struct xen_domctl_createdomain xen_domctl_createdomain_t; 75.22 @@ -485,6 +488,7 @@ struct xen_domctl_bind_pt_irq { 75.23 struct { 75.24 uint8_t gvec; 75.25 uint32_t gflags; 75.26 + uint64_t gtable; 75.27 } msi; 75.28 } u; 75.29 };
76.1 --- a/xen/include/xen/compat.h Mon Mar 02 18:26:56 2009 +0900 76.2 +++ b/xen/include/xen/compat.h Fri Mar 06 12:22:22 2009 +0900 76.3 @@ -178,15 +178,10 @@ void xlat_vcpu_runstate_info(struct vcpu 76.4 int switch_compat(struct domain *); 76.5 int switch_native(struct domain *); 76.6 76.7 -#define BITS_PER_GUEST_LONG(d) \ 76.8 - (!IS_COMPAT(d) ? BITS_PER_LONG : COMPAT_BITS_PER_LONG) 76.9 - 76.10 #else 76.11 76.12 #define compat_handle_is_null(hnd) 0 76.13 76.14 -#define BITS_PER_GUEST_LONG(d) BITS_PER_LONG 76.15 - 76.16 #endif 76.17 76.18 #endif /* __XEN_COMPAT_H__ */
77.1 --- a/xen/include/xen/mm.h Mon Mar 02 18:26:56 2009 +0900 77.2 +++ b/xen/include/xen/mm.h Fri Mar 06 12:22:22 2009 +0900 77.3 @@ -273,8 +273,12 @@ unsigned long avail_scrub_pages(void); 77.4 77.5 int guest_remove_page(struct domain *d, unsigned long gmfn); 77.6 77.7 -/* Returns TRUE if the whole page at @mfn is ordinary RAM. */ 77.8 -int page_is_conventional_ram(unsigned long mfn); 77.9 +#define RAM_TYPE_CONVENTIONAL 0x00000001 77.10 +#define RAM_TYPE_RESERVED 0x00000002 77.11 +#define RAM_TYPE_UNUSABLE 0x00000004 77.12 +#define RAM_TYPE_ACPI 0x00000008 77.13 +/* Returns TRUE if the whole page at @mfn is of the requested RAM type(s) above. */ 77.14 +int page_is_ram_type(unsigned long mfn, unsigned long mem_type); 77.15 77.16 extern unsigned long *alloc_bitmap; /* for vmcoreinfo */ 77.17
78.1 --- a/xen/include/xen/pci.h Mon Mar 02 18:26:56 2009 +0900 78.2 +++ b/xen/include/xen/pci.h Fri Mar 06 12:22:22 2009 +0900 78.3 @@ -29,7 +29,8 @@ 78.4 #define PCI_BDF(b,d,f) ((((b) & 0xff) << 8) | PCI_DEVFN(d,f)) 78.5 #define PCI_BDF2(b,df) ((((b) & 0xff) << 8) | ((df) & 0xff)) 78.6 78.7 -#define MAX_MSIX_TABLE_PAGES 8 /* 2048 entries */ 78.8 +#define MAX_MSIX_TABLE_ENTRIES 2048 78.9 +#define MAX_MSIX_TABLE_PAGES 8 78.10 struct pci_dev { 78.11 struct list_head alldevs_list; 78.12 struct list_head domain_list; 78.13 @@ -84,4 +85,7 @@ void pci_conf_write32( 78.14 int pci_find_cap_offset(u8 bus, u8 dev, u8 func, u8 cap); 78.15 int pci_find_next_cap(u8 bus, unsigned int devfn, u8 pos, int cap); 78.16 78.17 +int msixtbl_pt_register(struct domain *d, int pirq, uint64_t gtable); 78.18 +void msixtbl_pt_unregister(struct domain *d, int pirq); 78.19 + 78.20 #endif /* __XEN_PCI_H__ */
79.1 --- a/xen/include/xen/pci_regs.h Mon Mar 02 18:26:56 2009 +0900 79.2 +++ b/xen/include/xen/pci_regs.h Fri Mar 06 12:22:22 2009 +0900 79.3 @@ -233,7 +233,7 @@ 79.4 #define PCI_PM_CAP_PME_D3cold 0x8000 /* PME# from D3 (cold) */ 79.5 #define PCI_PM_CTRL 4 /* PM control and status register */ 79.6 #define PCI_PM_CTRL_STATE_MASK 0x0003 /* Current power state (D0 to D3) */ 79.7 -#define PCI_PM_CTRL_NO_SOFT_RESET 0x0004 /* No reset for D3hot->D0 */ 79.8 +#define PCI_PM_CTRL_NO_SOFT_RESET 0x0008 /* No reset for D3hot->D0 */ 79.9 #define PCI_PM_CTRL_PME_ENABLE 0x0100 /* PME pin enable */ 79.10 #define PCI_PM_CTRL_DATA_SEL_MASK 0x1e00 /* Data select (??) */ 79.11 #define PCI_PM_CTRL_DATA_SCALE_MASK 0x6000 /* Data scale (??) */
80.1 --- a/xen/include/xen/sched.h Mon Mar 02 18:26:56 2009 +0900 80.2 +++ b/xen/include/xen/sched.h Fri Mar 06 12:22:22 2009 +0900 80.3 @@ -30,12 +30,11 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_runstate_in 80.4 extern struct domain *dom0; 80.5 80.6 #ifndef CONFIG_COMPAT 80.7 -#define MAX_EVTCHNS(d) NR_EVENT_CHANNELS 80.8 +#define BITS_PER_EVTCHN_WORD(d) BITS_PER_LONG 80.9 #else 80.10 -#define MAX_EVTCHNS(d) (!IS_COMPAT(d) ? \ 80.11 - NR_EVENT_CHANNELS : \ 80.12 - sizeof(unsigned int) * sizeof(unsigned int) * 64) 80.13 +#define BITS_PER_EVTCHN_WORD(d) (has_32bit_shinfo(d) ? 32 : BITS_PER_LONG) 80.14 #endif 80.15 +#define MAX_EVTCHNS(d) (BITS_PER_EVTCHN_WORD(d) * BITS_PER_EVTCHN_WORD(d) * 64) 80.16 #define EVTCHNS_PER_BUCKET 128 80.17 #define NR_EVTCHN_BUCKETS (NR_EVENT_CHANNELS / EVTCHNS_PER_BUCKET) 80.18 80.19 @@ -341,14 +340,18 @@ static inline struct domain *get_current 80.20 struct domain *domain_create( 80.21 domid_t domid, unsigned int domcr_flags, ssidref_t ssidref); 80.22 /* DOMCRF_hvm: Create an HVM domain, as opposed to a PV domain. */ 80.23 -#define _DOMCRF_hvm 0 80.24 -#define DOMCRF_hvm (1U<<_DOMCRF_hvm) 80.25 +#define _DOMCRF_hvm 0 80.26 +#define DOMCRF_hvm (1U<<_DOMCRF_hvm) 80.27 /* DOMCRF_hap: Create a domain with hardware-assisted paging. */ 80.28 -#define _DOMCRF_hap 1 80.29 -#define DOMCRF_hap (1U<<_DOMCRF_hap) 80.30 +#define _DOMCRF_hap 1 80.31 +#define DOMCRF_hap (1U<<_DOMCRF_hap) 80.32 + /* DOMCRF_s3_integrity: Create a domain with tboot memory integrity protection 80.33 + by tboot */ 80.34 +#define _DOMCRF_s3_integrity 2 80.35 +#define DOMCRF_s3_integrity (1U<<_DOMCRF_s3_integrity) 80.36 /* DOMCRF_dummy: Create a dummy domain (not scheduled; not on domain list) */ 80.37 -#define _DOMCRF_dummy 2 80.38 -#define DOMCRF_dummy (1U<<_DOMCRF_dummy) 80.39 +#define _DOMCRF_dummy 3 80.40 +#define DOMCRF_dummy (1U<<_DOMCRF_dummy) 80.41 80.42 /* 80.43 * rcu_lock_domain_by_id() is more efficient than get_domain_by_id(). 80.44 @@ -538,10 +541,6 @@ uint64_t get_cpu_idle_time(unsigned int 80.45 #define IS_PRIV(_d) ((_d)->is_privileged) 80.46 #define IS_PRIV_FOR(_d, _t) (IS_PRIV(_d) || ((_d)->target && (_d)->target == (_t))) 80.47 80.48 -#ifndef IS_COMPAT 80.49 -#define IS_COMPAT(d) 0 80.50 -#endif 80.51 - 80.52 #define VM_ASSIST(_d,_t) (test_bit((_t), &(_d)->vm_assist)) 80.53 80.54 #define is_hvm_domain(d) ((d)->is_hvm)