ia64/xen-unstable
changeset 1302:92668c30fa16
bitkeeper revision 1.864 (407d63b3Kv7jrCnhzfWFt1VQd3vpFQ)
More TLB-flush fixes.
More TLB-flush fixes.
| author | kaf24@scramble.cl.cam.ac.uk |
|---|---|
| date | Wed Apr 14 16:15:47 2004 +0000 (2004-04-14) |
| parents | d8fc9ec791e9 |
| children | 3cab1817a752 f2844bf60c6f |
| files | xen/arch/i386/flushtlb.c xen/arch/i386/smp.c xen/common/memory.c xen/common/shadow.c xen/include/asm-i386/flushtlb.h xen/include/asm-x86_64/flushtlb.h xen/include/xen/mm.h xen/net/dev.c |
line diff
1.1 --- a/xen/arch/i386/flushtlb.c Tue Apr 13 16:30:13 2004 +0000 1.2 +++ b/xen/arch/i386/flushtlb.c Wed Apr 14 16:15:47 2004 +0000 1.3 @@ -11,21 +11,18 @@ 1.4 #include <xen/sched.h> 1.5 #include <asm/flushtlb.h> 1.6 1.7 -unsigned long tlbflush_mask; 1.8 -unsigned long tlbflush_clock; 1.9 -unsigned long tlbflush_time[NR_CPUS]; 1.10 +u32 tlbflush_clock; 1.11 +u32 tlbflush_time[NR_CPUS]; 1.12 1.13 static inline void tlb_clocktick(unsigned int cpu) 1.14 { 1.15 - unsigned long x, nx, y, ny; 1.16 - 1.17 - clear_bit(cpu, &tlbflush_mask); 1.18 + u32 y, ny; 1.19 1.20 /* Tick the clock. 'y' contains the current time after the tick. */ 1.21 ny = tlbflush_clock; 1.22 do { 1.23 #ifdef CONFIG_SMP 1.24 - if ( unlikely(((y = ny+1) & (GLOBAL_FLUSH_PERIOD - 1)) == 0) ) 1.25 + if ( unlikely(((y = ny+1) & TLBCLOCK_EPOCH_MASK) == 0) ) 1.26 { 1.27 new_tlbflush_clock_period(); 1.28 y = tlbflush_clock; 1.29 @@ -37,13 +34,8 @@ static inline void tlb_clocktick(unsigne 1.30 } 1.31 while ( unlikely((ny = cmpxchg(&tlbflush_clock, y-1, y)) != y-1) ); 1.32 1.33 - /* Update cpu's timestamp to current time, unless someone else beats us. */ 1.34 - nx = tlbflush_time[cpu]; 1.35 - do { 1.36 - if ( unlikely((x = nx) >= y) ) 1.37 - break; 1.38 - } 1.39 - while ( unlikely((nx = cmpxchg(&tlbflush_time[cpu], x, y)) != x) ); 1.40 + /* Update cpu's timestamp to new time. */ 1.41 + tlbflush_time[cpu] = y; 1.42 } 1.43 1.44 void write_cr3_counted(unsigned long pa)
2.1 --- a/xen/arch/i386/smp.c Tue Apr 13 16:30:13 2004 +0000 2.2 +++ b/xen/arch/i386/smp.c Wed Apr 14 16:15:47 2004 +0000 2.3 @@ -284,10 +284,15 @@ void new_tlbflush_clock_period(void) 2.4 if ( unlikely(!spin_trylock(&synchronous_ipi_lock)) ) 2.5 return; 2.6 2.7 - flush_cpumask = tlbflush_mask & ~(1 << smp_processor_id()); 2.8 - if ( unlikely(flush_cpumask != 0) ) 2.9 + /* Someone may acquire the lock and execute the flush before us. */ 2.10 + if ( ((tlbflush_clock+1) & TLBCLOCK_EPOCH_MASK) != 0 ) 2.11 + goto out; 2.12 + 2.13 + if ( smp_num_cpus > 1 ) 2.14 { 2.15 - send_IPI_mask(flush_cpumask, INVALIDATE_TLB_VECTOR); 2.16 + /* Flush everyone else. We definitely flushed just before entry. */ 2.17 + flush_cpumask = ((1 << smp_num_cpus) - 1) & ~(1 << smp_processor_id()); 2.18 + send_IPI_allbutself(INVALIDATE_TLB_VECTOR); 2.19 while ( flush_cpumask != 0 ) 2.20 { 2.21 rep_nop(); 2.22 @@ -295,11 +300,10 @@ void new_tlbflush_clock_period(void) 2.23 } 2.24 } 2.25 2.26 - /* No need for cmpxchg updates here: we are protected by tlbstate lock. */ 2.27 - tlbflush_mask = (1 << smp_num_cpus) - 1; 2.28 - wmb(); /* Reset the mask before allowing the clock to continue ticking. */ 2.29 + /* No need for atomicity: we are the only possible updater. */ 2.30 tlbflush_clock++; 2.31 2.32 + out: 2.33 spin_unlock(&synchronous_ipi_lock); 2.34 } 2.35
3.1 --- a/xen/common/memory.c Tue Apr 13 16:30:13 2004 +0000 3.2 +++ b/xen/common/memory.c Wed Apr 14 16:15:47 2004 +0000 3.3 @@ -153,7 +153,7 @@ static int alloc_l2_table(struct pfn_inf 3.4 static int alloc_l1_table(struct pfn_info *page); 3.5 static int get_page_from_pagenr(unsigned long page_nr, int check_level); 3.6 static int get_page_and_type_from_pagenr(unsigned long page_nr, 3.7 - unsigned int type, 3.8 + u32 type, 3.9 int check_level); 3.10 #define CHECK_STRICT 0 /* Subject domain must own the page */ 3.11 #define CHECK_ANYDOM 1 /* Any domain may own the page (if subject is priv.) */ 3.12 @@ -299,7 +299,7 @@ static int get_page_from_pagenr(unsigned 3.13 { 3.14 struct task_struct *p = current; 3.15 struct pfn_info *page = &frame_table[page_nr]; 3.16 - unsigned long y, x, nx; 3.17 + u32 y, x, nx; 3.18 3.19 if ( unlikely(!pfn_is_ram(page_nr)) ) 3.20 { 3.21 @@ -345,7 +345,7 @@ static int get_page_from_pagenr(unsigned 3.22 3.23 3.24 static int get_page_and_type_from_pagenr(unsigned long page_nr, 3.25 - unsigned int type, 3.26 + u32 type, 3.27 int check_level) 3.28 { 3.29 struct pfn_info *page = &frame_table[page_nr]; 3.30 @@ -355,7 +355,7 @@ static int get_page_and_type_from_pagenr 3.31 3.32 if ( unlikely(!get_page_type(page, type)) ) 3.33 { 3.34 - MEM_LOG("Bad page type for pfn %08lx (%08lx)", 3.35 + MEM_LOG("Bad page type for pfn %08lx (%08x)", 3.36 page_nr, page->type_and_flags); 3.37 put_page(page); 3.38 return 0; 3.39 @@ -379,7 +379,7 @@ static int get_page_and_type_from_pagenr 3.40 */ 3.41 static int get_linear_pagetable(l2_pgentry_t l2e, unsigned long pfn) 3.42 { 3.43 - unsigned long x, y; 3.44 + u32 x, y; 3.45 struct pfn_info *page; 3.46 3.47 if ( (l2_pgentry_val(l2e) & _PAGE_RW) ) 3.48 @@ -1207,7 +1207,7 @@ void __audit_page(unsigned long pfn) { 3.49 page = &frame_table[pfn]; 3.50 page_addr = pfn << PAGE_SHIFT; 3.51 3.52 - printk("audit page: pfn=%lx info: cf=%lx tf=%lx ts=%lx dom=%lx\n", pfn, 3.53 + printk("audit page: pfn=%lx info: cf=%x tf=%x ts=%x dom=%lx\n", pfn, 3.54 page->count_and_flags, page->type_and_flags, 3.55 page->tlbflush_timestamp, (unsigned long)page->u.domain); 3.56 3.57 @@ -1234,7 +1234,7 @@ void __audit_page(unsigned long pfn) { 3.58 continue; 3.59 if ( l1_pgentry_to_pagenr(l1e) == pfn ) 3.60 { 3.61 - printk(" pte_pfn=%06lx cf=%08lx tf=%08lx dom=%08lx\n", 3.62 + printk(" pte_pfn=%06lx cf=%08x tf=%08x dom=%08lx\n", 3.63 i, frame_table[i].count_and_flags, 3.64 frame_table[i].type_and_flags, 3.65 (unsigned long)frame_table[i].u.domain); 3.66 @@ -1311,7 +1311,7 @@ void audit_all_pages(u_char key, void *d 3.67 if ( ((frame_table[i].count_and_flags & PGC_count_mask) != 0) && 3.68 ((frame_table[i].count_and_flags & PGC_zombie) != 0) ) 3.69 { 3.70 - printk("zombie: pfn=%08lx cf=%08lx tf=%08lx dom=%08lx\n", 3.71 + printk("zombie: pfn=%08lx cf=%08x tf=%08x dom=%08lx\n", 3.72 i, frame_table[i].count_and_flags, 3.73 frame_table[i].type_and_flags, 3.74 (unsigned long)frame_table[i].u.domain); 3.75 @@ -1356,7 +1356,7 @@ void audit_all_pages(u_char key, void *d 3.76 if ( (frame_table[i].count_and_flags & PGC_count_mask) 3.77 != ref_count ) 3.78 { 3.79 - printk("refcount error: pfn=%06lx cf=%08lx refcount=%lx\n", 3.80 + printk("refcount error: pfn=%06lx cf=%08x refcount=%lx\n", 3.81 i, frame_table[i].count_and_flags, ref_count); 3.82 __audit_page(i); 3.83 printk("\n");
4.1 --- a/xen/common/shadow.c Tue Apr 13 16:30:13 2004 +0000 4.2 +++ b/xen/common/shadow.c Wed Apr 14 16:15:47 2004 +0000 4.3 @@ -27,20 +27,20 @@ hypercall lock anyhow (at least initiall 4.4 ********/ 4.5 4.6 static inline void free_shadow_page( struct mm_struct *m, 4.7 - struct pfn_info *pfn_info ) 4.8 + struct pfn_info *pfn_info ) 4.9 { 4.10 - unsigned long flags; 4.11 + unsigned long flags; 4.12 unsigned long type = pfn_info->type_and_flags & PGT_type_mask; 4.13 4.14 m->shadow_page_count--; 4.15 4.16 if (type == PGT_l1_page_table) 4.17 - perfc_decr(shadow_l1_pages); 4.18 + perfc_decr(shadow_l1_pages); 4.19 else if (type == PGT_l2_page_table) 4.20 - perfc_decr(shadow_l2_pages); 4.21 - else printk("Free shadow weird page type pfn=%08x type=%08lx\n", 4.22 - frame_table-pfn_info, pfn_info->type_and_flags); 4.23 - 4.24 + perfc_decr(shadow_l2_pages); 4.25 + else printk("Free shadow weird page type pfn=%08x type=%08x\n", 4.26 + frame_table-pfn_info, pfn_info->type_and_flags); 4.27 + 4.28 pfn_info->type_and_flags = 0; 4.29 4.30 spin_lock_irqsave(&free_list_lock, flags); 4.31 @@ -53,81 +53,7 @@ static void __free_shadow_table( struct 4.32 { 4.33 int j, free=0; 4.34 struct shadow_status *a,*next; 4.35 - 4.36 - // the code assumes you're not using the page tables i.e. 4.37 - // the domain is stopped and cr3 is something else!! 4.38 - 4.39 - // walk the hash table and call free_shadow_page on all pages 4.40 - 4.41 - shadow_audit(m,1); 4.42 - 4.43 - for(j=0;j<shadow_ht_buckets;j++) 4.44 - { 4.45 - a = &m->shadow_ht[j]; 4.46 - if (a->pfn) 4.47 - { 4.48 - free_shadow_page( m, 4.49 - &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.50 - a->pfn = 0; 4.51 - a->spfn_and_flags = 0; 4.52 - free++; 4.53 - } 4.54 - next=a->next; 4.55 - a->next=NULL; 4.56 - a=next; 4.57 - while(a) 4.58 - { 4.59 - struct shadow_status *next = a->next; 4.60 - 4.61 - free_shadow_page( m, 4.62 - &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.63 - a->pfn = 0; 4.64 - a->spfn_and_flags = 0; 4.65 - free++; 4.66 - a->next = m->shadow_ht_free; 4.67 - m->shadow_ht_free = a; 4.68 - a=next; 4.69 - } 4.70 - shadow_audit(m,0); 4.71 - } 4.72 - SH_LOG("Free shadow table. Freed= %d",free); 4.73 -} 4.74 - 4.75 -static inline int shadow_page_op( struct mm_struct *m, unsigned int op, 4.76 - struct pfn_info *spfn_info ) 4.77 -{ 4.78 - int work = 0; 4.79 - unsigned int spfn = spfn_info-frame_table; 4.80 - 4.81 - switch( op ) 4.82 - { 4.83 - case DOM0_SHADOW_CONTROL_OP_CLEAN: 4.84 - { 4.85 - int i; 4.86 - if ( (spfn_info->type_and_flags & PGT_type_mask) == 4.87 - PGT_l1_page_table ) 4.88 - { 4.89 - unsigned long * spl1e = map_domain_mem( spfn<<PAGE_SHIFT ); 4.90 - 4.91 - for (i=0;i<ENTRIES_PER_L1_PAGETABLE;i++) 4.92 - { 4.93 - if ( spl1e[i] & _PAGE_RW ) 4.94 - { 4.95 - work++; 4.96 - spl1e[i] &= ~_PAGE_RW; 4.97 - } 4.98 - } 4.99 - unmap_domain_mem( spl1e ); 4.100 - } 4.101 - } 4.102 - } 4.103 - return work; 4.104 -} 4.105 -static void __scan_shadow_table( struct mm_struct *m, unsigned int op ) 4.106 -{ 4.107 - int j, work=0; 4.108 - struct shadow_status *a; 4.109 - 4.110 + 4.111 // the code assumes you're not using the page tables i.e. 4.112 // the domain is stopped and cr3 is something else!! 4.113 4.114 @@ -137,18 +63,92 @@ static void __scan_shadow_table( struct 4.115 4.116 for(j=0;j<shadow_ht_buckets;j++) 4.117 { 4.118 - a = &m->shadow_ht[j]; 4.119 - if (a->pfn) 4.120 + a = &m->shadow_ht[j]; 4.121 + if (a->pfn) 4.122 { 4.123 - work += shadow_page_op( m, op, &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.124 + free_shadow_page( m, 4.125 + &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.126 + a->pfn = 0; 4.127 + a->spfn_and_flags = 0; 4.128 + free++; 4.129 + } 4.130 + next=a->next; 4.131 + a->next=NULL; 4.132 + a=next; 4.133 + while(a) 4.134 + { 4.135 + struct shadow_status *next = a->next; 4.136 + 4.137 + free_shadow_page( m, 4.138 + &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.139 + a->pfn = 0; 4.140 + a->spfn_and_flags = 0; 4.141 + free++; 4.142 + a->next = m->shadow_ht_free; 4.143 + m->shadow_ht_free = a; 4.144 + a=next; 4.145 } 4.146 - a=a->next; 4.147 - while(a) 4.148 - { 4.149 - work += shadow_page_op( m, op, &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.150 - a=a->next; 4.151 - } 4.152 - shadow_audit(m,0); 4.153 + shadow_audit(m,0); 4.154 + } 4.155 + SH_LOG("Free shadow table. Freed= %d",free); 4.156 +} 4.157 + 4.158 +static inline int shadow_page_op( struct mm_struct *m, unsigned int op, 4.159 + struct pfn_info *spfn_info ) 4.160 +{ 4.161 + int work = 0; 4.162 + unsigned int spfn = spfn_info-frame_table; 4.163 + 4.164 + switch( op ) 4.165 + { 4.166 + case DOM0_SHADOW_CONTROL_OP_CLEAN: 4.167 + { 4.168 + int i; 4.169 + if ( (spfn_info->type_and_flags & PGT_type_mask) == 4.170 + PGT_l1_page_table ) 4.171 + { 4.172 + unsigned long * spl1e = map_domain_mem( spfn<<PAGE_SHIFT ); 4.173 + 4.174 + for (i=0;i<ENTRIES_PER_L1_PAGETABLE;i++) 4.175 + { 4.176 + if ( spl1e[i] & _PAGE_RW ) 4.177 + { 4.178 + work++; 4.179 + spl1e[i] &= ~_PAGE_RW; 4.180 + } 4.181 + } 4.182 + unmap_domain_mem( spl1e ); 4.183 + } 4.184 + } 4.185 + } 4.186 + return work; 4.187 +} 4.188 +static void __scan_shadow_table( struct mm_struct *m, unsigned int op ) 4.189 +{ 4.190 + int j, work=0; 4.191 + struct shadow_status *a; 4.192 + 4.193 + // the code assumes you're not using the page tables i.e. 4.194 + // the domain is stopped and cr3 is something else!! 4.195 + 4.196 + // walk the hash table and call free_shadow_page on all pages 4.197 + 4.198 + shadow_audit(m,1); 4.199 + 4.200 + for(j=0;j<shadow_ht_buckets;j++) 4.201 + { 4.202 + a = &m->shadow_ht[j]; 4.203 + if (a->pfn) 4.204 + { 4.205 + work += shadow_page_op( m, op, &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.206 + } 4.207 + a=a->next; 4.208 + while(a) 4.209 + { 4.210 + work += shadow_page_op( m, op, &frame_table[a->spfn_and_flags & PSH_pfn_mask] ); 4.211 + a=a->next; 4.212 + } 4.213 + shadow_audit(m,0); 4.214 } 4.215 SH_LOG("Scan shadow table. Work=%d l1=%d l2=%d", work, perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages)); 4.216 } 4.217 @@ -165,35 +165,35 @@ int shadow_mode_enable( struct task_stru 4.218 spin_lock(&m->shadow_lock); 4.219 4.220 m->shadow_mode = mode; 4.221 - 4.222 + 4.223 // allocate hashtable 4.224 m->shadow_ht = kmalloc( shadow_ht_buckets * 4.225 - sizeof(struct shadow_status), GFP_KERNEL ); 4.226 + sizeof(struct shadow_status), GFP_KERNEL ); 4.227 if( ! m->shadow_ht ) 4.228 - goto nomem; 4.229 + goto nomem; 4.230 4.231 memset( m->shadow_ht, 0, shadow_ht_buckets * 4.232 - sizeof(struct shadow_status) ); 4.233 + sizeof(struct shadow_status) ); 4.234 4.235 4.236 // allocate space for first lot of extra nodes 4.237 m->shadow_ht_extras = kmalloc( sizeof(void*) + (shadow_ht_extra_size * 4.238 - sizeof(struct shadow_status)), GFP_KERNEL ); 4.239 + sizeof(struct shadow_status)), GFP_KERNEL ); 4.240 4.241 if( ! m->shadow_ht_extras ) 4.242 - goto nomem; 4.243 + goto nomem; 4.244 4.245 memset( m->shadow_ht_extras, 0, sizeof(void*) + (shadow_ht_extra_size * 4.246 - sizeof(struct shadow_status)) ); 4.247 + sizeof(struct shadow_status)) ); 4.248 4.249 m->shadow_extras_count++; 4.250 - 4.251 + 4.252 // add extras to free list 4.253 fptr = &m->shadow_ht_free; 4.254 for ( i=0; i<shadow_ht_extra_size; i++ ) 4.255 { 4.256 - *fptr = &m->shadow_ht_extras[i]; 4.257 - fptr = &(m->shadow_ht_extras[i].next); 4.258 + *fptr = &m->shadow_ht_extras[i]; 4.259 + fptr = &(m->shadow_ht_extras[i].next); 4.260 } 4.261 *fptr = NULL; 4.262 *((struct shadow_status ** ) 4.263 @@ -201,16 +201,16 @@ int shadow_mode_enable( struct task_stru 4.264 4.265 if ( mode == SHM_logdirty ) 4.266 { 4.267 - m->shadow_dirty_bitmap_size = (p->max_pages+63)&(~63); 4.268 - m->shadow_dirty_bitmap = 4.269 - kmalloc( m->shadow_dirty_bitmap_size/8, GFP_KERNEL ); 4.270 + m->shadow_dirty_bitmap_size = (p->max_pages+63)&(~63); 4.271 + m->shadow_dirty_bitmap = 4.272 + kmalloc( m->shadow_dirty_bitmap_size/8, GFP_KERNEL ); 4.273 4.274 - if( !m->shadow_dirty_bitmap ) 4.275 - { 4.276 - m->shadow_dirty_bitmap_size = 0; 4.277 - goto nomem; 4.278 - } 4.279 - memset(m->shadow_dirty_bitmap,0,m->shadow_dirty_bitmap_size/8); 4.280 + if( !m->shadow_dirty_bitmap ) 4.281 + { 4.282 + m->shadow_dirty_bitmap_size = 0; 4.283 + goto nomem; 4.284 + } 4.285 + memset(m->shadow_dirty_bitmap,0,m->shadow_dirty_bitmap_size/8); 4.286 } 4.287 4.288 spin_unlock(&m->shadow_lock); 4.289 @@ -220,7 +220,7 @@ int shadow_mode_enable( struct task_stru 4.290 4.291 return 0; 4.292 4.293 -nomem: 4.294 + nomem: 4.295 spin_unlock(&m->shadow_lock); 4.296 return -ENOMEM; 4.297 } 4.298 @@ -236,24 +236,24 @@ void shadow_mode_disable( struct task_st 4.299 spin_unlock(&m->shadow_lock); 4.300 4.301 SH_LOG("freed tables count=%d l1=%d l2=%d", 4.302 - m->shadow_page_count, perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages)); 4.303 + m->shadow_page_count, perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages)); 4.304 4.305 next = m->shadow_ht_extras; 4.306 while( next ) 4.307 { 4.308 - struct shadow_status * this = next; 4.309 - m->shadow_extras_count--; 4.310 - next = *((struct shadow_status **)(&next[shadow_ht_extra_size])); 4.311 - kfree( this ); 4.312 + struct shadow_status * this = next; 4.313 + m->shadow_extras_count--; 4.314 + next = *((struct shadow_status **)(&next[shadow_ht_extra_size])); 4.315 + kfree( this ); 4.316 } 4.317 4.318 SH_LOG("freed extras, now %d", m->shadow_extras_count); 4.319 4.320 if( m->shadow_dirty_bitmap ) 4.321 { 4.322 - kfree( m->shadow_dirty_bitmap ); 4.323 - m->shadow_dirty_bitmap = 0; 4.324 - m->shadow_dirty_bitmap_size = 0; 4.325 + kfree( m->shadow_dirty_bitmap ); 4.326 + m->shadow_dirty_bitmap = 0; 4.327 + m->shadow_dirty_bitmap_size = 0; 4.328 } 4.329 4.330 // free the hashtable itself 4.331 @@ -270,8 +270,8 @@ static void shadow_mode_table_op( struct 4.332 4.333 if ( m == ¤t->mm ) 4.334 { 4.335 - printk("Don't try and flush your own page tables!\n"); 4.336 - return; 4.337 + printk("Don't try and flush your own page tables!\n"); 4.338 + return; 4.339 } 4.340 4.341 4.342 @@ -284,14 +284,14 @@ static void shadow_mode_table_op( struct 4.343 switch(op) 4.344 { 4.345 case DOM0_SHADOW_CONTROL_OP_FLUSH: 4.346 - __free_shadow_table( m ); 4.347 - break; 4.348 + __free_shadow_table( m ); 4.349 + break; 4.350 4.351 case DOM0_SHADOW_CONTROL_OP_CLEAN: 4.352 - __scan_shadow_table( m, op ); 4.353 - // we used to bzero dirty bitmap here, but now leave this to user space 4.354 - // if we were double buffering we'd do the flip here 4.355 - break; 4.356 + __scan_shadow_table( m, op ); 4.357 + // we used to bzero dirty bitmap here, but now leave this to user space 4.358 + // if we were double buffering we'd do the flip here 4.359 + break; 4.360 } 4.361 4.362 spin_unlock(&m->shadow_lock); 4.363 @@ -315,29 +315,29 @@ int shadow_mode_control( struct task_str 4.364 // sychronously stop domain 4.365 if( 0 && !(p->state & TASK_STOPPED) && !(p->state & TASK_PAUSED)) 4.366 { 4.367 - printk("about to pause domain\n"); 4.368 - sched_pause_sync(p); 4.369 - printk("paused domain\n"); 4.370 - we_paused = 1; 4.371 + printk("about to pause domain\n"); 4.372 + sched_pause_sync(p); 4.373 + printk("paused domain\n"); 4.374 + we_paused = 1; 4.375 } 4.376 4.377 if ( p->mm.shadow_mode && op == DOM0_SHADOW_CONTROL_OP_OFF ) 4.378 { 4.379 - shadow_mode_disable(p); 4.380 + shadow_mode_disable(p); 4.381 } 4.382 else if ( op == DOM0_SHADOW_CONTROL_OP_ENABLE_TEST ) 4.383 { 4.384 - if(p->mm.shadow_mode) shadow_mode_disable(p); 4.385 - shadow_mode_enable(p, SHM_test); 4.386 - } 4.387 + if(p->mm.shadow_mode) shadow_mode_disable(p); 4.388 + shadow_mode_enable(p, SHM_test); 4.389 + } 4.390 else if ( p->mm.shadow_mode && op >= DOM0_SHADOW_CONTROL_OP_FLUSH && op<=DOM0_SHADOW_CONTROL_OP_CLEAN ) 4.391 { 4.392 - shadow_mode_table_op(p, op); 4.393 + shadow_mode_table_op(p, op); 4.394 } 4.395 else 4.396 { 4.397 - if ( we_paused ) wake_up(p); 4.398 - return -EINVAL; 4.399 + if ( we_paused ) wake_up(p); 4.400 + return -EINVAL; 4.401 } 4.402 4.403 if ( we_paused ) wake_up(p); 4.404 @@ -359,8 +359,8 @@ void unshadow_table( unsigned long gpfn, 4.405 unsigned long spfn; 4.406 4.407 SH_VLOG("unshadow_table type=%08x gpfn=%08lx", 4.408 - type, 4.409 - gpfn ); 4.410 + type, 4.411 + gpfn ); 4.412 4.413 perfc_incrc(unshadow_table_count); 4.414 4.415 @@ -404,21 +404,21 @@ unsigned long shadow_l2_table( 4.416 4.417 // mark pfn as being shadowed, update field to point at shadow 4.418 set_shadow_status(m, gpfn, spfn | PSH_shadowed); 4.419 - 4.420 + 4.421 // we need to do this before the linear map is set up 4.422 spl2e = (l2_pgentry_t *) map_domain_mem(spfn << PAGE_SHIFT); 4.423 4.424 // get hypervisor and 2x linear PT mapings installed 4.425 memcpy(&spl2e[DOMAIN_ENTRIES_PER_L2_PAGETABLE], 4.426 - &idle_pg_table[DOMAIN_ENTRIES_PER_L2_PAGETABLE], 4.427 - HYPERVISOR_ENTRIES_PER_L2_PAGETABLE * sizeof(l2_pgentry_t)); 4.428 + &idle_pg_table[DOMAIN_ENTRIES_PER_L2_PAGETABLE], 4.429 + HYPERVISOR_ENTRIES_PER_L2_PAGETABLE * sizeof(l2_pgentry_t)); 4.430 spl2e[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] = 4.431 - mk_l2_pgentry((gpfn << PAGE_SHIFT) | __PAGE_HYPERVISOR); 4.432 + mk_l2_pgentry((gpfn << PAGE_SHIFT) | __PAGE_HYPERVISOR); 4.433 spl2e[SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] = 4.434 - mk_l2_pgentry((spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR); 4.435 + mk_l2_pgentry((spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR); 4.436 spl2e[PERDOMAIN_VIRT_START >> L2_PAGETABLE_SHIFT] = 4.437 - mk_l2_pgentry(__pa(frame_table[gpfn].u.domain->mm.perdomain_pt) | 4.438 - __PAGE_HYPERVISOR); 4.439 + mk_l2_pgentry(__pa(frame_table[gpfn].u.domain->mm.perdomain_pt) | 4.440 + __PAGE_HYPERVISOR); 4.441 4.442 // can't use the linear map as we may not be in the right PT 4.443 gpl2e = (l2_pgentry_t *) map_domain_mem(gpfn << PAGE_SHIFT); 4.444 @@ -426,24 +426,24 @@ unsigned long shadow_l2_table( 4.445 // proactively create entries for pages that are already shadowed 4.446 for ( i = 0; i < DOMAIN_ENTRIES_PER_L2_PAGETABLE; i++ ) 4.447 { 4.448 - unsigned long spte = 0; 4.449 + unsigned long spte = 0; 4.450 4.451 #if 0 // Turns out this doesn't really help 4.452 - unsigned long gpte; 4.453 + unsigned long gpte; 4.454 4.455 - gpte = l2_pgentry_val(gpl2e[i]); 4.456 + gpte = l2_pgentry_val(gpl2e[i]); 4.457 4.458 - if (gpte & _PAGE_PRESENT) 4.459 - { 4.460 - unsigned long s_sh = 4.461 - __shadow_status(p, gpte>>PAGE_SHIFT); 4.462 + if (gpte & _PAGE_PRESENT) 4.463 + { 4.464 + unsigned long s_sh = 4.465 + __shadow_status(p, gpte>>PAGE_SHIFT); 4.466 4.467 - l2pde_general( m, &gpte, &spte, s_sh ); 4.468 + l2pde_general( m, &gpte, &spte, s_sh ); 4.469 4.470 - } 4.471 + } 4.472 #endif 4.473 4.474 - spl2e[i] = mk_l2_pgentry( spte ); 4.475 + spl2e[i] = mk_l2_pgentry( spte ); 4.476 4.477 } 4.478 4.479 @@ -470,20 +470,20 @@ int shadow_fault( unsigned long va, long 4.480 4.481 if ( unlikely(__get_user(gpte, (unsigned long*)&linear_pg_table[va>>PAGE_SHIFT])) ) 4.482 { 4.483 - SH_VVLOG("shadow_fault - EXIT: read gpte faulted" ); 4.484 - return 0; // propagate to guest 4.485 + SH_VVLOG("shadow_fault - EXIT: read gpte faulted" ); 4.486 + return 0; // propagate to guest 4.487 } 4.488 4.489 if ( ! (gpte & _PAGE_PRESENT) ) 4.490 { 4.491 - SH_VVLOG("shadow_fault - EXIT: gpte not present (%lx)",gpte ); 4.492 - return 0; // we're not going to be able to help 4.493 + SH_VVLOG("shadow_fault - EXIT: gpte not present (%lx)",gpte ); 4.494 + return 0; // we're not going to be able to help 4.495 } 4.496 4.497 if ( (error_code & 2) && ! (gpte & _PAGE_RW) ) 4.498 { 4.499 - // write fault on RO page 4.500 - return 0; 4.501 + // write fault on RO page 4.502 + return 0; 4.503 } 4.504 4.505 spin_lock(¤t->mm.shadow_lock); 4.506 @@ -491,35 +491,35 @@ int shadow_fault( unsigned long va, long 4.507 4.508 if ( unlikely(__get_user(gpte, (unsigned long*)&linear_pg_table[va>>PAGE_SHIFT])) ) 4.509 { 4.510 - SH_VVLOG("shadow_fault - EXIT: read gpte faulted" ); 4.511 - spin_unlock(&m->shadow_lock); 4.512 - return 0; // propagate to guest 4.513 + SH_VVLOG("shadow_fault - EXIT: read gpte faulted" ); 4.514 + spin_unlock(&m->shadow_lock); 4.515 + return 0; // propagate to guest 4.516 } 4.517 4.518 if ( unlikely(!(gpte & _PAGE_PRESENT)) ) 4.519 { 4.520 - SH_VVLOG("shadow_fault - EXIT: gpte not present (%lx)",gpte ); 4.521 - spin_unlock(&m->shadow_lock); 4.522 - return 0; // we're not going to be able to help 4.523 + SH_VVLOG("shadow_fault - EXIT: gpte not present (%lx)",gpte ); 4.524 + spin_unlock(&m->shadow_lock); 4.525 + return 0; // we're not going to be able to help 4.526 } 4.527 4.528 if ( error_code & 2 ) 4.529 { // write fault 4.530 - if ( likely(gpte & _PAGE_RW) ) 4.531 - { 4.532 - l1pte_write_fault( m, &gpte, &spte ); 4.533 - } 4.534 - else 4.535 - { // write fault on RO page 4.536 - SH_VVLOG("shadow_fault - EXIT: write fault on RO page (%lx)",gpte ); 4.537 - spin_unlock(&m->shadow_lock); 4.538 - return 0; // propagate to guest 4.539 - // not clear whether we should set accessed bit here... 4.540 - } 4.541 + if ( likely(gpte & _PAGE_RW) ) 4.542 + { 4.543 + l1pte_write_fault( m, &gpte, &spte ); 4.544 + } 4.545 + else 4.546 + { // write fault on RO page 4.547 + SH_VVLOG("shadow_fault - EXIT: write fault on RO page (%lx)",gpte ); 4.548 + spin_unlock(&m->shadow_lock); 4.549 + return 0; // propagate to guest 4.550 + // not clear whether we should set accessed bit here... 4.551 + } 4.552 } 4.553 else 4.554 { 4.555 - l1pte_read_fault( m, &gpte, &spte ); 4.556 + l1pte_read_fault( m, &gpte, &spte ); 4.557 } 4.558 4.559 SH_VVLOG("plan: gpte=%08lx spte=%08lx", gpte, spte ); 4.560 @@ -527,77 +527,77 @@ int shadow_fault( unsigned long va, long 4.561 // write back updated gpte 4.562 // XXX watch out for read-only L2 entries! (not used in Linux) 4.563 if ( unlikely( __put_user( gpte, (unsigned long*)&linear_pg_table[va>>PAGE_SHIFT])) ) 4.564 - BUG(); // fixme! 4.565 + BUG(); // fixme! 4.566 4.567 if ( unlikely( __put_user( spte, (unsigned long*)&shadow_linear_pg_table[va>>PAGE_SHIFT])) ) 4.568 { 4.569 - // failed: 4.570 - // the L1 may not be shadowed, or the L2 entry may be insufficient 4.571 + // failed: 4.572 + // the L1 may not be shadowed, or the L2 entry may be insufficient 4.573 4.574 - unsigned long gpde, spde, gl1pfn, sl1pfn; 4.575 + unsigned long gpde, spde, gl1pfn, sl1pfn; 4.576 4.577 - SH_VVLOG("3: not shadowed or l2 insufficient gpte=%08lx spte=%08lx",gpte,spte ); 4.578 + SH_VVLOG("3: not shadowed or l2 insufficient gpte=%08lx spte=%08lx",gpte,spte ); 4.579 4.580 - gpde = l2_pgentry_val(linear_l2_table[va>>L2_PAGETABLE_SHIFT]); 4.581 + gpde = l2_pgentry_val(linear_l2_table[va>>L2_PAGETABLE_SHIFT]); 4.582 4.583 - gl1pfn = gpde>>PAGE_SHIFT; 4.584 + gl1pfn = gpde>>PAGE_SHIFT; 4.585 4.586 4.587 - if ( ! (sl1pfn=__shadow_status(¤t->mm, gl1pfn) ) ) 4.588 + if ( ! (sl1pfn=__shadow_status(¤t->mm, gl1pfn) ) ) 4.589 { 4.590 - // this L1 is NOT already shadowed so we need to shadow it 4.591 - struct pfn_info *sl1pfn_info; 4.592 - unsigned long *gpl1e, *spl1e; 4.593 - int i; 4.594 - sl1pfn_info = alloc_shadow_page( ¤t->mm ); 4.595 - sl1pfn_info->type_and_flags = PGT_l1_page_table; 4.596 + // this L1 is NOT already shadowed so we need to shadow it 4.597 + struct pfn_info *sl1pfn_info; 4.598 + unsigned long *gpl1e, *spl1e; 4.599 + int i; 4.600 + sl1pfn_info = alloc_shadow_page( ¤t->mm ); 4.601 + sl1pfn_info->type_and_flags = PGT_l1_page_table; 4.602 4.603 - sl1pfn = sl1pfn_info - frame_table; 4.604 + sl1pfn = sl1pfn_info - frame_table; 4.605 4.606 - SH_VVLOG("4a: l1 not shadowed ( %08lx )",sl1pfn); 4.607 - perfc_incrc(shadow_l1_table_count); 4.608 - perfc_incr(shadow_l1_pages); 4.609 + SH_VVLOG("4a: l1 not shadowed ( %08lx )",sl1pfn); 4.610 + perfc_incrc(shadow_l1_table_count); 4.611 + perfc_incr(shadow_l1_pages); 4.612 4.613 - set_shadow_status(¤t->mm, gl1pfn, PSH_shadowed | sl1pfn); 4.614 + set_shadow_status(¤t->mm, gl1pfn, PSH_shadowed | sl1pfn); 4.615 4.616 - l2pde_general( m, &gpde, &spde, sl1pfn ); 4.617 + l2pde_general( m, &gpde, &spde, sl1pfn ); 4.618 4.619 - linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(gpde); 4.620 - shadow_linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(spde); 4.621 + linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(gpde); 4.622 + shadow_linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(spde); 4.623 4.624 - gpl1e = (unsigned long *) &(linear_pg_table[ 4.625 - (va>>PAGE_SHIFT) & ~(ENTRIES_PER_L1_PAGETABLE-1) ]); 4.626 + gpl1e = (unsigned long *) &(linear_pg_table[ 4.627 + (va>>PAGE_SHIFT) & ~(ENTRIES_PER_L1_PAGETABLE-1) ]); 4.628 4.629 - spl1e = (unsigned long *) &shadow_linear_pg_table[ 4.630 - (va>>PAGE_SHIFT) & ~(ENTRIES_PER_L1_PAGETABLE-1) ]; 4.631 + spl1e = (unsigned long *) &shadow_linear_pg_table[ 4.632 + (va>>PAGE_SHIFT) & ~(ENTRIES_PER_L1_PAGETABLE-1) ]; 4.633 4.634 4.635 - for ( i = 0; i < ENTRIES_PER_L1_PAGETABLE; i++ ) 4.636 - { 4.637 - l1pte_no_fault( m, &gpl1e[i], &spl1e[i] ); 4.638 + for ( i = 0; i < ENTRIES_PER_L1_PAGETABLE; i++ ) 4.639 + { 4.640 + l1pte_no_fault( m, &gpl1e[i], &spl1e[i] ); 4.641 } 4.642 4.643 4.644 } 4.645 - else 4.646 + else 4.647 { 4.648 - // this L1 was shadowed (by another PT) but we didn't have an L2 4.649 - // entry for it 4.650 + // this L1 was shadowed (by another PT) but we didn't have an L2 4.651 + // entry for it 4.652 4.653 - SH_VVLOG("4b: was shadowed, l2 missing ( %08lx )",sl1pfn); 4.654 + SH_VVLOG("4b: was shadowed, l2 missing ( %08lx )",sl1pfn); 4.655 4.656 - l2pde_general( m, &gpde, &spde, sl1pfn ); 4.657 + l2pde_general( m, &gpde, &spde, sl1pfn ); 4.658 4.659 - linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(gpde); 4.660 - shadow_linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(spde); 4.661 - 4.662 + linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(gpde); 4.663 + shadow_linear_l2_table[va>>L2_PAGETABLE_SHIFT] = mk_l2_pgentry(spde); 4.664 + 4.665 } 4.666 4.667 - shadow_linear_pg_table[va>>PAGE_SHIFT] = mk_l1_pgentry(spte); 4.668 - // (we need to do the above even if we've just made the shadow L1) 4.669 + shadow_linear_pg_table[va>>PAGE_SHIFT] = mk_l1_pgentry(spte); 4.670 + // (we need to do the above even if we've just made the shadow L1) 4.671 4.672 } // end of fixup writing the shadow L1 directly failed 4.673 - 4.674 + 4.675 perfc_incrc(shadow_fixup_count); 4.676 4.677 check_pagetable( current, current->mm.pagetable, "post-sf" ); 4.678 @@ -611,14 +611,14 @@ int shadow_fault( unsigned long va, long 4.679 4.680 void shadow_l1_normal_pt_update( unsigned long pa, unsigned long gpte, 4.681 unsigned long *prev_spfn_ptr, 4.682 - l1_pgentry_t **prev_spl1e_ptr ) 4.683 + l1_pgentry_t **prev_spl1e_ptr ) 4.684 { 4.685 unsigned long gpfn, spfn, spte, prev_spfn = *prev_spfn_ptr; 4.686 l1_pgentry_t * spl1e, * prev_spl1e = *prev_spl1e_ptr; 4.687 4.688 4.689 SH_VVLOG("shadow_l1_normal_pt_update pa=%08lx, gpte=%08lx, prev_spfn=%08lx, prev_spl1e=%p\n", 4.690 - pa,gpte,prev_spfn, prev_spl1e); 4.691 + pa,gpte,prev_spfn, prev_spl1e); 4.692 4.693 // to get here, we know the l1 page *must* be shadowed 4.694 4.695 @@ -627,14 +627,14 @@ void shadow_l1_normal_pt_update( unsigne 4.696 4.697 if ( spfn == prev_spfn ) 4.698 { 4.699 - spl1e = prev_spl1e; 4.700 + spl1e = prev_spl1e; 4.701 } 4.702 else 4.703 { 4.704 - if( prev_spl1e ) unmap_domain_mem( prev_spl1e ); 4.705 - spl1e = (l1_pgentry_t *) map_domain_mem( spfn << PAGE_SHIFT ); 4.706 - *prev_spfn_ptr = spfn; 4.707 - *prev_spl1e_ptr = spl1e; 4.708 + if( prev_spl1e ) unmap_domain_mem( prev_spl1e ); 4.709 + spl1e = (l1_pgentry_t *) map_domain_mem( spfn << PAGE_SHIFT ); 4.710 + *prev_spfn_ptr = spfn; 4.711 + *prev_spl1e_ptr = spl1e; 4.712 } 4.713 4.714 // XXX we assume only pagetables can be shadowed; 4.715 @@ -664,17 +664,17 @@ void shadow_l2_normal_pt_update( unsigne 4.716 spte = 0; 4.717 4.718 if( gpte & _PAGE_PRESENT ) 4.719 - s_sh = __shadow_status(¤t->mm, gpte >> PAGE_SHIFT); 4.720 + s_sh = __shadow_status(¤t->mm, gpte >> PAGE_SHIFT); 4.721 4.722 sp2le = (l2_pgentry_t *) map_domain_mem( spfn << PAGE_SHIFT ); 4.723 // no real need for a cache here 4.724 4.725 - l2pde_general( ¤t->mm, &gpte, &spte, s_sh ); 4.726 + l2pde_general( ¤t->mm, &gpte, &spte, s_sh ); 4.727 4.728 // XXXX Should mark guest pte as DIRTY and ACCESSED too!!!!! 4.729 4.730 sp2le[(pa & ~PAGE_MASK) / sizeof(l2_pgentry_t) ] = 4.731 - mk_l2_pgentry( spte ); 4.732 + mk_l2_pgentry( spte ); 4.733 4.734 unmap_domain_mem( (void *) sp2le ); 4.735 } 4.736 @@ -690,57 +690,57 @@ char * sh_check_name; 4.737 {printk("XXX %s-FAIL (%d,%d)" _f " g=%08lx s=%08lx\n", sh_check_name, level, i, ## _a , gpte, spte ); BUG();} 4.738 4.739 static int check_pte( struct mm_struct *m, 4.740 - unsigned long gpte, unsigned long spte, int level, int i ) 4.741 + unsigned long gpte, unsigned long spte, int level, int i ) 4.742 { 4.743 unsigned long mask, gpfn, spfn; 4.744 4.745 if ( spte == 0 || spte == 0xdeadface || spte == 0x00000E00) 4.746 - return 1; // always safe 4.747 + return 1; // always safe 4.748 4.749 if ( !(spte & _PAGE_PRESENT) ) 4.750 - FAIL("Non zero not present spte"); 4.751 + FAIL("Non zero not present spte"); 4.752 4.753 if( level == 2 ) sh_l2_present++; 4.754 if( level == 1 ) sh_l1_present++; 4.755 4.756 if ( !(gpte & _PAGE_PRESENT) ) 4.757 - FAIL("Guest not present yet shadow is"); 4.758 + FAIL("Guest not present yet shadow is"); 4.759 4.760 mask = ~(_PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_RW|0xFFFFF000); 4.761 4.762 if ( (spte & mask) != (gpte & mask ) ) 4.763 - FAIL("Corrupt?"); 4.764 + FAIL("Corrupt?"); 4.765 4.766 if ( (spte & _PAGE_DIRTY ) && !(gpte & _PAGE_DIRTY) ) 4.767 - FAIL("Dirty coherence"); 4.768 + FAIL("Dirty coherence"); 4.769 4.770 if ( (spte & _PAGE_ACCESSED ) && !(gpte & _PAGE_ACCESSED) ) 4.771 - FAIL("Accessed coherence"); 4.772 + FAIL("Accessed coherence"); 4.773 4.774 if ( (spte & _PAGE_RW ) && !(gpte & _PAGE_RW) ) 4.775 - FAIL("RW coherence"); 4.776 + FAIL("RW coherence"); 4.777 4.778 if ( (spte & _PAGE_RW ) && !((gpte & _PAGE_RW) && (gpte & _PAGE_DIRTY) )) 4.779 - FAIL("RW2 coherence"); 4.780 - 4.781 + FAIL("RW2 coherence"); 4.782 + 4.783 spfn = spte>>PAGE_SHIFT; 4.784 gpfn = gpte>>PAGE_SHIFT; 4.785 4.786 if ( gpfn == spfn ) 4.787 { 4.788 - if ( level > 1 ) 4.789 - FAIL("Linear map ???"); // XXX this will fail on BSD 4.790 + if ( level > 1 ) 4.791 + FAIL("Linear map ???"); // XXX this will fail on BSD 4.792 4.793 - return 1; 4.794 + return 1; 4.795 } 4.796 else 4.797 { 4.798 - if ( level < 2 ) 4.799 - FAIL("Shadow in L1 entry?"); 4.800 + if ( level < 2 ) 4.801 + FAIL("Shadow in L1 entry?"); 4.802 4.803 - if ( __shadow_status(p, gpfn) != (PSH_shadowed | spfn) ) 4.804 - FAIL("spfn problem g.sf=%08lx", 4.805 - __shadow_status(p, gpfn) ); 4.806 + if ( __shadow_status(p, gpfn) != (PSH_shadowed | spfn) ) 4.807 + FAIL("spfn problem g.sf=%08lx", 4.808 + __shadow_status(p, gpfn) ); 4.809 } 4.810 4.811 return 1; 4.812 @@ -748,7 +748,7 @@ static int check_pte( struct mm_struct * 4.813 4.814 4.815 static int check_l1_table( struct mm_struct *m, unsigned long va, 4.816 - unsigned long g2, unsigned long s2 ) 4.817 + unsigned long g2, unsigned long s2 ) 4.818 { 4.819 int j; 4.820 unsigned long *gpl1e, *spl1e; 4.821 @@ -761,12 +761,12 @@ static int check_l1_table( struct mm_str 4.822 4.823 for ( j = 0; j < ENTRIES_PER_L1_PAGETABLE; j++ ) 4.824 { 4.825 - unsigned long gpte = gpl1e[j]; 4.826 - unsigned long spte = spl1e[j]; 4.827 - 4.828 - check_pte( p, gpte, spte, 1, j ); 4.829 + unsigned long gpte = gpl1e[j]; 4.830 + unsigned long spte = spl1e[j]; 4.831 + 4.832 + check_pte( p, gpte, spte, 1, j ); 4.833 } 4.834 - 4.835 + 4.836 unmap_domain_mem( spl1e ); 4.837 unmap_domain_mem( gpl1e ); 4.838 4.839 @@ -793,17 +793,17 @@ int check_pagetable( struct mm_struct *m 4.840 4.841 if ( ! (__shadow_status(p, gpfn) & PSH_shadowed) ) 4.842 { 4.843 - printk("%s-PT %08lx not shadowed\n", s, gptbase); 4.844 + printk("%s-PT %08lx not shadowed\n", s, gptbase); 4.845 4.846 - if( __shadow_status(p, gpfn) != 0 ) BUG(); 4.847 + if( __shadow_status(p, gpfn) != 0 ) BUG(); 4.848 4.849 - return 0; 4.850 + return 0; 4.851 } 4.852 - 4.853 + 4.854 spfn = __shadow_status(p, gpfn) & PSH_pfn_mask; 4.855 4.856 if ( ! __shadow_status(p, gpfn) == (PSH_shadowed | spfn) ) 4.857 - FAILPT("ptbase shadow inconsistent1"); 4.858 + FAILPT("ptbase shadow inconsistent1"); 4.859 4.860 gpl2e = (l2_pgentry_t *) map_domain_mem( gpfn << PAGE_SHIFT ); 4.861 spl2e = (l2_pgentry_t *) map_domain_mem( spfn << PAGE_SHIFT ); 4.862 @@ -812,55 +812,55 @@ int check_pagetable( struct mm_struct *m 4.863 4.864 4.865 if ( memcmp( &spl2e[DOMAIN_ENTRIES_PER_L2_PAGETABLE], 4.866 - &gpl2e[DOMAIN_ENTRIES_PER_L2_PAGETABLE], 4.867 - ((SH_LINEAR_PT_VIRT_START>>(L2_PAGETABLE_SHIFT))-DOMAIN_ENTRIES_PER_L2_PAGETABLE) 4.868 - * sizeof(l2_pgentry_t)) ) 4.869 + &gpl2e[DOMAIN_ENTRIES_PER_L2_PAGETABLE], 4.870 + ((SH_LINEAR_PT_VIRT_START>>(L2_PAGETABLE_SHIFT))-DOMAIN_ENTRIES_PER_L2_PAGETABLE) 4.871 + * sizeof(l2_pgentry_t)) ) 4.872 { 4.873 - printk("gpfn=%08lx spfn=%08lx\n", gpfn, spfn); 4.874 - for (i=DOMAIN_ENTRIES_PER_L2_PAGETABLE; 4.875 - i<(SH_LINEAR_PT_VIRT_START>>(L2_PAGETABLE_SHIFT)); 4.876 - i++ ) 4.877 - printk("+++ (%d) %08lx %08lx\n",i, 4.878 - l2_pgentry_val(gpl2e[i]), l2_pgentry_val(spl2e[i]) ); 4.879 - FAILPT("hypervisor entries inconsistent"); 4.880 + printk("gpfn=%08lx spfn=%08lx\n", gpfn, spfn); 4.881 + for (i=DOMAIN_ENTRIES_PER_L2_PAGETABLE; 4.882 + i<(SH_LINEAR_PT_VIRT_START>>(L2_PAGETABLE_SHIFT)); 4.883 + i++ ) 4.884 + printk("+++ (%d) %08lx %08lx\n",i, 4.885 + l2_pgentry_val(gpl2e[i]), l2_pgentry_val(spl2e[i]) ); 4.886 + FAILPT("hypervisor entries inconsistent"); 4.887 } 4.888 4.889 if ( (l2_pgentry_val(spl2e[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT]) != 4.890 - l2_pgentry_val(gpl2e[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT])) ) 4.891 - FAILPT("hypervisor linear map inconsistent"); 4.892 + l2_pgentry_val(gpl2e[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT])) ) 4.893 + FAILPT("hypervisor linear map inconsistent"); 4.894 4.895 if ( (l2_pgentry_val(spl2e[SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT]) != 4.896 - ((spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR)) ) 4.897 - FAILPT("hypervisor shadow linear map inconsistent %08lx %08lx", 4.898 - l2_pgentry_val(spl2e[SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT]), 4.899 - (spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR 4.900 - ); 4.901 + ((spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR)) ) 4.902 + FAILPT("hypervisor shadow linear map inconsistent %08lx %08lx", 4.903 + l2_pgentry_val(spl2e[SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT]), 4.904 + (spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR 4.905 + ); 4.906 4.907 if ( (l2_pgentry_val(spl2e[PERDOMAIN_VIRT_START >> L2_PAGETABLE_SHIFT]) != 4.908 - ((__pa(frame_table[gpfn].u.domain->mm.perdomain_pt) | __PAGE_HYPERVISOR))) ) 4.909 - FAILPT("hypervisor per-domain map inconsistent"); 4.910 + ((__pa(frame_table[gpfn].u.domain->mm.perdomain_pt) | __PAGE_HYPERVISOR))) ) 4.911 + FAILPT("hypervisor per-domain map inconsistent"); 4.912 4.913 4.914 // check the whole L2 4.915 for ( i = 0; i < DOMAIN_ENTRIES_PER_L2_PAGETABLE; i++ ) 4.916 { 4.917 - unsigned long gpte = l2_pgentry_val(gpl2e[i]); 4.918 - unsigned long spte = l2_pgentry_val(spl2e[i]); 4.919 + unsigned long gpte = l2_pgentry_val(gpl2e[i]); 4.920 + unsigned long spte = l2_pgentry_val(spl2e[i]); 4.921 4.922 - check_pte( p, gpte, spte, 2, i ); 4.923 + check_pte( p, gpte, spte, 2, i ); 4.924 } 4.925 4.926 4.927 // go back and recurse 4.928 for ( i = 0; i < DOMAIN_ENTRIES_PER_L2_PAGETABLE; i++ ) 4.929 { 4.930 - unsigned long gpte = l2_pgentry_val(gpl2e[i]); 4.931 - unsigned long spte = l2_pgentry_val(spl2e[i]); 4.932 + unsigned long gpte = l2_pgentry_val(gpl2e[i]); 4.933 + unsigned long spte = l2_pgentry_val(spl2e[i]); 4.934 4.935 - if ( spte ) 4.936 - check_l1_table( p, 4.937 - i<<L2_PAGETABLE_SHIFT, 4.938 - gpte>>PAGE_SHIFT, spte>>PAGE_SHIFT ); 4.939 + if ( spte ) 4.940 + check_l1_table( p, 4.941 + i<<L2_PAGETABLE_SHIFT, 4.942 + gpte>>PAGE_SHIFT, spte>>PAGE_SHIFT ); 4.943 4.944 } 4.945 4.946 @@ -868,15 +868,10 @@ int check_pagetable( struct mm_struct *m 4.947 unmap_domain_mem( gpl2e ); 4.948 4.949 SH_VVLOG("PT verified : l2_present = %d, l1_present = %d\n", 4.950 - sh_l2_present, sh_l1_present ); 4.951 - 4.952 + sh_l2_present, sh_l1_present ); 4.953 + 4.954 return 1; 4.955 } 4.956 4.957 4.958 #endif 4.959 - 4.960 - 4.961 - 4.962 - 4.963 -
5.1 --- a/xen/include/asm-i386/flushtlb.h Tue Apr 13 16:30:13 2004 +0000 5.2 +++ b/xen/include/asm-i386/flushtlb.h Wed Apr 14 16:15:47 2004 +0000 5.3 @@ -13,23 +13,35 @@ 5.4 #include <xen/smp.h> 5.5 5.6 /* 5.7 - * Every GLOBAL_FLUSH_PERIOD ticks of the tlbflush clock, every TLB in the 5.8 - * system is guaranteed to have been flushed. 5.9 + * Every time the TLB clock passes an "epoch", every CPU's TLB is flushed. 5.10 + * Therefore, if the current TLB time and a previously-read timestamp differ 5.11 + * in their significant bits (i.e., ~TLBCLOCK_EPOCH_MASK), then the TLB clock 5.12 + * has wrapped at least once and every CPU's TLB is guaranteed to have been 5.13 + * flushed meanwhile. 5.14 + * This allows us to deal gracefully with a bounded (a.k.a. wrapping) clock. 5.15 */ 5.16 -#define GLOBAL_FLUSH_PERIOD (1<<16) 5.17 +#define TLBCLOCK_EPOCH_MASK ((1U<<16)-1) 5.18 5.19 /* 5.20 - * '_cpu_stamp' is the current timestamp for the CPU we are testing. 5.21 - * '_lastuse_stamp' is a timestamp taken when the PFN we are testing was last 5.22 + * 'cpu_stamp' is the current timestamp for the CPU we are testing. 5.23 + * 'lastuse_stamp' is a timestamp taken when the PFN we are testing was last 5.24 * used for a purpose that may have caused the CPU's TLB to become tainted. 5.25 */ 5.26 -#define NEED_FLUSH(_cpu_stamp, _lastuse_stamp) \ 5.27 - (((_cpu_stamp) <= (_lastuse_stamp)) && \ 5.28 - (((_lastuse_stamp) - (_cpu_stamp)) <= (2*GLOBAL_FLUSH_PERIOD))) 5.29 +static inline int NEED_FLUSH(u32 cpu_stamp, u32 lastuse_stamp) 5.30 +{ 5.31 + /* 5.32 + * Why does this work? 5.33 + * 1. XOR sets high-order bits determines if stamps from differing epochs. 5.34 + * 2. Subtraction sets high-order bits if 'cpu_stamp > lastuse_stamp'. 5.35 + * In either case a flush is unnecessary: we therefore OR the results from 5.36 + * (1) and (2), mask the high-order bits, and return the inverse. 5.37 + */ 5.38 + return !(((lastuse_stamp^cpu_stamp)|(lastuse_stamp-cpu_stamp)) & 5.39 + ~TLBCLOCK_EPOCH_MASK); 5.40 +} 5.41 5.42 -extern unsigned long tlbflush_mask; 5.43 -extern unsigned long tlbflush_clock; 5.44 -extern unsigned long tlbflush_time[NR_CPUS]; 5.45 +extern u32 tlbflush_clock; 5.46 +extern u32 tlbflush_time[NR_CPUS]; 5.47 5.48 extern void new_tlbflush_clock_period(void); 5.49
6.1 --- a/xen/include/asm-x86_64/flushtlb.h Tue Apr 13 16:30:13 2004 +0000 6.2 +++ b/xen/include/asm-x86_64/flushtlb.h Wed Apr 14 16:15:47 2004 +0000 6.3 @@ -13,23 +13,35 @@ 6.4 #include <xen/smp.h> 6.5 6.6 /* 6.7 - * Every GLOBAL_FLUSH_PERIOD ticks of the tlbflush clock, every TLB in the 6.8 - * system is guaranteed to have been flushed. 6.9 + * Every time the TLB clock passes an "epoch", every CPU's TLB is flushed. 6.10 + * Therefore, if the current TLB time and a previously-read timestamp differ 6.11 + * in their significant bits (i.e., ~TLBCLOCK_EPOCH_MASK), then the TLB clock 6.12 + * has wrapped at least once and every CPU's TLB is guaranteed to have been 6.13 + * flushed meanwhile. 6.14 + * This allows us to deal gracefully with a bounded (a.k.a. wrapping) clock. 6.15 */ 6.16 -#define GLOBAL_FLUSH_PERIOD (1<<16) 6.17 +#define TLBCLOCK_EPOCH_MASK ((1U<<16)-1) 6.18 6.19 /* 6.20 - * '_cpu_stamp' is the current timestamp for the CPU we are testing. 6.21 - * '_lastuse_stamp' is a timestamp taken when the PFN we are testing was last 6.22 + * 'cpu_stamp' is the current timestamp for the CPU we are testing. 6.23 + * 'lastuse_stamp' is a timestamp taken when the PFN we are testing was last 6.24 * used for a purpose that may have caused the CPU's TLB to become tainted. 6.25 */ 6.26 -#define NEED_FLUSH(_cpu_stamp, _lastuse_stamp) \ 6.27 - (((_cpu_stamp) <= (_lastuse_stamp)) && \ 6.28 - (((_lastuse_stamp) - (_cpu_stamp)) <= (2*GLOBAL_FLUSH_PERIOD))) 6.29 +static inline int NEED_FLUSH(u32 cpu_stamp, u32 lastuse_stamp) 6.30 +{ 6.31 + /* 6.32 + * Why does this work? 6.33 + * 1. XOR sets high-order bits determines if stamps from differing epochs. 6.34 + * 2. Subtraction sets high-order bits if 'cpu_stamp > lastuse_stamp'. 6.35 + * In either case a flush is unnecessary: we therefore OR the results from 6.36 + * (1) and (2), mask the high-order bits, and return the inverse. 6.37 + */ 6.38 + return !(((lastuse_stamp^cpu_stamp)|(lastuse_stamp-cpu_stamp)) & 6.39 + ~TLBCLOCK_EPOCH_MASK); 6.40 +} 6.41 6.42 -extern unsigned long tlbflush_mask; 6.43 -extern unsigned long tlbflush_clock; 6.44 -extern unsigned long tlbflush_time[NR_CPUS]; 6.45 +extern u32 tlbflush_clock; 6.46 +extern u32 tlbflush_time[NR_CPUS]; 6.47 6.48 extern void new_tlbflush_clock_period(void); 6.49
7.1 --- a/xen/include/xen/mm.h Tue Apr 13 16:30:13 2004 +0000 7.2 +++ b/xen/include/xen/mm.h Wed Apr 14 16:15:47 2004 +0000 7.3 @@ -63,11 +63,11 @@ struct pfn_info 7.4 unsigned long _unused; 7.5 } u; 7.6 /* Reference count and various PGC_xxx flags and fields. */ 7.7 - unsigned long count_and_flags; 7.8 + u32 count_and_flags; 7.9 /* Type reference count and various PGT_xxx flags and fields. */ 7.10 - unsigned long type_and_flags; 7.11 + u32 type_and_flags; 7.12 /* Timestamp from 'TLB clock', used to reduce need for safety flushes. */ 7.13 - unsigned long tlbflush_timestamp; 7.14 + u32 tlbflush_timestamp; 7.15 }; 7.16 7.17 /* The following page types are MUTUALLY EXCLUSIVE. */ 7.18 @@ -136,7 +136,7 @@ void free_page_type(struct pfn_info *pag 7.19 7.20 static inline void put_page(struct pfn_info *page) 7.21 { 7.22 - unsigned long nx, x, y = page->count_and_flags; 7.23 + u32 nx, x, y = page->count_and_flags; 7.24 7.25 do { 7.26 x = y; 7.27 @@ -152,7 +152,7 @@ static inline void put_page(struct pfn_i 7.28 static inline int get_page(struct pfn_info *page, 7.29 struct task_struct *domain) 7.30 { 7.31 - unsigned long x, nx, y = page->count_and_flags; 7.32 + u32 x, nx, y = page->count_and_flags; 7.33 struct task_struct *p, *np = page->u.domain; 7.34 7.35 do { 7.36 @@ -164,7 +164,7 @@ static inline int get_page(struct pfn_in 7.37 unlikely(x & PGC_zombie) || /* Zombie? */ 7.38 unlikely(p != domain) ) /* Wrong owner? */ 7.39 { 7.40 - DPRINTK("Error pfn %08lx: ed=%p,sd=%p,caf=%08lx\n", 7.41 + DPRINTK("Error pfn %08lx: ed=%p,sd=%p,caf=%08x\n", 7.42 page_to_pfn(page), domain, p, x); 7.43 return 0; 7.44 } 7.45 @@ -182,7 +182,7 @@ static inline int get_page(struct pfn_in 7.46 7.47 static inline void put_page_type(struct pfn_info *page) 7.48 { 7.49 - unsigned long nx, x, y = page->type_and_flags; 7.50 + u32 nx, x, y = page->type_and_flags; 7.51 7.52 again: 7.53 do { 7.54 @@ -214,9 +214,9 @@ static inline void put_page_type(struct 7.55 } 7.56 7.57 7.58 -static inline int get_page_type(struct pfn_info *page, unsigned long type) 7.59 +static inline int get_page_type(struct pfn_info *page, u32 type) 7.60 { 7.61 - unsigned long nx, x, y = page->type_and_flags; 7.62 + u32 nx, x, y = page->type_and_flags; 7.63 again: 7.64 do { 7.65 x = y; 7.66 @@ -239,7 +239,7 @@ static inline int get_page_type(struct p 7.67 } 7.68 else if ( unlikely((x & PGT_type_mask) != type) ) 7.69 { 7.70 - DPRINTK("Unexpected type (saw %08lx != exp %08lx) for pfn %08lx\n", 7.71 + DPRINTK("Unexpected type (saw %08x != exp %08x) for pfn %08lx\n", 7.72 x & PGT_type_mask, type, page_to_pfn(page)); 7.73 return 0; 7.74 } 7.75 @@ -261,7 +261,7 @@ static inline int get_page_type(struct p 7.76 /* Try to validate page type; drop the new reference on failure. */ 7.77 if ( unlikely(!alloc_page_type(page, type)) ) 7.78 { 7.79 - DPRINTK("Error while validating pfn %08lx for type %08lx\n", 7.80 + DPRINTK("Error while validating pfn %08lx for type %08x\n", 7.81 page_to_pfn(page), type); 7.82 put_page_type(page); 7.83 return 0; 7.84 @@ -282,7 +282,7 @@ static inline void put_page_and_type(str 7.85 7.86 static inline int get_page_and_type(struct pfn_info *page, 7.87 struct task_struct *domain, 7.88 - unsigned int type) 7.89 + u32 type) 7.90 { 7.91 int rc = get_page(page, domain); 7.92
8.1 --- a/xen/net/dev.c Tue Apr 13 16:30:13 2004 +0000 8.2 +++ b/xen/net/dev.c Wed Apr 14 16:15:47 2004 +0000 8.3 @@ -553,7 +553,7 @@ void deliver_packet(struct sk_buff *skb, 8.4 unsigned long *sptr = map_domain_mem( (spte_pfn<<PAGE_SHIFT) | 8.5 (((unsigned long)ptep)&~PAGE_MASK) ); 8.6 8.7 - // avoid the fault later 8.8 + /* Avoid the fault later. */ 8.9 *sptr = new_pte; 8.10 8.11 unmap_domain_mem(sptr); 8.12 @@ -2086,13 +2086,11 @@ static void get_rx_bufs(net_vif_t *vif) 8.13 pte_pfn = rx.addr >> PAGE_SHIFT; 8.14 pte_page = &frame_table[pte_pfn]; 8.15 8.16 - //printk("MMM %08lx ", rx.addr); 8.17 - 8.18 /* The address passed down must be to a valid PTE. */ 8.19 if ( unlikely(pte_pfn >= max_page) || 8.20 unlikely(!get_page_and_type(pte_page, p, PGT_l1_page_table)) ) 8.21 { 8.22 - DPRINTK("Bad page frame for ppte %llu,%08lx,%08lx,%08lx\n", 8.23 + DPRINTK("Bad page frame for ppte %llu,%08lx,%08lx,%08x\n", 8.24 p->domain, pte_pfn, max_page, pte_page->type_and_flags); 8.25 make_rx_response(vif, rx.id, 0, RING_STATUS_BAD_PAGE, 0); 8.26 continue; 8.27 @@ -2100,7 +2098,7 @@ static void get_rx_bufs(net_vif_t *vif) 8.28 8.29 ptep = map_domain_mem(rx.addr); 8.30 pte = *ptep; 8.31 - //printk("%08lx\n",pte); 8.32 + 8.33 /* We must be passed a valid writeable mapping to swizzle. */ 8.34 if ( unlikely((pte & (_PAGE_PRESENT|_PAGE_RW)) != 8.35 (_PAGE_PRESENT|_PAGE_RW)) || 8.36 @@ -2143,7 +2141,7 @@ static void get_rx_bufs(net_vif_t *vif) 8.37 make_rx_response(vif, rx.id, 0, RING_STATUS_BAD_PAGE, 0); 8.38 goto rx_unmap_and_continue; 8.39 8.40 - // XXX IAP should SHADOW_CONFIG do something here? 8.41 + /* XXX IAP should SHADOW_CONFIG do something here? */ 8.42 } 8.43 8.44 /* 8.45 @@ -2155,7 +2153,7 @@ static void get_rx_bufs(net_vif_t *vif) 8.46 0) != 8.47 (PGC_allocated | PGC_tlb_flush_on_type_change | 2)) ) 8.48 { 8.49 - DPRINTK("Page held more than once %08lx\n", 8.50 + DPRINTK("Page held more than once %08x\n", 8.51 buf_page->count_and_flags); 8.52 if ( !get_page_type(buf_page, PGT_writeable_page) ) 8.53 put_page(buf_page);