ia64/xen-unstable

changeset 7907:8b384e741382

Improve make_page_readonly and make_page_writable:
1. Use update_va_mapping where possible (much faster)
2. If that fails, fall back to explicit pte update rather
than using writable pagetables (very slow for singleton
updates)

Also many x86/64 cleanups and fix to handle vmalloc space
correctly.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Fri Nov 18 17:42:45 2005 +0100 (2005-11-18)
parents 69ea493e415a
children 0970a2fdbee9
files linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c
line diff
     1.1 --- a/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c	Fri Nov 18 17:41:03 2005 +0100
     1.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c	Fri Nov 18 17:42:45 2005 +0100
     1.3 @@ -386,19 +386,26 @@ asmlinkage int xprintk(const char *fmt, 
     1.4  void make_lowmem_page_readonly(void *va)
     1.5  {
     1.6  	pte_t *pte = virt_to_ptep(va);
     1.7 -	set_pte(pte, pte_wrprotect(*pte));
     1.8 +	int rc = HYPERVISOR_update_va_mapping(
     1.9 +		(unsigned long)va, pte_wrprotect(*pte), 0);
    1.10 +	BUG_ON(rc);
    1.11  }
    1.12  
    1.13  void make_lowmem_page_writable(void *va)
    1.14  {
    1.15  	pte_t *pte = virt_to_ptep(va);
    1.16 -	set_pte(pte, pte_mkwrite(*pte));
    1.17 +	int rc = HYPERVISOR_update_va_mapping(
    1.18 +		(unsigned long)va, pte_mkwrite(*pte), 0);
    1.19 +	BUG_ON(rc);
    1.20  }
    1.21  
    1.22  void make_page_readonly(void *va)
    1.23  {
    1.24  	pte_t *pte = virt_to_ptep(va);
    1.25 -	set_pte(pte, pte_wrprotect(*pte));
    1.26 +	int rc = HYPERVISOR_update_va_mapping(
    1.27 +		(unsigned long)va, pte_wrprotect(*pte), 0);
    1.28 +	if (rc) /* fallback? */
    1.29 +		xen_l1_entry_update(pte, pte_wrprotect(*pte));
    1.30  	if ((unsigned long)va >= (unsigned long)high_memory) {
    1.31  		unsigned long pfn = pte_pfn(*pte);
    1.32  #ifdef CONFIG_HIGHMEM
    1.33 @@ -412,7 +419,10 @@ void make_page_readonly(void *va)
    1.34  void make_page_writable(void *va)
    1.35  {
    1.36  	pte_t *pte = virt_to_ptep(va);
    1.37 -	set_pte(pte, pte_mkwrite(*pte));
    1.38 +	int rc = HYPERVISOR_update_va_mapping(
    1.39 +		(unsigned long)va, pte_mkwrite(*pte), 0);
    1.40 +	if (rc) /* fallback? */
    1.41 +		xen_l1_entry_update(pte, pte_mkwrite(*pte));
    1.42  	if ((unsigned long)va >= (unsigned long)high_memory) {
    1.43  		unsigned long pfn = pte_pfn(*pte); 
    1.44  #ifdef CONFIG_HIGHMEM
     2.1 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c	Fri Nov 18 17:41:03 2005 +0100
     2.2 +++ b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c	Fri Nov 18 17:42:45 2005 +0100
     2.3 @@ -55,11 +55,9 @@ extern char _stext[];
     2.4  DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
     2.5  extern unsigned long start_pfn;
     2.6  
     2.7 -static int init_mapping_done;
     2.8 -
     2.9  /*
    2.10   * Use this until direct mapping is established, i.e. before __va() is 
    2.11 - * avaialble in init_memory_mapping().
    2.12 + * available in init_memory_mapping().
    2.13   */
    2.14  
    2.15  #define addr_to_page(addr, page)				\
    2.16 @@ -68,91 +66,65 @@ static int init_mapping_done;
    2.17  	(((mfn_to_pfn((addr) >> PAGE_SHIFT)) << PAGE_SHIFT) +	\
    2.18  	__START_KERNEL_map)))
    2.19  
    2.20 -static void __make_page_readonly(unsigned long va)
    2.21 +static void early_make_page_readonly(void *va)
    2.22  {
    2.23 -	unsigned long addr;
    2.24 -	pte_t pte, *ptep;
    2.25 -	unsigned long *page = (unsigned long *) init_level4_pgt;
    2.26 -
    2.27 -	addr = (unsigned long) page[pgd_index(va)];
    2.28 -	addr_to_page(addr, page);
    2.29 -
    2.30 -	addr = page[pud_index(va)];
    2.31 -	addr_to_page(addr, page);
    2.32 -
    2.33 -	addr = page[pmd_index(va)];
    2.34 -	addr_to_page(addr, page);
    2.35 -
    2.36 -	ptep = (pte_t *) &page[pte_index(va)];
    2.37 -	pte.pte = (ptep->pte & ~_PAGE_RW);
    2.38 -	xen_l1_entry_update(ptep, pte);
    2.39 -	__flush_tlb_one(addr);
    2.40 -}
    2.41 -
    2.42 -static void __make_page_writable(unsigned long va)
    2.43 -{
    2.44 -	unsigned long addr;
    2.45 +	unsigned long addr, _va = (unsigned long)va;
    2.46  	pte_t pte, *ptep;
    2.47  	unsigned long *page = (unsigned long *) init_level4_pgt;
    2.48  
    2.49 -	addr = (unsigned long) page[pgd_index(va)];
    2.50 +	addr = (unsigned long) page[pgd_index(_va)];
    2.51  	addr_to_page(addr, page);
    2.52  
    2.53 -	addr = page[pud_index(va)];
    2.54 -	addr_to_page(addr, page);
    2.55 - 
    2.56 -	addr = page[pmd_index(va)];
    2.57 +	addr = page[pud_index(_va)];
    2.58  	addr_to_page(addr, page);
    2.59  
    2.60 -	ptep = (pte_t *) &page[pte_index(va)];
    2.61 -	pte.pte = (ptep->pte | _PAGE_RW);
    2.62 -	xen_l1_entry_update(ptep, pte);
    2.63 -	__flush_tlb_one(addr);
    2.64 +	addr = page[pmd_index(_va)];
    2.65 +	addr_to_page(addr, page);
    2.66 +
    2.67 +	ptep = (pte_t *) &page[pte_index(_va)];
    2.68 +
    2.69 +	pte.pte = ptep->pte & ~_PAGE_RW;
    2.70 +	if (HYPERVISOR_update_va_mapping(_va, pte, 0))
    2.71 +		BUG();
    2.72  }
    2.73  
    2.74 -
    2.75 -/*
    2.76 - * Assume the translation is already established.
    2.77 - */
    2.78  void make_page_readonly(void *va)
    2.79  {
    2.80 -	pgd_t* pgd; pud_t *pud; pmd_t* pmd; pte_t pte, *ptep;
    2.81 +	pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t pte, *ptep;
    2.82  	unsigned long addr = (unsigned long) va;
    2.83  
    2.84 -	if (!init_mapping_done) {
    2.85 -		__make_page_readonly(addr);
    2.86 -		return;
    2.87 -	}
    2.88 -  
    2.89 -	pgd = pgd_offset_k(addr);
    2.90 -	pud = pud_offset(pgd, addr);
    2.91 -	pmd = pmd_offset(pud, addr);
    2.92 -	ptep = pte_offset_kernel(pmd, addr);
    2.93 -	pte.pte = (ptep->pte & ~_PAGE_RW);
    2.94 -	xen_l1_entry_update(ptep, pte);
    2.95 -	__flush_tlb_one(addr);
    2.96 -}
    2.97 -
    2.98 -void make_page_writable(void *va)
    2.99 -{
   2.100 -	pgd_t* pgd; pud_t *pud; pmd_t* pmd; pte_t pte, *ptep;
   2.101 -	unsigned long addr = (unsigned long) va;
   2.102 -
   2.103 -	if (!init_mapping_done) {
   2.104 -		__make_page_writable(addr);
   2.105 -		return;
   2.106 -	}
   2.107 -
   2.108  	pgd = pgd_offset_k(addr);
   2.109  	pud = pud_offset(pgd, addr);
   2.110  	pmd = pmd_offset(pud, addr);
   2.111  	ptep = pte_offset_kernel(pmd, addr);
   2.112 -	pte.pte = (ptep->pte | _PAGE_RW);
   2.113 -	xen_l1_entry_update(ptep, pte);
   2.114 -	__flush_tlb_one(addr);
   2.115 +
   2.116 +	pte.pte = ptep->pte & ~_PAGE_RW;
   2.117 +	if (HYPERVISOR_update_va_mapping(addr, pte, 0))
   2.118 +		xen_l1_entry_update(ptep, pte); /* fallback */
   2.119 +
   2.120 +	if ((addr >= VMALLOC_START) && (addr < VMALLOC_END))
   2.121 +		make_page_readonly(__va(pte_pfn(pte) << PAGE_SHIFT));
   2.122  }
   2.123  
   2.124 -void make_pages_readonly(void* va, unsigned nr)
   2.125 +void make_page_writable(void *va)
   2.126 +{
   2.127 +	pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t pte, *ptep;
   2.128 +	unsigned long addr = (unsigned long) va;
   2.129 +
   2.130 +	pgd = pgd_offset_k(addr);
   2.131 +	pud = pud_offset(pgd, addr);
   2.132 +	pmd = pmd_offset(pud, addr);
   2.133 +	ptep = pte_offset_kernel(pmd, addr);
   2.134 +
   2.135 +	pte.pte = ptep->pte | _PAGE_RW;
   2.136 +	if (HYPERVISOR_update_va_mapping(addr, pte, 0))
   2.137 +		xen_l1_entry_update(ptep, pte); /* fallback */
   2.138 +
   2.139 +	if ((addr >= VMALLOC_START) && (addr < VMALLOC_END))
   2.140 +		make_page_writable(__va(pte_pfn(pte) << PAGE_SHIFT));
   2.141 +}
   2.142 +
   2.143 +void make_pages_readonly(void *va, unsigned nr)
   2.144  {
   2.145  	while (nr-- != 0) {
   2.146  		make_page_readonly(va);
   2.147 @@ -160,7 +132,7 @@ void make_pages_readonly(void* va, unsig
   2.148  	}
   2.149  }
   2.150  
   2.151 -void make_pages_writable(void* va, unsigned nr)
   2.152 +void make_pages_writable(void *va, unsigned nr)
   2.153  {
   2.154  	while (nr-- != 0) {
   2.155  		make_page_writable(va);
   2.156 @@ -457,7 +429,7 @@ static void __init phys_pud_init(pud_t *
   2.157  		} 
   2.158  
   2.159  		pmd = alloc_static_page(&pmd_phys);
   2.160 -                make_page_readonly(pmd);
   2.161 +                early_make_page_readonly(pmd);
   2.162                  xen_pmd_pin(pmd_phys);
   2.163  		set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE));
   2.164  
   2.165 @@ -487,7 +459,7 @@ static void __init phys_pud_init(pud_t *
   2.166                                  __set_pte(pte, __pte(paddr | _KERNPG_TABLE));
   2.167                          }
   2.168                          pte = pte_save;
   2.169 -                        make_page_readonly(pte);  
   2.170 +                        early_make_page_readonly(pte);  
   2.171                          xen_pte_pin(pte_phys);
   2.172  			set_pmd(pmd, __pmd(pte_phys | _KERNPG_TABLE));
   2.173  		}
   2.174 @@ -536,11 +508,11 @@ void __init xen_init_pt(void)
   2.175  		      _KERNPG_TABLE | _PAGE_USER);
   2.176          memcpy((void *)level2_kernel_pgt, page, PAGE_SIZE);
   2.177  
   2.178 -	make_page_readonly(init_level4_pgt);
   2.179 -	make_page_readonly(init_level4_user_pgt);
   2.180 -	make_page_readonly(level3_kernel_pgt);
   2.181 -	make_page_readonly(level3_user_pgt);
   2.182 -	make_page_readonly(level2_kernel_pgt);
   2.183 +	early_make_page_readonly(init_level4_pgt);
   2.184 +	early_make_page_readonly(init_level4_user_pgt);
   2.185 +	early_make_page_readonly(level3_kernel_pgt);
   2.186 +	early_make_page_readonly(level3_user_pgt);
   2.187 +	early_make_page_readonly(level2_kernel_pgt);
   2.188  
   2.189  	xen_pgd_pin(__pa_symbol(init_level4_pgt));
   2.190  	xen_pgd_pin(__pa_symbol(init_level4_user_pgt));
   2.191 @@ -578,7 +550,7 @@ void __init extend_init_mapping(void)
   2.192  		pmd = (pmd_t *)&page[pmd_index(va)];
   2.193  		if (pmd_none(*pmd)) {
   2.194  			pte_page = alloc_static_page(&phys);
   2.195 -			make_page_readonly(pte_page);
   2.196 +			early_make_page_readonly(pte_page);
   2.197  			xen_pte_pin(phys);
   2.198  			set_pmd(pmd, __pmd(phys | _KERNPG_TABLE | _PAGE_USER));
   2.199  		} else {
   2.200 @@ -625,7 +597,7 @@ void __init init_memory_mapping(unsigned
   2.201  	for (; start < end; start = next) {
   2.202  		unsigned long pud_phys; 
   2.203  		pud_t *pud = alloc_static_page(&pud_phys);
   2.204 -		make_page_readonly(pud);
   2.205 +		early_make_page_readonly(pud);
   2.206  		xen_pud_pin(pud_phys);
   2.207  		next = start + PGDIR_SIZE;
   2.208  		if (next > end) 
   2.209 @@ -640,7 +612,6 @@ void __init init_memory_mapping(unsigned
   2.210  	BUG_ON(start_pfn != (table_start + (tables_space >> PAGE_SHIFT)));
   2.211  
   2.212  	__flush_tlb_all();
   2.213 -	init_mapping_done = 1;
   2.214  }
   2.215  
   2.216  extern struct x8664_pda cpu_pda[NR_CPUS];