X-Git-Url: http://pilppa.org/gitweb/gitweb.cgi?a=blobdiff_plain;f=arch%2Fx86%2Fmm%2Fpageattr.c;h=8493c855582bf56a5a36d2e6e64266332c9d26c0;hb=cc842b82cc513ebc78bef6eeaacb5f6335851bcb;hp=72880993af89bb9b23656e68e4e61f326d6cb5ac;hpb=07cf89c05f2bbafa002401ac4e09ac31678513e4;p=linux-2.6-omap-h63xx.git diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 72880993af8..8493c855582 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -16,6 +16,9 @@ #include #include +/* + * The current flushing context - we pass it instead of 5 arguments: + */ struct cpa_data { unsigned long vaddr; pgprot_t mask_set; @@ -24,11 +27,6 @@ struct cpa_data { int flushtlb; }; -enum { - CPA_NO_SPLIT = 0, - CPA_SPLIT, -}; - static inline int within(unsigned long addr, unsigned long start, unsigned long end) { @@ -119,7 +117,7 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache) /* * Only flush present addresses: */ - if (pte && pte_present(*pte)) + if (pte && (pte_val(*pte) & _PAGE_PRESENT)) clflush_cache_range((void *) addr, PAGE_SIZE); } } @@ -169,8 +167,6 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long address) if (within(address, virt_to_highmap(_text), virt_to_highmap(_etext))) pgprot_val(forbidden) |= _PAGE_NX; - -#ifdef CONFIG_DEBUG_RODATA /* The .rodata section needs to be read-only */ if (within(address, (unsigned long)__start_rodata, (unsigned long)__end_rodata)) @@ -181,7 +177,6 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long address) if (within(address, virt_to_highmap(__start_rodata), virt_to_highmap(__end_rodata))) pgprot_val(forbidden) |= _PAGE_RW; -#endif prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden)); @@ -206,9 +201,15 @@ pte_t *lookup_address(unsigned long address, int *level) if (pgd_none(*pgd)) return NULL; + pud = pud_offset(pgd, address); if (pud_none(*pud)) return NULL; + + *level = PG_LEVEL_1G; + if (pud_large(*pud) || !pud_present(*pud)) + return (pte_t *)pud; + pmd = pmd_offset(pud, address); if (pmd_none(*pmd)) return NULL; @@ -218,9 +219,13 @@ pte_t *lookup_address(unsigned long address, int *level) return (pte_t *)pmd; *level = PG_LEVEL_4K; + return pte_offset_kernel(pmd, address); } +/* + * Set the new pmd in all the pgds we know about: + */ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) { /* change init_mm */ @@ -243,24 +248,14 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) #endif } -static int try_preserve_large_page(pte_t *kpte, unsigned long address, - struct cpa_data *cpa) +static int +try_preserve_large_page(pte_t *kpte, unsigned long address, + struct cpa_data *cpa) { unsigned long nextpage_addr, numpages, pmask, psize, flags; pte_t new_pte, old_pte, *tmp; pgprot_t old_prot, new_prot; - int level, res = CPA_SPLIT; - - /* - * An Athlon 64 X2 showed hard hangs if we tried to preserve - * largepages and changed the PSE entry from RW to RO. - * - * As AMD CPUs have a long series of erratas in this area, - * (and none of the known ones seem to explain this hang), - * disable this code until the hang can be debugged: - */ - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) - return res; + int level, do_split = 1; spin_lock_irqsave(&pgd_lock, flags); /* @@ -276,9 +271,14 @@ static int try_preserve_large_page(pte_t *kpte, unsigned long address, psize = PMD_PAGE_SIZE; pmask = PMD_PAGE_MASK; break; +#ifdef CONFIG_X86_64 case PG_LEVEL_1G: + psize = PMD_PAGE_SIZE; + pmask = PMD_PAGE_MASK; + break; +#endif default: - res = -EINVAL; + do_split = -EINVAL; goto out_unlock; } @@ -306,7 +306,7 @@ static int try_preserve_large_page(pte_t *kpte, unsigned long address, * above: */ if (pgprot_val(new_prot) == pgprot_val(old_prot)) { - res = CPA_NO_SPLIT; + do_split = 0; goto out_unlock; } @@ -326,22 +326,23 @@ static int try_preserve_large_page(pte_t *kpte, unsigned long address, new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot)); __set_pmd_pte(kpte, address, new_pte); cpa->flushtlb = 1; - res = CPA_NO_SPLIT; + do_split = 0; } out_unlock: spin_unlock_irqrestore(&pgd_lock, flags); - return res; + + return do_split; } static int split_large_page(pte_t *kpte, unsigned long address) { - pgprot_t ref_prot; + unsigned long flags, pfn, pfninc = 1; gfp_t gfp_flags = GFP_KERNEL; - unsigned long flags, addr, pfn; + unsigned int i, level; pte_t *pbase, *tmp; + pgprot_t ref_prot; struct page *base; - unsigned int i, level; #ifdef CONFIG_DEBUG_PAGEALLOC gfp_flags = GFP_ATOMIC | __GFP_NOWARN; @@ -356,24 +357,27 @@ static int split_large_page(pte_t *kpte, unsigned long address) * up for us already: */ tmp = lookup_address(address, &level); - if (tmp != kpte) { - WARN_ON_ONCE(1); + if (tmp != kpte) goto out_unlock; - } - address = __pa(address); - addr = address & PMD_PAGE_MASK; pbase = (pte_t *)page_address(base); #ifdef CONFIG_X86_32 paravirt_alloc_pt(&init_mm, page_to_pfn(base)); #endif ref_prot = pte_pgprot(pte_clrhuge(*kpte)); +#ifdef CONFIG_X86_64 + if (level == PG_LEVEL_1G) { + pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT; + pgprot_val(ref_prot) |= _PAGE_PSE; + } +#endif + /* * Get the target pfn from the original entry: */ pfn = pte_pfn(*kpte); - for (i = 0; i < PTRS_PER_PTE; i++, pfn++) + for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc) set_pte(&pbase[i], pfn_pte(pfn, ref_prot)); /* @@ -402,8 +406,8 @@ out_unlock: static int __change_page_attr(unsigned long address, struct cpa_data *cpa) { + int level, do_split, err; struct page *kpte_page; - int level, res; pte_t *kpte; repeat: @@ -454,26 +458,25 @@ repeat: * Check, whether we can keep the large page intact * and just change the pte: */ - res = try_preserve_large_page(kpte, address, cpa); - if (res < 0) - return res; - + do_split = try_preserve_large_page(kpte, address, cpa); /* * When the range fits into the existing large page, * return. cp->numpages and cpa->tlbflush have been updated in * try_large_page: */ - if (res == CPA_NO_SPLIT) - return 0; + if (do_split <= 0) + return do_split; /* * We have to split the large page: */ - res = split_large_page(kpte, address); - if (res) - return res; - cpa->flushtlb = 1; - goto repeat; + err = split_large_page(kpte, address); + if (!err) { + cpa->flushtlb = 1; + goto repeat; + } + + return err; } /** @@ -489,7 +492,6 @@ repeat: * * Modules and drivers should use the set_memory_* APIs instead. */ - static int change_page_attr_addr(struct cpa_data *cpa) { int err;