X-Git-Url: http://pilppa.org/gitweb/?a=blobdiff_plain;f=include%2Fasm-arm%2Ftlbflush.h;h=ccd0de010e81bb515d3aec39acb8abcf04ffadbd;hb=a43266355b3d8cf6717a27159f9a417bdff73782;hp=d97fc76189a5e5c95570424a19e0e0f5790a8fea;hpb=45c091bb2d453ce4a8b06cf19872ec7a77fc4799;p=linux-2.6-omap-h63xx.git diff --git a/include/asm-arm/tlbflush.h b/include/asm-arm/tlbflush.h index d97fc76189a..ccd0de010e8 100644 --- a/include/asm-arm/tlbflush.h +++ b/include/asm-arm/tlbflush.h @@ -144,6 +144,8 @@ #ifndef __ASSEMBLY__ +#include + struct cpu_tlb_fns { void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *); void (*flush_kern_range)(unsigned long, unsigned long); @@ -247,16 +249,25 @@ static inline void local_flush_tlb_all(void) const unsigned int __tlb_flag = __cpu_tlb_flags; if (tlb_flag(TLB_WB)) - asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero)); + dsb(); if (tlb_flag(TLB_V3_FULL)) - asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_U_FULL | TLB_V6_U_FULL)) - asm("mcr%? p15, 0, %0, c8, c7, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_D_FULL | TLB_V6_D_FULL)) - asm("mcr%? p15, 0, %0, c8, c6, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL)) - asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); + + if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | + TLB_V6_I_PAGE | TLB_V6_D_PAGE | + TLB_V6_I_ASID | TLB_V6_D_ASID)) { + /* flush the branch target cache */ + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); + dsb(); + isb(); + } } static inline void local_flush_tlb_mm(struct mm_struct *mm) @@ -266,25 +277,33 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) const unsigned int __tlb_flag = __cpu_tlb_flags; if (tlb_flag(TLB_WB)) - asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero)); + dsb(); if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) { if (tlb_flag(TLB_V3_FULL)) - asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_U_FULL)) - asm("mcr%? p15, 0, %0, c8, c7, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_D_FULL)) - asm("mcr%? p15, 0, %0, c8, c6, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_I_FULL)) - asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); } if (tlb_flag(TLB_V6_U_ASID)) - asm("mcr%? p15, 0, %0, c8, c7, 2" : : "r" (asid)); + asm("mcr p15, 0, %0, c8, c7, 2" : : "r" (asid) : "cc"); if (tlb_flag(TLB_V6_D_ASID)) - asm("mcr%? p15, 0, %0, c8, c6, 2" : : "r" (asid)); + asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc"); if (tlb_flag(TLB_V6_I_ASID)) - asm("mcr%? p15, 0, %0, c8, c5, 2" : : "r" (asid)); + asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc"); + + if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | + TLB_V6_I_PAGE | TLB_V6_D_PAGE | + TLB_V6_I_ASID | TLB_V6_D_ASID)) { + /* flush the branch target cache */ + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); + dsb(); + } } static inline void @@ -296,27 +315,35 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); if (tlb_flag(TLB_WB)) - asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero)); + dsb(); if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { if (tlb_flag(TLB_V3_PAGE)) - asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (uaddr)); + asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_V4_U_PAGE)) - asm("mcr%? p15, 0, %0, c8, c7, 1" : : "r" (uaddr)); + asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_V4_D_PAGE)) - asm("mcr%? p15, 0, %0, c8, c6, 1" : : "r" (uaddr)); + asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_V4_I_PAGE)) - asm("mcr%? p15, 0, %0, c8, c5, 1" : : "r" (uaddr)); + asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) - asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); } if (tlb_flag(TLB_V6_U_PAGE)) - asm("mcr%? p15, 0, %0, c8, c7, 1" : : "r" (uaddr)); + asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_V6_D_PAGE)) - asm("mcr%? p15, 0, %0, c8, c6, 1" : : "r" (uaddr)); + asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_V6_I_PAGE)) - asm("mcr%? p15, 0, %0, c8, c5, 1" : : "r" (uaddr)); + asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); + + if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | + TLB_V6_I_PAGE | TLB_V6_D_PAGE | + TLB_V6_I_ASID | TLB_V6_D_ASID)) { + /* flush the branch target cache */ + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); + dsb(); + } } static inline void local_flush_tlb_kernel_page(unsigned long kaddr) @@ -327,31 +354,34 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) kaddr &= PAGE_MASK; if (tlb_flag(TLB_WB)) - asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero)); + dsb(); if (tlb_flag(TLB_V3_PAGE)) - asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (kaddr)); + asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (kaddr) : "cc"); if (tlb_flag(TLB_V4_U_PAGE)) - asm("mcr%? p15, 0, %0, c8, c7, 1" : : "r" (kaddr)); + asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); if (tlb_flag(TLB_V4_D_PAGE)) - asm("mcr%? p15, 0, %0, c8, c6, 1" : : "r" (kaddr)); + asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); if (tlb_flag(TLB_V4_I_PAGE)) - asm("mcr%? p15, 0, %0, c8, c5, 1" : : "r" (kaddr)); + asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) - asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V6_U_PAGE)) - asm("mcr%? p15, 0, %0, c8, c7, 1" : : "r" (kaddr)); + asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); if (tlb_flag(TLB_V6_D_PAGE)) - asm("mcr%? p15, 0, %0, c8, c6, 1" : : "r" (kaddr)); + asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); if (tlb_flag(TLB_V6_I_PAGE)) - asm("mcr%? p15, 0, %0, c8, c5, 1" : : "r" (kaddr)); - - /* The ARM ARM states that the completion of a TLB maintenance - * operation is only guaranteed by a DSB instruction - */ - if (tlb_flag(TLB_V6_U_PAGE | TLB_V6_D_PAGE | TLB_V6_I_PAGE)) - asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero)); + asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); + + if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | + TLB_V6_I_PAGE | TLB_V6_D_PAGE | + TLB_V6_I_ASID | TLB_V6_D_ASID)) { + /* flush the branch target cache */ + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); + dsb(); + isb(); + } } /* @@ -369,15 +399,13 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) */ static inline void flush_pmd_entry(pmd_t *pmd) { - const unsigned int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; if (tlb_flag(TLB_DCLEAN)) - asm("mcr%? p15, 0, %0, c7, c10, 1 @ flush_pmd" - : : "r" (pmd)); + asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" + : : "r" (pmd) : "cc"); if (tlb_flag(TLB_WB)) - asm("mcr%? p15, 0, %0, c7, c10, 4 @ flush_pmd" - : : "r" (zero)); + dsb(); } static inline void clean_pmd_entry(pmd_t *pmd) @@ -385,8 +413,8 @@ static inline void clean_pmd_entry(pmd_t *pmd) const unsigned int __tlb_flag = __cpu_tlb_flags; if (tlb_flag(TLB_DCLEAN)) - asm("mcr%? p15, 0, %0, c7, c10, 1 @ flush_pmd" - : : "r" (pmd)); + asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" + : : "r" (pmd) : "cc"); } #undef tlb_flag