diff options
Diffstat (limited to 'arch/arm/include/asm/tlbflush.h')
| -rw-r--r-- | arch/arm/include/asm/tlbflush.h | 452 |
1 files changed, 273 insertions, 179 deletions
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index ce7378ea15a..def9e570199 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -10,16 +10,10 @@ #ifndef _ASMARM_TLBFLUSH_H #define _ASMARM_TLBFLUSH_H - -#ifndef CONFIG_MMU - -#define tlb_flush(tlb) ((void) tlb) - -#else /* CONFIG_MMU */ +#ifdef CONFIG_MMU #include <asm/glue.h> -#define TLB_V3_PAGE (1 << 0) #define TLB_V4_U_PAGE (1 << 1) #define TLB_V4_D_PAGE (1 << 2) #define TLB_V4_I_PAGE (1 << 3) @@ -27,7 +21,6 @@ #define TLB_V6_D_PAGE (1 << 5) #define TLB_V6_I_PAGE (1 << 6) -#define TLB_V3_FULL (1 << 8) #define TLB_V4_U_FULL (1 << 9) #define TLB_V4_D_FULL (1 << 10) #define TLB_V4_I_FULL (1 << 11) @@ -39,16 +32,15 @@ #define TLB_V6_D_ASID (1 << 17) #define TLB_V6_I_ASID (1 << 18) -#define TLB_BTB (1 << 28) +#define TLB_V6_BP (1 << 19) /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ -#define TLB_V7_UIS_PAGE (1 << 19) -#define TLB_V7_UIS_FULL (1 << 20) -#define TLB_V7_UIS_ASID (1 << 21) - -/* Inner Shareable BTB operation (ARMv7 MP extensions) */ -#define TLB_V7_IS_BTB (1 << 22) +#define TLB_V7_UIS_PAGE (1 << 20) +#define TLB_V7_UIS_FULL (1 << 21) +#define TLB_V7_UIS_ASID (1 << 22) +#define TLB_V7_UIS_BP (1 << 23) +#define TLB_BARRIER (1 << 28) #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ #define TLB_DCLEAN (1 << 30) #define TLB_WB (1 << 31) @@ -58,12 +50,11 @@ * ============= * * We have the following to choose from: - * v3 - ARMv3 * v4 - ARMv4 without write buffer * v4wb - ARMv4 with write buffer without I TLB flush entry instruction * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) - * fa - Faraday (v4 with write buffer with UTLB and branch target buffer (BTB)) + * fa - Faraday (v4 with write buffer with UTLB) * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction * v7wbi - identical to v6wbi */ @@ -74,21 +65,6 @@ #define MULTI_TLB 1 #endif -#define v3_tlb_flags (TLB_V3_FULL | TLB_V3_PAGE) - -#ifdef CONFIG_CPU_TLB_V3 -# define v3_possible_flags v3_tlb_flags -# define v3_always_flags v3_tlb_flags -# ifdef _TLB -# define MULTI_TLB 1 -# else -# define _TLB v3 -# endif -#else -# define v3_possible_flags 0 -# define v3_always_flags (-1UL) -#endif - #define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE) #ifdef CONFIG_CPU_TLB_V4WT @@ -104,7 +80,7 @@ # define v4_always_flags (-1UL) #endif -#define fa_tlb_flags (TLB_WB | TLB_BTB | TLB_DCLEAN | \ +#define fa_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ TLB_V4_U_FULL | TLB_V4_U_PAGE) #ifdef CONFIG_CPU_TLB_FA @@ -171,10 +147,11 @@ # define v4wb_always_flags (-1UL) #endif -#define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \ +#define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ TLB_V6_I_FULL | TLB_V6_D_FULL | \ TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ - TLB_V6_I_ASID | TLB_V6_D_ASID) + TLB_V6_I_ASID | TLB_V6_D_ASID | \ + TLB_V6_BP) #ifdef CONFIG_CPU_TLB_V6 # define v6wbi_possible_flags v6wbi_tlb_flags @@ -189,10 +166,12 @@ # define v6wbi_always_flags (-1UL) #endif -#define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_V7_IS_BTB | \ - TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) -#define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BTB | \ - TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) +#define v7wbi_tlb_flags_smp (TLB_WB | TLB_BARRIER | \ + TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \ + TLB_V7_UIS_ASID | TLB_V7_UIS_BP) +#define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ + TLB_V6_U_FULL | TLB_V6_U_PAGE | \ + TLB_V6_U_ASID | TLB_V6_BP) #ifdef CONFIG_CPU_TLB_V7 @@ -307,8 +286,7 @@ extern struct cpu_tlb_fns cpu_tlb; * implemented the "%?" method, but this has been discontinued due to too * many people getting it wrong. */ -#define possible_tlb_flags (v3_possible_flags | \ - v4_possible_flags | \ +#define possible_tlb_flags (v4_possible_flags | \ v4wbi_possible_flags | \ fr_possible_flags | \ v4wb_possible_flags | \ @@ -316,8 +294,7 @@ extern struct cpu_tlb_fns cpu_tlb; v6wbi_possible_flags | \ v7wbi_possible_flags) -#define always_tlb_flags (v3_always_flags & \ - v4_always_flags & \ +#define always_tlb_flags (v4_always_flags & \ v4wbi_always_flags & \ fr_always_flags & \ v4wb_always_flags & \ @@ -327,181 +304,263 @@ extern struct cpu_tlb_fns cpu_tlb; #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f))) +#define __tlb_op(f, insnarg, arg) \ + do { \ + if (always_tlb_flags & (f)) \ + asm("mcr " insnarg \ + : : "r" (arg) : "cc"); \ + else if (possible_tlb_flags & (f)) \ + asm("tst %1, %2\n\t" \ + "mcrne " insnarg \ + : : "r" (arg), "r" (__tlb_flag), "Ir" (f) \ + : "cc"); \ + } while (0) + +#define tlb_op(f, regs, arg) __tlb_op(f, "p15, 0, %0, " regs, arg) +#define tlb_l2_op(f, regs, arg) __tlb_op(f, "p15, 1, %0, " regs, arg) + +static inline void __local_flush_tlb_all(void) +{ + const int zero = 0; + const unsigned int __tlb_flag = __cpu_tlb_flags; + + tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero); + tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero); + tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero); +} + static inline void local_flush_tlb_all(void) { const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; if (tlb_flag(TLB_WB)) - dsb(); - - if (tlb_flag(TLB_V3_FULL)) - asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); - if (tlb_flag(TLB_V4_U_FULL | TLB_V6_U_FULL)) - asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); - if (tlb_flag(TLB_V4_D_FULL | TLB_V6_D_FULL)) - asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); - if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL)) - asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); - if (tlb_flag(TLB_V7_UIS_FULL)) - asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); + dsb(nshst); - if (tlb_flag(TLB_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); - dsb(); + __local_flush_tlb_all(); + tlb_op(TLB_V7_UIS_FULL, "c8, c7, 0", zero); + + if (tlb_flag(TLB_BARRIER)) { + dsb(nsh); isb(); } - if (tlb_flag(TLB_V7_IS_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); - dsb(); +} + +static inline void __flush_tlb_all(void) +{ + const int zero = 0; + const unsigned int __tlb_flag = __cpu_tlb_flags; + + if (tlb_flag(TLB_WB)) + dsb(ishst); + + __local_flush_tlb_all(); + tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero); + + if (tlb_flag(TLB_BARRIER)) { + dsb(ish); isb(); } } -static inline void local_flush_tlb_mm(struct mm_struct *mm) +static inline void __local_flush_tlb_mm(struct mm_struct *mm) { const int zero = 0; const int asid = ASID(mm); const unsigned int __tlb_flag = __cpu_tlb_flags; - if (tlb_flag(TLB_WB)) - dsb(); - - if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) { - if (tlb_flag(TLB_V3_FULL)) - asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); - if (tlb_flag(TLB_V4_U_FULL)) - asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); - if (tlb_flag(TLB_V4_D_FULL)) - asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); - if (tlb_flag(TLB_V4_I_FULL)) - asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); + if (possible_tlb_flags & (TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) { + if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) { + tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero); + tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero); + tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero); + } } - put_cpu(); - - if (tlb_flag(TLB_V6_U_ASID)) - asm("mcr p15, 0, %0, c8, c7, 2" : : "r" (asid) : "cc"); - if (tlb_flag(TLB_V6_D_ASID)) - asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc"); - if (tlb_flag(TLB_V6_I_ASID)) - asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc"); - if (tlb_flag(TLB_V7_UIS_ASID)) + + tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid); + tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid); + tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid); +} + +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ + const int asid = ASID(mm); + const unsigned int __tlb_flag = __cpu_tlb_flags; + + if (tlb_flag(TLB_WB)) + dsb(nshst); + + __local_flush_tlb_mm(mm); + tlb_op(TLB_V7_UIS_ASID, "c8, c7, 2", asid); + + if (tlb_flag(TLB_BARRIER)) + dsb(nsh); +} + +static inline void __flush_tlb_mm(struct mm_struct *mm) +{ + const unsigned int __tlb_flag = __cpu_tlb_flags; + + if (tlb_flag(TLB_WB)) + dsb(ishst); + + __local_flush_tlb_mm(mm); #ifdef CONFIG_ARM_ERRATA_720789 - asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); + tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", 0); #else - asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc"); + tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", ASID(mm)); #endif - if (tlb_flag(TLB_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); - dsb(); - } - if (tlb_flag(TLB_V7_IS_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); - dsb(); - isb(); - } + if (tlb_flag(TLB_BARRIER)) + dsb(ish); } static inline void -local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) +__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); - if (tlb_flag(TLB_WB)) - dsb(); - - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { - if (tlb_flag(TLB_V3_PAGE)) - asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr) : "cc"); - if (tlb_flag(TLB_V4_U_PAGE)) - asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); - if (tlb_flag(TLB_V4_D_PAGE)) - asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); - if (tlb_flag(TLB_V4_I_PAGE)) - asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); + if (possible_tlb_flags & (TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) && + cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { + tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr); + tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", uaddr); + tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", uaddr); if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); } - if (tlb_flag(TLB_V6_U_PAGE)) - asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); - if (tlb_flag(TLB_V6_D_PAGE)) - asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); - if (tlb_flag(TLB_V6_I_PAGE)) - asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); - if (tlb_flag(TLB_V7_UIS_PAGE)) + tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr); + tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr); + tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr); +} + +static inline void +local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) +{ + const unsigned int __tlb_flag = __cpu_tlb_flags; + + uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); + + if (tlb_flag(TLB_WB)) + dsb(nshst); + + __local_flush_tlb_page(vma, uaddr); + tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", uaddr); + + if (tlb_flag(TLB_BARRIER)) + dsb(nsh); +} + +static inline void +__flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) +{ + const unsigned int __tlb_flag = __cpu_tlb_flags; + + uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); + + if (tlb_flag(TLB_WB)) + dsb(ishst); + + __local_flush_tlb_page(vma, uaddr); #ifdef CONFIG_ARM_ERRATA_720789 - asm("mcr p15, 0, %0, c8, c3, 3" : : "r" (uaddr & PAGE_MASK) : "cc"); + tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK); #else - asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc"); + tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", uaddr); #endif - if (tlb_flag(TLB_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); - dsb(); - } - if (tlb_flag(TLB_V7_IS_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); - dsb(); - isb(); - } + if (tlb_flag(TLB_BARRIER)) + dsb(ish); } -static inline void local_flush_tlb_kernel_page(unsigned long kaddr) +static inline void __local_flush_tlb_kernel_page(unsigned long kaddr) { const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; + tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr); + tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr); + tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr); + if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) + asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); + + tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr); + tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr); + tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr); +} + +static inline void local_flush_tlb_kernel_page(unsigned long kaddr) +{ + const unsigned int __tlb_flag = __cpu_tlb_flags; + kaddr &= PAGE_MASK; if (tlb_flag(TLB_WB)) - dsb(); - - if (tlb_flag(TLB_V3_PAGE)) - asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (kaddr) : "cc"); - if (tlb_flag(TLB_V4_U_PAGE)) - asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); - if (tlb_flag(TLB_V4_D_PAGE)) - asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); - if (tlb_flag(TLB_V4_I_PAGE)) - asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); - if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) - asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); + dsb(nshst); + + __local_flush_tlb_kernel_page(kaddr); + tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", kaddr); - if (tlb_flag(TLB_V6_U_PAGE)) - asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); - if (tlb_flag(TLB_V6_D_PAGE)) - asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); - if (tlb_flag(TLB_V6_I_PAGE)) - asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); - if (tlb_flag(TLB_V7_UIS_PAGE)) - asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc"); - - if (tlb_flag(TLB_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); - dsb(); + if (tlb_flag(TLB_BARRIER)) { + dsb(nsh); isb(); } - if (tlb_flag(TLB_V7_IS_BTB)) { - /* flush the branch target cache */ - asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero) : "cc"); - dsb(); +} + +static inline void __flush_tlb_kernel_page(unsigned long kaddr) +{ + const unsigned int __tlb_flag = __cpu_tlb_flags; + + kaddr &= PAGE_MASK; + + if (tlb_flag(TLB_WB)) + dsb(ishst); + + __local_flush_tlb_kernel_page(kaddr); + tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr); + + if (tlb_flag(TLB_BARRIER)) { + dsb(ish); isb(); } } /* + * Branch predictor maintenance is paired with full TLB invalidation, so + * there is no need for any barriers here. + */ +static inline void __local_flush_bp_all(void) +{ + const int zero = 0; + const unsigned int __tlb_flag = __cpu_tlb_flags; + + if (tlb_flag(TLB_V6_BP)) + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero)); +} + +static inline void local_flush_bp_all(void) +{ + const int zero = 0; + const unsigned int __tlb_flag = __cpu_tlb_flags; + + __local_flush_bp_all(); + if (tlb_flag(TLB_V7_UIS_BP)) + asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero)); +} + +static inline void __flush_bp_all(void) +{ + const int zero = 0; + const unsigned int __tlb_flag = __cpu_tlb_flags; + + __local_flush_bp_all(); + if (tlb_flag(TLB_V7_UIS_BP)) + asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero)); +} + +/* * flush_pmd_entry * * Flush a PMD entry (word aligned, or double-word aligned) to @@ -514,35 +573,26 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) * these operations. This is typically used when we are removing * PMD entries. */ -static inline void flush_pmd_entry(pmd_t *pmd) +static inline void flush_pmd_entry(void *pmd) { const unsigned int __tlb_flag = __cpu_tlb_flags; - if (tlb_flag(TLB_DCLEAN)) - asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" - : : "r" (pmd) : "cc"); - - if (tlb_flag(TLB_L2CLEAN_FR)) - asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" - : : "r" (pmd) : "cc"); + tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd); + tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd); if (tlb_flag(TLB_WB)) - dsb(); + dsb(ishst); } -static inline void clean_pmd_entry(pmd_t *pmd) +static inline void clean_pmd_entry(void *pmd) { const unsigned int __tlb_flag = __cpu_tlb_flags; - if (tlb_flag(TLB_DCLEAN)) - asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" - : : "r" (pmd) : "cc"); - - if (tlb_flag(TLB_L2CLEAN_FR)) - asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" - : : "r" (pmd) : "cc"); + tlb_op(TLB_DCLEAN, "c7, c10, 1 @ flush_pmd", pmd); + tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd); } +#undef tlb_op #undef tlb_flag #undef always_tlb_flags #undef possible_tlb_flags @@ -560,6 +610,7 @@ static inline void clean_pmd_entry(pmd_t *pmd) #define flush_tlb_kernel_page local_flush_tlb_kernel_page #define flush_tlb_range local_flush_tlb_range #define flush_tlb_kernel_range local_flush_tlb_kernel_range +#define flush_bp_all local_flush_bp_all #else extern void flush_tlb_all(void); extern void flush_tlb_mm(struct mm_struct *mm); @@ -567,6 +618,7 @@ extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); extern void flush_tlb_kernel_page(unsigned long kaddr); extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); +extern void flush_bp_all(void); #endif /* @@ -585,8 +637,50 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, } #endif +#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) + +#endif + +#elif defined(CONFIG_SMP) /* !CONFIG_MMU */ + +#ifndef __ASSEMBLY__ + +#include <linux/mm_types.h> + +static inline void local_flush_tlb_all(void) { } +static inline void local_flush_tlb_mm(struct mm_struct *mm) { } +static inline void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { } +static inline void local_flush_tlb_kernel_page(unsigned long kaddr) { } +static inline void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { } +static inline void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) { } +static inline void local_flush_bp_all(void) { } + +extern void flush_tlb_all(void); +extern void flush_tlb_mm(struct mm_struct *mm); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); +extern void flush_tlb_kernel_page(unsigned long kaddr); +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); +extern void flush_bp_all(void); +#endif /* __ASSEMBLY__ */ + +#endif + +#ifndef __ASSEMBLY__ +#ifdef CONFIG_ARM_ERRATA_798181 +extern void erratum_a15_798181_init(void); +#else +static inline void erratum_a15_798181_init(void) {} #endif +extern bool (*erratum_a15_798181_handler)(void); -#endif /* CONFIG_MMU */ +static inline bool erratum_a15_798181(void) +{ + if (unlikely(IS_ENABLED(CONFIG_ARM_ERRATA_798181) && + erratum_a15_798181_handler)) + return erratum_a15_798181_handler(); + return false; +} +#endif #endif |