diff options
Diffstat (limited to 'arch/x86/include/asm/tlbflush.h')
| -rw-r--r-- | arch/x86/include/asm/tlbflush.h | 57 |
1 files changed, 44 insertions, 13 deletions
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index 0fee48e279c..04905bfc508 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -20,10 +20,20 @@ static inline void __native_flush_tlb(void) native_write_cr3(native_read_cr3()); } +static inline void __native_flush_tlb_global_irq_disabled(void) +{ + unsigned long cr4; + + cr4 = native_read_cr4(); + /* clear PGE */ + native_write_cr4(cr4 & ~X86_CR4_PGE); + /* write old PGE again and flush TLBs */ + native_write_cr4(cr4); +} + static inline void __native_flush_tlb_global(void) { unsigned long flags; - unsigned long cr4; /* * Read-modify-write to CR4 - protect it from preemption and @@ -32,11 +42,7 @@ static inline void __native_flush_tlb_global(void) */ raw_local_irq_save(flags); - cr4 = native_read_cr4(); - /* clear PGE */ - native_write_cr4(cr4 & ~X86_CR4_PGE); - /* write old PGE again and flush TLBs */ - native_write_cr4(cr4); + __native_flush_tlb_global_irq_disabled(); raw_local_irq_restore(flags); } @@ -56,7 +62,8 @@ static inline void __flush_tlb_all(void) static inline void __flush_tlb_one(unsigned long addr) { - __flush_tlb_single(addr); + count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); + __flush_tlb_single(addr); } #define TLB_FLUSH_ALL -1UL @@ -78,14 +85,38 @@ static inline void __flush_tlb_one(unsigned long addr) #ifndef CONFIG_SMP -#define flush_tlb() __flush_tlb() -#define flush_tlb_all() __flush_tlb_all() -#define local_flush_tlb() __flush_tlb() +/* "_up" is for UniProcessor. + * + * This is a helper for other header functions. *Not* intended to be called + * directly. All global TLB flushes need to either call this, or to bump the + * vm statistics themselves. + */ +static inline void __flush_tlb_up(void) +{ + count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); + __flush_tlb(); +} + +static inline void flush_tlb_all(void) +{ + count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); + __flush_tlb_all(); +} + +static inline void flush_tlb(void) +{ + __flush_tlb_up(); +} + +static inline void local_flush_tlb(void) +{ + __flush_tlb_up(); +} static inline void flush_tlb_mm(struct mm_struct *mm) { if (mm == current->active_mm) - __flush_tlb(); + __flush_tlb_up(); } static inline void flush_tlb_page(struct vm_area_struct *vma, @@ -99,14 +130,14 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { if (vma->vm_mm == current->active_mm) - __flush_tlb(); + __flush_tlb_up(); } static inline void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned long vmflag) { if (mm == current->active_mm) - __flush_tlb(); + __flush_tlb_up(); } static inline void native_flush_tlb_others(const struct cpumask *cpumask, |