aboutsummaryrefslogtreecommitdiff
path: root/arch/arm/mm/flush.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm/flush.c')
-rw-r--r--arch/arm/mm/flush.c125
1 files changed, 91 insertions, 34 deletions
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 391ffae7509..43d54f5b26b 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -10,28 +10,25 @@
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
+#include <linux/highmem.h>
#include <asm/cacheflush.h>
#include <asm/cachetype.h>
#include <asm/highmem.h>
#include <asm/smp_plat.h>
-#include <asm/system.h>
#include <asm/tlbflush.h>
-#include <asm/smp_plat.h>
+#include <linux/hugetlb.h>
#include "mm.h"
#ifdef CONFIG_CPU_CACHE_VIPT
-#define ALIAS_FLUSH_START 0xffff4000
-
static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
{
- unsigned long to = ALIAS_FLUSH_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
+ unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
const int zero = 0;
- set_pte_ext(TOP_PTE(to), pfn_pte(pfn, PAGE_KERNEL), 0);
- flush_tlb_kernel_page(to);
+ set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
asm( "mcrr p15, 0, %1, %0, c14\n"
" mcr p15, 0, %2, c7, c10, 4"
@@ -42,13 +39,12 @@ static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
{
- unsigned long colour = CACHE_COLOUR(vaddr);
+ unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
unsigned long offset = vaddr & (PAGE_SIZE - 1);
unsigned long to;
- set_pte_ext(TOP_PTE(ALIAS_FLUSH_START) + colour, pfn_pte(pfn, PAGE_KERNEL), 0);
- to = ALIAS_FLUSH_START + (colour << PAGE_SHIFT) + offset;
- flush_tlb_kernel_page(to);
+ set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
+ to = va + offset;
flush_icache_range(to, to + len);
}
@@ -108,17 +104,20 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig
#define flush_icache_alias(pfn,vaddr,len) do { } while (0)
#endif
+#define FLAG_PA_IS_EXEC 1
+#define FLAG_PA_CORE_IN_MM 2
+
static void flush_ptrace_access_other(void *args)
{
__flush_icache_all();
}
-static
-void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
- unsigned long uaddr, void *kaddr, unsigned long len)
+static inline
+void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
+ unsigned long len, unsigned int flags)
{
if (cache_is_vivt()) {
- if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
+ if (flags & FLAG_PA_CORE_IN_MM) {
unsigned long addr = (unsigned long)kaddr;
__cpuc_coherent_kern_range(addr, addr + len);
}
@@ -132,7 +131,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
}
/* VIPT non-aliasing D-cache */
- if (vma->vm_flags & VM_EXEC) {
+ if (flags & FLAG_PA_IS_EXEC) {
unsigned long addr = (unsigned long)kaddr;
if (icache_is_vipt_aliasing())
flush_icache_alias(page_to_pfn(page), uaddr, len);
@@ -144,6 +143,26 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
}
}
+static
+void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
+ unsigned long uaddr, void *kaddr, unsigned long len)
+{
+ unsigned int flags = 0;
+ if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
+ flags |= FLAG_PA_CORE_IN_MM;
+ if (vma->vm_flags & VM_EXEC)
+ flags |= FLAG_PA_IS_EXEC;
+ __flush_ptrace_access(page, uaddr, kaddr, len, flags);
+}
+
+void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
+ void *kaddr, unsigned long len)
+{
+ unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;
+
+ __flush_ptrace_access(page, uaddr, kaddr, len, flags);
+}
+
/*
* Copy user data from/to a page which is mapped into a different
* processes address space. Really, we want to allow our "user
@@ -173,17 +192,24 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
* coherent with the kernels mapping.
*/
if (!PageHighMem(page)) {
- __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
+ size_t page_size = PAGE_SIZE << compound_order(page);
+ __cpuc_flush_dcache_area(page_address(page), page_size);
} else {
- void *addr = kmap_high_get(page);
- if (addr) {
- __cpuc_flush_dcache_area(addr, PAGE_SIZE);
- kunmap_high(page);
- } else if (cache_is_vipt()) {
- pte_t saved_pte;
- addr = kmap_high_l1_vipt(page, &saved_pte);
- __cpuc_flush_dcache_area(addr, PAGE_SIZE);
- kunmap_high_l1_vipt(page, saved_pte);
+ unsigned long i;
+ if (cache_is_vipt_nonaliasing()) {
+ for (i = 0; i < (1 << compound_order(page)); i++) {
+ void *addr = kmap_atomic(page + i);
+ __cpuc_flush_dcache_area(addr, PAGE_SIZE);
+ kunmap_atomic(addr);
+ }
+ } else {
+ for (i = 0; i < (1 << compound_order(page)); i++) {
+ void *addr = kmap_high_get(page + i);
+ if (addr) {
+ __cpuc_flush_dcache_area(addr, PAGE_SIZE);
+ kunmap_high(page + i);
+ }
+ }
}
}
@@ -201,7 +227,6 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p
{
struct mm_struct *mm = current->active_mm;
struct vm_area_struct *mpnt;
- struct prio_tree_iter iter;
pgoff_t pgoff;
/*
@@ -213,7 +238,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p
pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
flush_dcache_mmap_lock(mapping);
- vma_prio_tree_foreach(mpnt, &iter, &mapping->i_mmap, pgoff, pgoff) {
+ vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
unsigned long offset;
/*
@@ -236,8 +261,6 @@ void __sync_icache_dcache(pte_t pteval)
struct page *page;
struct address_space *mapping;
- if (!pte_present_user(pteval))
- return;
if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
/* only flush non-aliasing VIPT caches for exec mappings */
return;
@@ -253,8 +276,8 @@ void __sync_icache_dcache(pte_t pteval)
if (!test_and_set_bit(PG_dcache_clean, &page->flags))
__flush_dcache_page(mapping, page);
- /* pte_exec() already checked above for non-aliasing VIPT cache */
- if (cache_is_vipt_nonaliasing() || pte_exec(pteval))
+
+ if (pte_exec(pteval))
__flush_icache_all();
}
#endif
@@ -275,7 +298,8 @@ void __sync_icache_dcache(pte_t pteval)
* kernel cache lines for later. Otherwise, we assume we have
* aliasing mappings.
*
- * Note that we disable the lazy flush for SMP.
+ * Note that we disable the lazy flush for SMP configurations where
+ * the cache maintenance operations are not automatically broadcasted.
*/
void flush_dcache_page(struct page *page)
{
@@ -291,7 +315,7 @@ void flush_dcache_page(struct page *page)
mapping = page_mapping(page);
if (!cache_ops_need_broadcast() &&
- mapping && !mapping_mapped(mapping))
+ mapping && !page_mapped(page))
clear_bit(PG_dcache_clean, &page->flags);
else {
__flush_dcache_page(mapping, page);
@@ -305,6 +329,39 @@ void flush_dcache_page(struct page *page)
EXPORT_SYMBOL(flush_dcache_page);
/*
+ * Ensure cache coherency for the kernel mapping of this page. We can
+ * assume that the page is pinned via kmap.
+ *
+ * If the page only exists in the page cache and there are no user
+ * space mappings, this is a no-op since the page was already marked
+ * dirty at creation. Otherwise, we need to flush the dirty kernel
+ * cache lines directly.
+ */
+void flush_kernel_dcache_page(struct page *page)
+{
+ if (cache_is_vivt() || cache_is_vipt_aliasing()) {
+ struct address_space *mapping;
+
+ mapping = page_mapping(page);
+
+ if (!mapping || mapping_mapped(mapping)) {
+ void *addr;
+
+ addr = page_address(page);
+ /*
+ * kmap_atomic() doesn't set the page virtual
+ * address for highmem pages, and
+ * kunmap_atomic() takes care of cache
+ * flushing already.
+ */
+ if (!IS_ENABLED(CONFIG_HIGHMEM) || addr)
+ __cpuc_flush_dcache_area(addr, PAGE_SIZE);
+ }
+ }
+}
+EXPORT_SYMBOL(flush_kernel_dcache_page);
+
+/*
* Flush an anonymous page so that users of get_user_pages()
* can safely access the data. The expected sequence is:
*
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-05 07:48:48 +0000