aboutsummaryrefslogtreecommitdiff
path: root/arch/xtensa/mm/tlb.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/xtensa/mm/tlb.c')
-rw-r--r--arch/xtensa/mm/tlb.c563
1 files changed, 148 insertions, 415 deletions
diff --git a/arch/xtensa/mm/tlb.c b/arch/xtensa/mm/tlb.c
index d3bd3bfc3b3..5ece856c572 100644
--- a/arch/xtensa/mm/tlb.c
+++ b/arch/xtensa/mm/tlb.c
@@ -1,5 +1,5 @@
/*
- * arch/xtensa/mm/mmu.c
+ * arch/xtensa/mm/tlb.c
*
* Logic that manipulates the Xtensa MMU. Derived from MIPS.
*
@@ -18,18 +18,17 @@
#include <asm/processor.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>
-#include <asm/system.h>
#include <asm/cacheflush.h>
static inline void __flush_itlb_all (void)
{
- int way, index;
+ int w, i;
- for (way = 0; way < XCHAL_ITLB_ARF_WAYS; way++) {
- for (index = 0; index < ITLB_ENTRIES_PER_ARF_WAY; index++) {
- int entry = way + (index << PAGE_SHIFT);
- invalidate_itlb_entry_no_isync (entry);
+ for (w = 0; w < ITLB_ARF_WAYS; w++) {
+ for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) {
+ int e = w + (i << PAGE_SHIFT);
+ invalidate_itlb_entry_no_isync(e);
}
}
asm volatile ("isync\n");
@@ -37,19 +36,19 @@ static inline void __flush_itlb_all (void)
static inline void __flush_dtlb_all (void)
{
- int way, index;
+ int w, i;
- for (way = 0; way < XCHAL_DTLB_ARF_WAYS; way++) {
- for (index = 0; index < DTLB_ENTRIES_PER_ARF_WAY; index++) {
- int entry = way + (index << PAGE_SHIFT);
- invalidate_dtlb_entry_no_isync (entry);
+ for (w = 0; w < DTLB_ARF_WAYS; w++) {
+ for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) {
+ int e = w + (i << PAGE_SHIFT);
+ invalidate_dtlb_entry_no_isync(e);
}
}
asm volatile ("isync\n");
}
-void flush_tlb_all (void)
+void local_flush_tlb_all(void)
{
__flush_itlb_all();
__flush_dtlb_all();
@@ -61,43 +60,53 @@ void flush_tlb_all (void)
* a new context will be assigned to it.
*/
-void flush_tlb_mm(struct mm_struct *mm)
+void local_flush_tlb_mm(struct mm_struct *mm)
{
-#if 0
- printk("[tlbmm<%lx>]\n", (unsigned long)mm->context);
-#endif
+ int cpu = smp_processor_id();
if (mm == current->active_mm) {
- int flags;
- local_save_flags(flags);
- get_new_mmu_context(mm, asid_cache);
- set_rasid_register(ASID_INSERT(mm->context));
+ unsigned long flags;
+ local_irq_save(flags);
+ mm->context.asid[cpu] = NO_CONTEXT;
+ activate_context(mm, cpu);
local_irq_restore(flags);
+ } else {
+ mm->context.asid[cpu] = NO_CONTEXT;
+ mm->context.cpu = -1;
}
- else
- mm->context = 0;
}
-void flush_tlb_range (struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
+
+#define _ITLB_ENTRIES (ITLB_ARF_WAYS << XCHAL_ITLB_ARF_ENTRIES_LOG2)
+#define _DTLB_ENTRIES (DTLB_ARF_WAYS << XCHAL_DTLB_ARF_ENTRIES_LOG2)
+#if _ITLB_ENTRIES > _DTLB_ENTRIES
+# define _TLB_ENTRIES _ITLB_ENTRIES
+#else
+# define _TLB_ENTRIES _DTLB_ENTRIES
+#endif
+
+void local_flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
{
+ int cpu = smp_processor_id();
struct mm_struct *mm = vma->vm_mm;
unsigned long flags;
- if (mm->context == NO_CONTEXT)
+ if (mm->context.asid[cpu] == NO_CONTEXT)
return;
#if 0
printk("[tlbrange<%02lx,%08lx,%08lx>]\n",
- (unsigned long)mm->context, start, end);
+ (unsigned long)mm->context.asid[cpu], start, end);
#endif
- local_save_flags(flags);
+ local_irq_save(flags);
- if (end-start + (PAGE_SIZE-1) <= SMALLEST_NTLB_ENTRIES << PAGE_SHIFT) {
+ if (end-start + (PAGE_SIZE-1) <= _TLB_ENTRIES << PAGE_SHIFT) {
int oldpid = get_rasid_register();
- set_rasid_register (ASID_INSERT(mm->context));
+
+ set_rasid_register(ASID_INSERT(mm->context.asid[cpu]));
start &= PAGE_MASK;
- if (vma->vm_flags & VM_EXEC)
+ if (vma->vm_flags & VM_EXEC)
while(start < end) {
invalidate_itlb_mapping(start);
invalidate_dtlb_mapping(start);
@@ -111,29 +120,25 @@ void flush_tlb_range (struct vm_area_struct *vma,
set_rasid_register(oldpid);
} else {
- get_new_mmu_context(mm, asid_cache);
- if (mm == current->active_mm)
- set_rasid_register(ASID_INSERT(mm->context));
+ local_flush_tlb_mm(mm);
}
local_irq_restore(flags);
}
-void flush_tlb_page (struct vm_area_struct *vma, unsigned long page)
+void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
+ int cpu = smp_processor_id();
struct mm_struct* mm = vma->vm_mm;
unsigned long flags;
int oldpid;
-#if 0
- printk("[tlbpage<%02lx,%08lx>]\n",
- (unsigned long)mm->context, page);
-#endif
- if(mm->context == NO_CONTEXT)
+ if (mm->context.asid[cpu] == NO_CONTEXT)
return;
- local_save_flags(flags);
+ local_irq_save(flags);
- oldpid = get_rasid_register();
+ oldpid = get_rasid_register();
+ set_rasid_register(ASID_INSERT(mm->context.asid[cpu]));
if (vma->vm_flags & VM_EXEC)
invalidate_itlb_mapping(page);
@@ -142,404 +147,132 @@ void flush_tlb_page (struct vm_area_struct *vma, unsigned long page)
set_rasid_register(oldpid);
local_irq_restore(flags);
-
-#if 0
- flush_tlb_all();
- return;
-#endif
-}
-
-
-#ifdef DEBUG_TLB
-
-#define USE_ITLB 0
-#define USE_DTLB 1
-
-struct way_config_t {
- int indicies;
- int indicies_log2;
- int pgsz_log2;
- int arf;
-};
-
-static struct way_config_t itlb[XCHAL_ITLB_WAYS] =
-{
- { XCHAL_ITLB_SET(XCHAL_ITLB_WAY0_SET, ENTRIES),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY0_SET, ENTRIES_LOG2),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY0_SET, PAGESZ_LOG2_MIN),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY0_SET, ARF)
- },
- { XCHAL_ITLB_SET(XCHAL_ITLB_WAY1_SET, ENTRIES),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY1_SET, ENTRIES_LOG2),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY1_SET, PAGESZ_LOG2_MIN),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY1_SET, ARF)
- },
- { XCHAL_ITLB_SET(XCHAL_ITLB_WAY2_SET, ENTRIES),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY2_SET, ENTRIES_LOG2),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY2_SET, PAGESZ_LOG2_MIN),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY2_SET, ARF)
- },
- { XCHAL_ITLB_SET(XCHAL_ITLB_WAY3_SET, ENTRIES),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY3_SET, ENTRIES_LOG2),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY3_SET, PAGESZ_LOG2_MIN),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY3_SET, ARF)
- },
- { XCHAL_ITLB_SET(XCHAL_ITLB_WAY4_SET, ENTRIES),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY4_SET, ENTRIES_LOG2),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY4_SET, PAGESZ_LOG2_MIN),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY4_SET, ARF)
- },
- { XCHAL_ITLB_SET(XCHAL_ITLB_WAY5_SET, ENTRIES),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY5_SET, ENTRIES_LOG2),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY5_SET, PAGESZ_LOG2_MIN),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY5_SET, ARF)
- },
- { XCHAL_ITLB_SET(XCHAL_ITLB_WAY6_SET, ENTRIES),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY6_SET, ENTRIES_LOG2),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY6_SET, PAGESZ_LOG2_MIN),
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY6_SET, ARF)
- }
-};
-
-static struct way_config_t dtlb[XCHAL_DTLB_WAYS] =
-{
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY0_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY0_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY0_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY0_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY1_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY1_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY1_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY1_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY2_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY2_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY2_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY2_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY3_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY3_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY3_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY3_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY4_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY4_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY4_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY4_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY5_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY5_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY5_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY5_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY6_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY6_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY6_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY6_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY7_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY7_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY7_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY7_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY8_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY8_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY8_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY8_SET, ARF)
- },
- { XCHAL_DTLB_SET(XCHAL_DTLB_WAY9_SET, ENTRIES),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY9_SET, ENTRIES_LOG2),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY9_SET, PAGESZ_LOG2_MIN),
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY9_SET, ARF)
- }
-};
-
-/* Total number of entries: */
-#define ITLB_TOTAL_ENTRIES \
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY0_SET, ENTRIES) + \
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY1_SET, ENTRIES) + \
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY2_SET, ENTRIES) + \
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY3_SET, ENTRIES) + \
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY4_SET, ENTRIES) + \
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY5_SET, ENTRIES) + \
- XCHAL_ITLB_SET(XCHAL_ITLB_WAY6_SET, ENTRIES)
-#define DTLB_TOTAL_ENTRIES \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY0_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY1_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY2_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY3_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY4_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY5_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY6_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY7_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY8_SET, ENTRIES) + \
- XCHAL_DTLB_SET(XCHAL_DTLB_WAY9_SET, ENTRIES)
-
-
-typedef struct {
- unsigned va;
- unsigned pa;
- unsigned char asid;
- unsigned char ca;
- unsigned char way;
- unsigned char index;
- unsigned char pgsz_log2; /* 0 .. 32 */
- unsigned char type; /* 0=ITLB 1=DTLB */
-} tlb_dump_entry_t;
-
-/* Return -1 if a precedes b, +1 if a follows b, 0 if same: */
-int cmp_tlb_dump_info( tlb_dump_entry_t *a, tlb_dump_entry_t *b )
-{
- if (a->asid < b->asid) return -1;
- if (a->asid > b->asid) return 1;
- if (a->va < b->va) return -1;
- if (a->va > b->va) return 1;
- if (a->pa < b->pa) return -1;
- if (a->pa > b->pa) return 1;
- if (a->ca < b->ca) return -1;
- if (a->ca > b->ca) return 1;
- if (a->way < b->way) return -1;
- if (a->way > b->way) return 1;
- if (a->index < b->index) return -1;
- if (a->index > b->index) return 1;
- return 0;
-}
-
-void sort_tlb_dump_info( tlb_dump_entry_t *t, int n )
-{
- int i, j;
- /* Simple O(n*n) sort: */
- for (i = 0; i < n-1; i++)
- for (j = i+1; j < n; j++)
- if (cmp_tlb_dump_info(t+i, t+j) > 0) {
- tlb_dump_entry_t tmp = t[i];
- t[i] = t[j];
- t[j] = tmp;
- }
-}
-
-
-static tlb_dump_entry_t itlb_dump_info[ITLB_TOTAL_ENTRIES];
-static tlb_dump_entry_t dtlb_dump_info[DTLB_TOTAL_ENTRIES];
-
-
-static inline char *way_type (int type)
-{
- return type ? "autorefill" : "non-autorefill";
-}
-
-void print_entry (struct way_config_t *way_info,
- unsigned int way,
- unsigned int index,
- unsigned int virtual,
- unsigned int translation)
-{
- char valid_chr;
- unsigned int va, pa, asid, ca;
-
- va = virtual &
- ~((1 << (way_info->pgsz_log2 + way_info->indicies_log2)) - 1);
- asid = virtual & ((1 << XCHAL_MMU_ASID_BITS) - 1);
- pa = translation & ~((1 << way_info->pgsz_log2) - 1);
- ca = translation & ((1 << XCHAL_MMU_CA_BITS) - 1);
- valid_chr = asid ? 'V' : 'I';
-
- /* Compute and incorporate the effect of the index bits on the
- * va. It's more useful for kernel debugging, since we always
- * want to know the effective va anyway. */
-
- va += index << way_info->pgsz_log2;
-
- printk ("\t[%d,%d] (%c) vpn 0x%.8x ppn 0x%.8x asid 0x%.2x am 0x%x\n",
- way, index, valid_chr, va, pa, asid, ca);
-}
-
-void print_itlb_entry (struct way_config_t *way_info, int way, int index)
-{
- print_entry (way_info, way, index,
- read_itlb_virtual (way + (index << way_info->pgsz_log2)),
- read_itlb_translation (way + (index << way_info->pgsz_log2)));
-}
-
-void print_dtlb_entry (struct way_config_t *way_info, int way, int index)
-{
- print_entry (way_info, way, index,
- read_dtlb_virtual (way + (index << way_info->pgsz_log2)),
- read_dtlb_translation (way + (index << way_info->pgsz_log2)));
-}
-
-void dump_itlb (void)
-{
- int way, index;
-
- printk ("\nITLB: ways = %d\n", XCHAL_ITLB_WAYS);
-
- for (way = 0; way < XCHAL_ITLB_WAYS; way++) {
- printk ("\nWay: %d, Entries: %d, MinPageSize: %d, Type: %s\n",
- way, itlb[way].indicies,
- itlb[way].pgsz_log2, way_type(itlb[way].arf));
- for (index = 0; index < itlb[way].indicies; index++) {
- print_itlb_entry(&itlb[way], way, index);
- }
- }
}
-void dump_dtlb (void)
+void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
- int way, index;
-
- printk ("\nDTLB: ways = %d\n", XCHAL_DTLB_WAYS);
-
- for (way = 0; way < XCHAL_DTLB_WAYS; way++) {
- printk ("\nWay: %d, Entries: %d, MinPageSize: %d, Type: %s\n",
- way, dtlb[way].indicies,
- dtlb[way].pgsz_log2, way_type(dtlb[way].arf));
- for (index = 0; index < dtlb[way].indicies; index++) {
- print_dtlb_entry(&dtlb[way], way, index);
+ if (end > start && start >= TASK_SIZE && end <= PAGE_OFFSET &&
+ end - start < _TLB_ENTRIES << PAGE_SHIFT) {
+ start &= PAGE_MASK;
+ while (start < end) {
+ invalidate_itlb_mapping(start);
+ invalidate_dtlb_mapping(start);
+ start += PAGE_SIZE;
}
+ } else {
+ local_flush_tlb_all();
}
}
-void dump_tlb (tlb_dump_entry_t *tinfo, struct way_config_t *config,
- int entries, int ways, int type, int show_invalid)
-{
- tlb_dump_entry_t *e = tinfo;
- int way, i;
-
- /* Gather all info: */
- for (way = 0; way < ways; way++) {
- struct way_config_t *cfg = config + way;
- for (i = 0; i < cfg->indicies; i++) {
- unsigned wayindex = way + (i << cfg->pgsz_log2);
- unsigned vv = (type ? read_dtlb_virtual (wayindex)
- : read_itlb_virtual (wayindex));
- unsigned pp = (type ? read_dtlb_translation (wayindex)
- : read_itlb_translation (wayindex));
-
- /* Compute and incorporate the effect of the index bits on the
- * va. It's more useful for kernel debugging, since we always
- * want to know the effective va anyway. */
-
- e->va = (vv & ~((1 << (cfg->pgsz_log2 + cfg->indicies_log2)) - 1));
- e->va += (i << cfg->pgsz_log2);
- e->pa = (pp & ~((1 << cfg->pgsz_log2) - 1));
- e->asid = (vv & ((1 << XCHAL_MMU_ASID_BITS) - 1));
- e->ca = (pp & ((1 << XCHAL_MMU_CA_BITS) - 1));
- e->way = way;
- e->index = i;
- e->pgsz_log2 = cfg->pgsz_log2;
- e->type = type;
- e++;
- }
- }
-#if 1
- /* Sort by ASID and VADDR: */
- sort_tlb_dump_info (tinfo, entries);
-#endif
-
- /* Display all sorted info: */
- printk ("\n%cTLB dump:\n", (type ? 'D' : 'I'));
- for (e = tinfo, i = 0; i < entries; i++, e++) {
-#if 0
- if (e->asid == 0 && !show_invalid)
- continue;
-#endif
- printk ("%c way=%d i=%d ASID=%02X V=%08X -> P=%08X CA=%X (%d %cB)\n",
- (e->type ? 'D' : 'I'), e->way, e->index,
- e->asid, e->va, e->pa, e->ca,
- (1 << (e->pgsz_log2 % 10)),
- " kMG"[e->pgsz_log2 / 10]
- );
- }
-}
+#ifdef CONFIG_DEBUG_TLB_SANITY
-void dump_tlbs2 (int showinv)
+static unsigned get_pte_for_vaddr(unsigned vaddr)
{
- dump_tlb (itlb_dump_info, itlb, ITLB_TOTAL_ENTRIES, XCHAL_ITLB_WAYS, 0, showinv);
- dump_tlb (dtlb_dump_info, dtlb, DTLB_TOTAL_ENTRIES, XCHAL_DTLB_WAYS, 1, showinv);
+ struct task_struct *task = get_current();
+ struct mm_struct *mm = task->mm;
+ pgd_t *pgd;
+ pmd_t *pmd;
+ pte_t *pte;
+
+ if (!mm)
+ mm = task->active_mm;
+ pgd = pgd_offset(mm, vaddr);
+ if (pgd_none_or_clear_bad(pgd))
+ return 0;
+ pmd = pmd_offset(pgd, vaddr);
+ if (pmd_none_or_clear_bad(pmd))
+ return 0;
+ pte = pte_offset_map(pmd, vaddr);
+ if (!pte)
+ return 0;
+ return pte_val(*pte);
}
-void dump_all_tlbs (void)
-{
- dump_tlbs2 (1);
-}
+enum {
+ TLB_SUSPICIOUS = 1,
+ TLB_INSANE = 2,
+};
-void dump_valid_tlbs (void)
+static void tlb_insane(void)
{
- dump_tlbs2 (0);
+ BUG_ON(1);
}
-
-void dump_tlbs (void)
+static void tlb_suspicious(void)
{
- dump_itlb();
- dump_dtlb();
+ WARN_ON(1);
}
-void dump_cache_tag(int dcache, int idx)
+/*
+ * Check that TLB entries with kernel ASID (1) have kernel VMA (>= TASK_SIZE),
+ * and TLB entries with user ASID (>=4) have VMA < TASK_SIZE.
+ *
+ * Check that valid TLB entries either have the same PA as the PTE, or PTE is
+ * marked as non-present. Non-present PTE and the page with non-zero refcount
+ * and zero mapcount is normal for batched TLB flush operation. Zero refcount
+ * means that the page was freed prematurely. Non-zero mapcount is unusual,
+ * but does not necessary means an error, thus marked as suspicious.
+ */
+static int check_tlb_entry(unsigned w, unsigned e, bool dtlb)
{
- int w, i, s, e;
- unsigned long tag, index;
- unsigned long num_lines, num_ways, cache_size, line_size;
-
- num_ways = dcache ? XCHAL_DCACHE_WAYS : XCHAL_ICACHE_WAYS;
- cache_size = dcache ? XCHAL_DCACHE_SIZE : XCHAL_ICACHE_SIZE;
- line_size = dcache ? XCHAL_DCACHE_LINESIZE : XCHAL_ICACHE_LINESIZE;
-
- num_lines = cache_size / num_ways;
-
- s = 0; e = num_lines;
-
- if (idx >= 0)
- e = (s = idx * line_size) + 1;
-
- for (i = s; i < e; i+= line_size) {
- printk("\nline %#08x:", i);
- for (w = 0; w < num_ways; w++) {
- index = w * num_lines + i;
- if (dcache)
- __asm__ __volatile__("ldct %0, %1\n\t"
- : "=a"(tag) : "a"(index));
- else
- __asm__ __volatile__("lict %0, %1\n\t"
- : "=a"(tag) : "a"(index));
-
- printk(" %#010lx", tag);
- }
+ unsigned tlbidx = w | (e << PAGE_SHIFT);
+ unsigned r0 = dtlb ?
+ read_dtlb_virtual(tlbidx) : read_itlb_virtual(tlbidx);
+ unsigned vpn = (r0 & PAGE_MASK) | (e << PAGE_SHIFT);
+ unsigned pte = get_pte_for_vaddr(vpn);
+ unsigned mm_asid = (get_rasid_register() >> 8) & ASID_MASK;
+ unsigned tlb_asid = r0 & ASID_MASK;
+ bool kernel = tlb_asid == 1;
+ int rc = 0;
+
+ if (tlb_asid > 0 && ((vpn < TASK_SIZE) == kernel)) {
+ pr_err("%cTLB: way: %u, entry: %u, VPN %08x in %s PTE\n",
+ dtlb ? 'D' : 'I', w, e, vpn,
+ kernel ? "kernel" : "user");
+ rc |= TLB_INSANE;
}
- printk ("\n");
-}
-
-void dump_icache(int index)
-{
- unsigned long data, addr;
- int w, i;
-
- const unsigned long num_ways = XCHAL_ICACHE_WAYS;
- const unsigned long cache_size = XCHAL_ICACHE_SIZE;
- const unsigned long line_size = XCHAL_ICACHE_LINESIZE;
- const unsigned long num_lines = cache_size / num_ways / line_size;
- for (w = 0; w < num_ways; w++) {
- printk ("\nWay %d", w);
-
- for (i = 0; i < line_size; i+= 4) {
- addr = w * num_lines + index * line_size + i;
- __asm__ __volatile__("licw %0, %1\n\t"
- : "=a"(data) : "a"(addr));
- printk(" %#010lx", data);
+ if (tlb_asid == mm_asid) {
+ unsigned r1 = dtlb ? read_dtlb_translation(tlbidx) :
+ read_itlb_translation(tlbidx);
+ if ((pte ^ r1) & PAGE_MASK) {
+ pr_err("%cTLB: way: %u, entry: %u, mapping: %08x->%08x, PTE: %08x\n",
+ dtlb ? 'D' : 'I', w, e, r0, r1, pte);
+ if (pte == 0 || !pte_present(__pte(pte))) {
+ struct page *p = pfn_to_page(r1 >> PAGE_SHIFT);
+ pr_err("page refcount: %d, mapcount: %d\n",
+ page_count(p),
+ page_mapcount(p));
+ if (!page_count(p))
+ rc |= TLB_INSANE;
+ else if (page_mapped(p))
+ rc |= TLB_SUSPICIOUS;
+ } else {
+ rc |= TLB_INSANE;
+ }
}
}
- printk ("\n");
+ return rc;
}
-void dump_cache_tags(void)
+void check_tlb_sanity(void)
{
- printk("Instruction cache\n");
- dump_cache_tag(0, -1);
- printk("Data cache\n");
- dump_cache_tag(1, -1);
+ unsigned long flags;
+ unsigned w, e;
+ int bug = 0;
+
+ local_irq_save(flags);
+ for (w = 0; w < DTLB_ARF_WAYS; ++w)
+ for (e = 0; e < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); ++e)
+ bug |= check_tlb_entry(w, e, true);
+ for (w = 0; w < ITLB_ARF_WAYS; ++w)
+ for (e = 0; e < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); ++e)
+ bug |= check_tlb_entry(w, e, false);
+ if (bug & TLB_INSANE)
+ tlb_insane();
+ if (bug & TLB_SUSPICIOUS)
+ tlb_suspicious();
+ local_irq_restore(flags);
}
-#endif
+#endif /* CONFIG_DEBUG_TLB_SANITY */
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-12-05 21:01:20 +0000