diff options
Diffstat (limited to 'arch/powerpc/mm/hash_low_64.S')
| -rw-r--r-- | arch/powerpc/mm/hash_low_64.S | 164 |
1 files changed, 101 insertions, 63 deletions
diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S index 56585086413..057cbbb4c57 100644 --- a/arch/powerpc/mm/hash_low_64.S +++ b/arch/powerpc/mm/hash_low_64.S @@ -115,11 +115,13 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) sldi r29,r5,SID_SHIFT - VPN_SHIFT rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) or r29,r28,r29 - - /* Calculate hash value for primary slot and store it in r28 */ - rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ - rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ - xor r28,r5,r0 + /* + * Calculate hash value for primary slot and store it in r28 + * r3 = va, r5 = vsid + * r0 = (va >> 12) & ((1ul << (28 - 12)) -1) + */ + rldicl r0,r3,64-12,48 + xor r28,r5,r0 /* hash */ b 4f 3: /* Calc vpn and put it in r29 */ @@ -130,11 +132,12 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) /* * calculate hash value for primary slot and * store it in r28 for 1T segment + * r3 = va, r5 = vsid */ - rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ - clrldi r5,r5,40 /* vsid & 0xffffff */ - rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ - xor r28,r28,r5 + sldi r28,r5,25 /* vsid << 25 */ + /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */ + rldicl r0,r3,64-12,36 + xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ xor r28,r28,r0 /* hash */ /* Convert linux PTE bits into HW equivalents */ @@ -145,7 +148,10 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ andc r0,r30,r0 /* r0 = pte & ~r0 */ rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ - ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ + /* + * Always add "C" bit for perf. Memory coherence is always enabled + */ + ori r3,r3,HPTE_R_C | HPTE_R_M /* We eventually do the icache sync here (maybe inline that * code rather than call a C function...) @@ -153,7 +159,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) BEGIN_FTR_SECTION mr r4,r30 mr r5,r7 - bl .hash_page_do_lazy_icache + bl hash_page_do_lazy_icache END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) /* At this point, r3 contains new PP bits, save them in @@ -193,8 +199,10 @@ htab_insert_pte: mr r4,r29 /* Retrieve vpn */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ -_GLOBAL(htab_call_hpte_insert1) + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ +.globl htab_call_hpte_insert1 +htab_call_hpte_insert1: bl . /* Patched by htab_finish_init() */ cmpdi 0,r3,0 bge htab_pte_insert_ok /* Insertion successful */ @@ -216,8 +224,10 @@ _GLOBAL(htab_call_hpte_insert1) mr r4,r29 /* Retrieve vpn */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ -_GLOBAL(htab_call_hpte_insert2) + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ +.globl htab_call_hpte_insert2 +htab_call_hpte_insert2: bl . /* Patched by htab_finish_init() */ cmpdi 0,r3,0 bge+ htab_pte_insert_ok /* Insertion successful */ @@ -234,7 +244,8 @@ _GLOBAL(htab_call_hpte_insert2) 2: and r0,r5,r27 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ /* Call ppc_md.hpte_remove */ -_GLOBAL(htab_call_hpte_remove) +.globl htab_call_hpte_remove +htab_call_hpte_remove: bl . /* Patched by htab_finish_init() */ /* Try all again */ @@ -284,10 +295,12 @@ htab_modify_pte: /* Call ppc_md.hpte_updatepp */ mr r5,r29 /* vpn */ - li r6,MMU_PAGE_4K /* page size */ - ld r7,STK_PARAM(R9)(r1) /* segment size */ - ld r8,STK_PARAM(R8)(r1) /* get "local" param */ -_GLOBAL(htab_call_hpte_updatepp) + li r6,MMU_PAGE_4K /* base page size */ + li r7,MMU_PAGE_4K /* actual page size */ + ld r8,STK_PARAM(R9)(r1) /* segment size */ + ld r9,STK_PARAM(R8)(r1) /* get "local" param */ +.globl htab_call_hpte_updatepp +htab_call_hpte_updatepp: bl . /* Patched by htab_finish_init() */ /* if we failed because typically the HPTE wasn't really here @@ -407,11 +420,13 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) */ rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) or r29,r28,r29 - - /* Calculate hash value for primary slot and store it in r28 */ - rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ - rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ - xor r28,r5,r0 + /* + * Calculate hash value for primary slot and store it in r28 + * r3 = va, r5 = vsid + * r0 = (va >> 12) & ((1ul << (28 - 12)) -1) + */ + rldicl r0,r3,64-12,48 + xor r28,r5,r0 /* hash */ b 4f 3: /* Calc vpn and put it in r29 */ @@ -426,11 +441,12 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) /* * Calculate hash value for primary slot and * store it in r28 for 1T segment + * r3 = va, r5 = vsid */ - rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ - clrldi r5,r5,40 /* vsid & 0xffffff */ - rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ - xor r28,r28,r5 + sldi r28,r5,25 /* vsid << 25 */ + /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */ + rldicl r0,r3,64-12,36 + xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ xor r28,r28,r0 /* hash */ /* Convert linux PTE bits into HW equivalents */ @@ -448,7 +464,10 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ andc r0,r3,r0 /* r0 = pte & ~r0 */ rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ - ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ + /* + * Always add "C" bit for perf. Memory coherence is always enabled + */ + ori r3,r3,HPTE_R_C | HPTE_R_M /* We eventually do the icache sync here (maybe inline that * code rather than call a C function...) @@ -456,7 +475,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) BEGIN_FTR_SECTION mr r4,r30 mr r5,r7 - bl .hash_page_do_lazy_icache + bl hash_page_do_lazy_icache END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) /* At this point, r3 contains new PP bits, save them in @@ -484,7 +503,7 @@ END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) beq htab_inval_old_hpte ld r6,STK_PARAM(R6)(r1) - ori r26,r6,0x8000 /* Load the hidx mask */ + ori r26,r6,PTE_PAGE_HIDX_OFFSET /* Load the hidx mask. */ ld r26,0(r26) addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ @@ -509,8 +528,10 @@ htab_special_pfn: mr r4,r29 /* Retrieve vpn */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ -_GLOBAL(htab_call_hpte_insert1) + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ +.globl htab_call_hpte_insert1 +htab_call_hpte_insert1: bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 bge htab_pte_insert_ok /* Insertion successful */ @@ -536,8 +557,10 @@ _GLOBAL(htab_call_hpte_insert1) mr r4,r29 /* Retrieve vpn */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ -_GLOBAL(htab_call_hpte_insert2) + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ +.globl htab_call_hpte_insert2 +htab_call_hpte_insert2: bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 bge+ htab_pte_insert_ok /* Insertion successful */ @@ -554,7 +577,8 @@ _GLOBAL(htab_call_hpte_insert2) 2: and r0,r5,r27 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ /* Call ppc_md.hpte_remove */ -_GLOBAL(htab_call_hpte_remove) +.globl htab_call_hpte_remove +htab_call_hpte_remove: bl . /* patched by htab_finish_init() */ /* Try all again */ @@ -571,7 +595,7 @@ htab_inval_old_hpte: li r6,MMU_PAGE_64K /* psize */ ld r7,STK_PARAM(R9)(r1) /* ssize */ ld r8,STK_PARAM(R8)(r1) /* local */ - bl .flush_hash_page + bl flush_hash_page /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */ lis r0,_PAGE_HPTE_SUB@h ori r0,r0,_PAGE_HPTE_SUB@l @@ -601,7 +625,7 @@ htab_pte_insert_ok: sld r4,r4,r5 andc r26,r26,r4 or r26,r26,r3 - ori r5,r6,0x8000 + ori r5,r6,PTE_PAGE_HIDX_OFFSET std r26,0(r5) lwsync std r30,0(r6) @@ -639,10 +663,12 @@ htab_modify_pte: /* Call ppc_md.hpte_updatepp */ mr r5,r29 /* vpn */ - li r6,MMU_PAGE_4K /* page size */ - ld r7,STK_PARAM(R9)(r1) /* segment size */ - ld r8,STK_PARAM(R8)(r1) /* get "local" param */ -_GLOBAL(htab_call_hpte_updatepp) + li r6,MMU_PAGE_4K /* base page size */ + li r7,MMU_PAGE_4K /* actual page size */ + ld r8,STK_PARAM(R9)(r1) /* segment size */ + ld r9,STK_PARAM(R8)(r1) /* get "local" param */ +.globl htab_call_hpte_updatepp +htab_call_hpte_updatepp: bl . /* patched by htab_finish_init() */ /* if we failed because typically the HPTE wasn't really here @@ -752,25 +778,27 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) or r29,r28,r29 - /* Calculate hash value for primary slot and store it in r28 */ - rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ - rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ - xor r28,r5,r0 + /* Calculate hash value for primary slot and store it in r28 + * r3 = va, r5 = vsid + * r0 = (va >> 16) & ((1ul << (28 - 16)) -1) + */ + rldicl r0,r3,64-16,52 + xor r28,r5,r0 /* hash */ b 4f 3: /* Calc vpn and put it in r29 */ sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT) or r29,r28,r29 - /* * calculate hash value for primary slot and * store it in r28 for 1T segment + * r3 = va, r5 = vsid */ - rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ - clrldi r5,r5,40 /* vsid & 0xffffff */ - rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */ - xor r28,r28,r5 + sldi r28,r5,25 /* vsid << 25 */ + /* r0 = (va >> 16) & ((1ul << (40 - 16)) -1) */ + rldicl r0,r3,64-16,40 + xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ xor r28,r28,r0 /* hash */ /* Convert linux PTE bits into HW equivalents */ @@ -781,7 +809,10 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ andc r0,r30,r0 /* r0 = pte & ~r0 */ rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ - ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ + /* + * Always add "C" bit for perf. Memory coherence is always enabled + */ + ori r3,r3,HPTE_R_C | HPTE_R_M /* We eventually do the icache sync here (maybe inline that * code rather than call a C function...) @@ -789,7 +820,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) BEGIN_FTR_SECTION mr r4,r30 mr r5,r7 - bl .hash_page_do_lazy_icache + bl hash_page_do_lazy_icache END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) /* At this point, r3 contains new PP bits, save them in @@ -832,8 +863,10 @@ ht64_insert_pte: mr r4,r29 /* Retrieve vpn */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_64K - ld r9,STK_PARAM(R9)(r1) /* segment size */ -_GLOBAL(ht64_call_hpte_insert1) + li r9,MMU_PAGE_64K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ +.globl ht64_call_hpte_insert1 +ht64_call_hpte_insert1: bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 bge ht64_pte_insert_ok /* Insertion successful */ @@ -855,8 +888,10 @@ _GLOBAL(ht64_call_hpte_insert1) mr r4,r29 /* Retrieve vpn */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_64K - ld r9,STK_PARAM(R9)(r1) /* segment size */ -_GLOBAL(ht64_call_hpte_insert2) + li r9,MMU_PAGE_64K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ +.globl ht64_call_hpte_insert2 +ht64_call_hpte_insert2: bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 bge+ ht64_pte_insert_ok /* Insertion successful */ @@ -873,7 +908,8 @@ _GLOBAL(ht64_call_hpte_insert2) 2: and r0,r5,r27 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ /* Call ppc_md.hpte_remove */ -_GLOBAL(ht64_call_hpte_remove) +.globl ht64_call_hpte_remove +ht64_call_hpte_remove: bl . /* patched by htab_finish_init() */ /* Try all again */ @@ -923,10 +959,12 @@ ht64_modify_pte: /* Call ppc_md.hpte_updatepp */ mr r5,r29 /* vpn */ - li r6,MMU_PAGE_64K - ld r7,STK_PARAM(R9)(r1) /* segment size */ - ld r8,STK_PARAM(R8)(r1) /* get "local" param */ -_GLOBAL(ht64_call_hpte_updatepp) + li r6,MMU_PAGE_64K /* base page size */ + li r7,MMU_PAGE_64K /* actual page size */ + ld r8,STK_PARAM(R9)(r1) /* segment size */ + ld r9,STK_PARAM(R8)(r1) /* get "local" param */ +.globl ht64_call_hpte_updatepp +ht64_call_hpte_updatepp: bl . /* patched by htab_finish_init() */ /* if we failed because typically the HPTE wasn't really here |