path: root/arch/powerpc/mm/hugetlbpage.c

/*
 * PPC64 (POWER4) Huge TLB Page Support for Kernel.
 *
 * Copyright (C) 2003 David Gibson, IBM Corporation.
 *
 * Based on the IA-32 version:
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/smp_lock.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <asm/mman.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/machdep.h>
#include <asm/cputable.h>
#include <asm/tlb.h>

#include <linux/sysctl.h>

#define NUM_LOW_AREAS	(0x100000000UL >> SID_SHIFT)
#define NUM_HIGH_AREAS	(PGTABLE_RANGE >> HTLB_AREA_SHIFT)

#ifdef CONFIG_PPC_64K_PAGES
#define HUGEPTE_INDEX_SIZE	(PMD_SHIFT-HPAGE_SHIFT)
#else
#define HUGEPTE_INDEX_SIZE	(PUD_SHIFT-HPAGE_SHIFT)
#endif
#define PTRS_PER_HUGEPTE	(1 << HUGEPTE_INDEX_SIZE)
#define HUGEPTE_TABLE_SIZE	(sizeof(pte_t) << HUGEPTE_INDEX_SIZE)

#define HUGEPD_SHIFT		(HPAGE_SHIFT + HUGEPTE_INDEX_SIZE)
#define HUGEPD_SIZE		(1UL << HUGEPD_SHIFT)
#define HUGEPD_MASK		(~(HUGEPD_SIZE-1))

#define huge_pgtable_cache	(pgtable_cache[HUGEPTE_CACHE_NUM])

/* Flag to mark huge PD pointers.  This means pmd_bad() and pud_bad()
 * will choke on pointers to hugepte tables, which is handy for
 * catching screwups early. */
#define HUGEPD_OK	0x1

typedef struct { unsigned long pd; } hugepd_t;

#define hugepd_none(hpd)	((hpd).pd == 0)

static inline pte_t *hugepd_page(hugepd_t hpd)
{
	BUG_ON(!(hpd.pd & HUGEPD_OK));
	return (pte_t *)(hpd.pd & ~HUGEPD_OK);
}

static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr)
{
	unsigned long idx = ((addr >> HPAGE_SHIFT) & (PTRS_PER_HUGEPTE-1));
	pte_t *dir = hugepd_page(*hpdp);

	return dir + idx;
}

static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
			   unsigned long address)
{
	pte_t *new = kmem_cache_alloc(huge_pgtable_cache,
				      GFP_KERNEL|__GFP_REPEAT);

	if (! new)
		return -ENOMEM;

	spin_lock(&mm->page_table_lock);
	if (!hugepd_none(*hpdp))
		kmem_cache_free(huge_pgtable_cache, new);
	else
		hpdp->pd = (unsigned long)new | HUGEPD_OK;
	spin_unlock(&mm->page_table_lock);
	return 0;
}

/* Modelled after find_linux_pte() */
pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pg;
	pud_t *pu;

	BUG_ON(! in_hugepage_area(mm->context, addr));

	addr &= HPAGE_MASK;

	pg = pgd_offset(mm, addr);
	if (!pgd_none(*pg)) {
		pu = pud_offset(pg, addr);
		if (!pud_none(*pu)) {
#ifdef CONFIG_PPC_64K_PAGES
			pmd_t *pm;
			pm = pmd_offset(pu, addr);
			if (!pmd_none(*pm))
				return hugepte_offset((hugepd_t *)pm, addr);
#else
			return hugepte_offset((hugepd_t *)pu, addr);
#endif
		}
	}

	return NULL;
}

pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pg;
	pud_t *pu;
	hugepd_t *hpdp = NULL;

	BUG_ON(! in_hugepage_area(mm->context, addr));

	addr &= HPAGE_MASK;

	pg = pgd_offset(mm, addr);
	pu = pud_alloc(mm, pg, addr);

	if (pu) {
#ifdef CONFIG_PPC_64K_PAGES
		pmd_t *pm;
		pm = pmd_alloc(mm, pu, addr);
		if (pm)
			hpdp = (hugepd_t *)pm;
#else
		hpdp = (hugepd_t *)pu;
#endif
	}

	if (! hpdp)
		return NULL;

	if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr))
		return NULL;

	return hugepte_offset(hpdp, addr);
}

int huge_pmd_unshare(struct mm_struct *mm, unsigned long *