path: root/arch/ppc64/mm/hugetlbpage.c

/*
 * PPC64 (POWER4) Huge TLB Page Support for Kernel.
 *
 * Copyright (C) 2003 David Gibson, IBM Corporation.
 *
 * Based on the IA-32 version:
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/smp_lock.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <asm/mman.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/machdep.h>
#include <asm/cputable.h>
#include <asm/tlb.h>

#include <linux/sysctl.h>

#define	HUGEPGDIR_SHIFT		(HPAGE_SHIFT + PAGE_SHIFT - 3)
#define HUGEPGDIR_SIZE		(1UL << HUGEPGDIR_SHIFT)
#define HUGEPGDIR_MASK		(~(HUGEPGDIR_SIZE-1))

#define HUGEPTE_INDEX_SIZE	9
#define HUGEPGD_INDEX_SIZE	10

#define PTRS_PER_HUGEPTE	(1 << HUGEPTE_INDEX_SIZE)
#define PTRS_PER_HUGEPGD	(1 << HUGEPGD_INDEX_SIZE)

static inline int hugepgd_index(unsigned long addr)
{
	return (addr & ~REGION_MASK) >> HUGEPGDIR_SHIFT;
}

static pgd_t *hugepgd_offset(struct mm_struct *mm, unsigned long addr)
{
	int index;

	if (! mm->context.huge_pgdir)
		return NULL;


	index = hugepgd_index(addr);
	BUG_ON(index >= PTRS_PER_HUGEPGD);
	return mm->context.huge_pgdir + index;
}

static inline pte_t *hugepte_offset(pgd_t *dir, unsigned long addr)
{
	int index;

	if (pgd_none(*dir))
		return NULL;

	index = (addr >> HPAGE_SHIFT) % PTRS_PER_HUGEPTE;
	return (pte_t *)pgd_page(*dir) + index;
}

static pgd_t *hugepgd_alloc(struct mm_struct *mm, unsigned long addr)
{
	BUG_ON(! in_hugepage_area(mm->context, addr));

	if (! mm->context.huge_pgdir) {
		pgd_t *new;
		spin_unlock(&mm->page_table_lock);
		/* Don't use pgd_alloc(), because we want __GFP_REPEAT */
		new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
		BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
		spin_lock(&mm->page_table_lock);

		/*
		 * Because we dropped the lock, we should re-check the
		 * entry, as somebody else could have populated it..
		 */
		if (mm->context.huge_pgdir)
			pgd_free(new);
		else
			mm->context.huge_pgdir = new;
	}
	return hugepgd_offset(mm, addr);
}

static pte_t *hugepte_alloc(struct mm_struct *mm, pgd_t *dir,
			    unsigned long addr)
{
	if (! pgd_present(*dir)) {
		pte_t *new;

		spin_unlock(&mm->page_table_lock);
		new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
		BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
		spin_lock(&mm->page_table_lock);
		/*
		 * Because we dropped the lock, we should re-check the
		 * entry, as somebody else could have populated it..
		 */
		if (pgd_present(*dir)) {
			if (new)
				kmem_cache_free(zero_cache, new);
		} else {
			struct page *ptepage;

			if (! new)
				return NULL;
			ptepage = virt_to_page(new);
			ptepage->mapping = (void *) mm;
			ptepage->index = addr & HUGEPGDIR_MASK;
			pgd_populate(mm, dir, new);
		}
	}

	return hugepte_offset(dir, addr);
}

static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgd;

	BUG_ON(! in_hugepage_area(mm->context, addr));

	pgd = hugepgd_offset(mm, addr);
	if (! pgd)
		return NULL;

	return hugepte_offset(pgd, addr);
}

static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgd;

	BUG_ON(! in_hugepage_area(mm->context, addr));

	pgd = hugepgd_alloc(mm, addr);
	if (! pgd)
		return NULL;

	return hugepte_alloc(mm, pgd, addr);
}

static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma,
			 unsigned long addr, struct page *page,
			 pte_t *ptep, int write_access)
{
	pte_t entry;

	add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
	if (write_access) {
		entry =
		    pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
	} else {
		entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
	}
	entry = pte_mkyoung(entry);
	entry = pte_mkhuge(entry);

	set_pte_at(mm, addr, ptep, entry);
}

/*
 * This function checks for proper alignment of input addr and len parameters.
 */
int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
{
	if (len & ~HPAGE_MASK)
		return -EINVAL;
	if