path: root/mm/hugetlb.c

/*
 * Generic hugetlb support.
 * (C) William Irwin, April 2004
 */
#include <linux/gfp.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/sysctl.h>
#include <linux/highmem.h>
#include <linux/nodemask.h>
#include <linux/pagemap.h>
#include <linux/mempolicy.h>
#include <linux/cpuset.h>
#include <linux/mutex.h>

#include <asm/page.h>
#include <asm/pgtable.h>

#include <linux/hugetlb.h>
#include "internal.h"

const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
unsigned long max_huge_pages;
static struct list_head hugepage_freelists[MAX_NUMNODES];
static unsigned int nr_huge_pages_node[MAX_NUMNODES];
static unsigned int free_huge_pages_node[MAX_NUMNODES];
static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
unsigned long hugepages_treat_as_movable;

/*
 * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
 */
static DEFINE_SPINLOCK(hugetlb_lock);

static void clear_huge_page(struct page *page, unsigned long addr)
{
	int i;

	might_sleep();
	for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
		cond_resched();
		clear_user_highpage(page + i, addr + i * PAGE_SIZE);
	}
}

static void copy_huge_page(struct page *dst, struct page *src,
			   unsigned long addr, struct vm_area_struct *vma)
{
	int i;

	might_sleep();
	for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
		cond_resched();
		copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
	}
}

static void enqueue_huge_page(struct page *page)
{
	int nid = page_to_nid(page);
	list_add(&page->lru, &hugepage_freelists[nid]);
	free_huge_pages++;
	free_huge_pages_node[nid]++;
}

static struct page *dequeue_huge_page(struct vm_area_struct *vma,
				unsigned long address)
{
	int nid;
	struct page *page = NULL;
	struct mempolicy *mpol;
	struct zonelist *zonelist = huge_zonelist(vma, address,
					htlb_alloc_mask, &mpol);
	struct zone **z;

	for (z = zonelist->zones; *z; z++) {
		nid = zone_to_nid(*z);
		if (cpuset_zone_allowed_softwall(*z, htlb_alloc_mask) &&
		    !list_empty(&hugepage_freelists[nid])) {
			page = list_entry(hugepage_freelists[nid].next,
					  struct page, lru);
			list_del(&page->lru);
			free_huge_pages--;
			free_huge_pages_node[nid]--;
			break;
		}
	}
	mpol_free(mpol);	/* unref if mpol !NULL */
	return page;
}

static void free_huge_page(struct page *page)
{
	BUG_ON(page_count(page));

	INIT_LIST_HEAD(&page->lru);

	spin_lock(&hugetlb_lock);
	enqueue_huge_page(page);
	spin_unlock(&hugetlb_lock);
}

static int alloc_fresh_huge_page(void)
{
	static int prev_nid;
	struct page *page;
	int nid;

	/*
	 * Copy static prev_nid to local nid, work on that, then copy it
	 * back to prev_nid afterwards: otherwise there's a window in which
	 * a racer might pass invalid nid MAX_NUMNODES to alloc_pages_node.
	 * But we don't need to use a spin_lock here: it really doesn't
	 * matter if occasionally a racer chooses the same nid as we do.
	 */
	nid = next_node(prev_nid, node_online_map);
	if (nid == MAX_NUMNODES)
		nid = first_node(node_online_map);
	prev_nid = nid;

	page = alloc_pages_node(nid, htlb_alloc_mask|__GFP_COMP|__GFP_NOWARN,
					HUGETLB_PAGE_ORDER);
	if (page) {
		set_compound_page_dtor(page, free_huge_page);
		spin_lock(&hugetlb_lock);
		nr_huge_pages++;
		nr_huge_pages_node[page_to_nid(page)]++;
		spin_unlock(&hugetlb_lock);
		put_page(page); /* free it into the hugepage allocator */
		return 1;
	}
	return 0;
}

static struct page *alloc_huge_page(struct vm_area_struct *vma,
				    unsigned long addr)
{
	struct page *page;

	spin_lock(&hugetlb_lock);
	if (vma->vm_flags & VM_MAYSHARE)
		resv_huge_pages--;
	else if (free_huge_pages <= resv_huge_pages)
		goto fail;

	page = dequeue_huge_page(vma, addr);
	if (!page)
		goto fail;

	spin_unlock(&hugetlb_lock);
	set_page_refcounted(page);
	return page;

fail:
	if (vma->vm_flags & VM_MAYSHARE)
		resv_huge_pages++;
	spin_unlock(&hugetlb_lock);
	return NULL;
}

static int __init hugetlb_init(void)
{
	unsigned long i;

	if (HPAGE_SHIFT == 0)
		return 0;

	for (i = 0; i < MAX_NUMNODES; ++i)
		INIT_LIST_HEAD(&hugepage_freelists[i]);