path: root/mm/compaction.c

/*
 * linux/mm/compaction.c
 *
 * Memory compaction for the reduction of external fragmentation. Note that
 * this heavily depends upon page migration to do all the real heavy
 * lifting
 *
 * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
 */
#include <linux/swap.h>
#include <linux/migrate.h>
#include <linux/compaction.h>
#include <linux/mm_inline.h>
#include <linux/backing-dev.h>
#include <linux/sysctl.h>
#include <linux/sysfs.h>
#include "internal.h"

#if defined CONFIG_COMPACTION || defined CONFIG_CMA

#define CREATE_TRACE_POINTS
#include <trace/events/compaction.h>

static unsigned long release_freepages(struct list_head *freelist)
{
	struct page *page, *next;
	unsigned long count = 0;

	list_for_each_entry_safe(page, next, freelist, lru) {
		list_del(&page->lru);
		__free_page(page);
		count++;
	}

	return count;
}

static void map_pages(struct list_head *list)
{
	struct page *page;

	list_for_each_entry(page, list, lru) {
		arch_alloc_page(page, 0);
		kernel_map_pages(page, 1, 1);
	}
}

static inline bool migrate_async_suitable(int migratetype)
{
	return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;
}

/*
 * Compaction requires the taking of some coarse locks that are potentially
 * very heavily contended. Check if the process needs to be scheduled or
 * if the lock is contended. For async compaction, back out in the event
 * if contention is severe. For sync compaction, schedule.
 *
 * Returns true if the lock is held.
 * Returns false if the lock is released and compaction should abort
 */
static bool compact_checklock_irqsave(spinlock_t *lock, unsigned long *flags,
				      bool locked, struct compact_control *cc)
{
	if (need_resched() || spin_is_contended(lock)) {
		if (locked) {
			spin_unlock_irqrestore(lock, *flags);
			locked = false;
		}

		/* async aborts if taking too long or contended */
		if (!cc->sync) {
			cc->contended = true;
			return false;
		}

		cond_resched();
	}

	if (!locked)
		spin_lock_irqsave(lock, *flags);
	return true;
}

static inline bool compact_trylock_irqsave(spinlock_t *lock,
			unsigned long *flags, struct compact_control *cc)
{
	return compact_checklock_irqsave(lock, flags, false, cc);
}

static void compact_capture_page(struct compact_control *cc)
{
	unsigned long flags;
	int mtype, mtype_low, mtype_high;

	if (!cc->page || *cc->page)
		return;

	/*
	 * For MIGRATE_MOVABLE allocations we capture a suitable page ASAP
	 * regardless of the migratetype of the freelist is is captured from.
	 * This is fine because the order for a high-order MIGRATE_MOVABLE
	 * allocation is typically at least a pageblock size and overall
	 * fragmentation is not impaired. Other allocation types must
	 * capture pages from their own migratelist because otherwise they
	 * could pollute other pageblocks like MIGRATE_MOVABLE with
	 * difficult to move pages and making fragmentation worse overall.
	 */
	if (cc->migratetype == MIGRATE_MOVABLE) {
		mtype_low = 0;
		mtype_high = MIGRATE_PCPTYPES;
	} else {
		mtype_low = cc->migratetype;
		mtype_high = cc->migratetype + 1;
	}

	/* Speculatively examine the free lists without zone lock */
	for (mtype = mtype_low; mtype < mtype_high; mtype++) {
		int order;
		for (order = cc->order; order < MAX_ORDER; order++) {
			struct page *page;
			struct free_area *area;
			area = &(cc->zone->free_area[order]);
			if (list_empty(&area->free_list[mtype]))
				continue;

			/* Take the lock and attempt capture of the page */
			if (!compact_trylock_irqsave(&cc->zone->lock, &flags, cc))
				return;
			if (!list_empty(&area->free_list[mtype])) {
				page = list_entry(area->free_list[mtype].next,
							struct page, lru);
				if (capture_free_page(page, cc