path: root/kernel/sched_fair.c

/*
 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
 *
 *  Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 *
 *  Interactivity improvements by Mike Galbraith
 *  (C) 2007 Mike Galbraith <efault@gmx.de>
 *
 *  Various enhancements by Dmitry Adamushko.
 *  (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
 *
 *  Group scheduling enhancements by Srivatsa Vaddagiri
 *  Copyright IBM Corporation, 2007
 *  Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
 *
 *  Scaled math optimizations by Thomas Gleixner
 *  Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
 *
 *  Adaptive scheduling granularity, math enhancements by Peter Zijlstra
 *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
 */

/*
 * Targeted preemption latency for CPU-bound tasks:
 * (default: 20ms, units: nanoseconds)
 *
 * NOTE: this latency value is not the same as the concept of
 * 'timeslice length' - timeslices in CFS are of variable length.
 * (to see the precise effective timeslice length of your workload,
 *  run vmstat and monitor the context-switches field)
 *
 * On SMP systems the value of this is multiplied by the log2 of the
 * number of CPUs. (i.e. factor 2x on 2-way systems, 3x on 4-way
 * systems, 4x on 8-way systems, 5x on 16-way systems, etc.)
 * Targeted preemption latency for CPU-bound tasks:
 */
unsigned int sysctl_sched_latency __read_mostly = 20000000ULL;

/*
 * Minimal preemption granularity for CPU-bound tasks:
 * (default: 2 msec, units: nanoseconds)
 */
unsigned int sysctl_sched_min_granularity __read_mostly = 2000000ULL;

/*
 * SCHED_BATCH wake-up granularity.
 * (default: 25 msec, units: nanoseconds)
 *
 * This option delays the preemption effects of decoupled workloads
 * and reduces their over-scheduling. Synchronous workloads will still
 * have immediate wakeup/sleep latencies.
 */
unsigned int sysctl_sched_batch_wakeup_granularity __read_mostly = 25000000UL;

/*
 * SCHED_OTHER wake-up granularity.
 * (default: 1 msec, units: nanoseconds)
 *
 * This option delays the preemption effects of decoupled workloads
 * and reduces their over-scheduling. Synchronous workloads will still
 * have immediate wakeup/sleep latencies.
 */
unsigned int sysctl_sched_wakeup_granularity __read_mostly = 1000000UL;

unsigned int sysctl_sched_stat_granularity __read_mostly;

/*
 * Initialized in sched_init_granularity() [to 5 times the base granularity]:
 */
unsigned int sysctl_sched_runtime_limit __read_mostly;

/*
 * Debugging: various feature bits
 */
enum {
	SCHED_FEAT_FAIR_SLEEPERS	= 1,
	SCHED_FEAT_SLEEPER_AVG		= 2,
	SCHED_FEAT_SLEEPER_LOAD_AVG	= 4,
	SCHED_FEAT_PRECISE_CPU_LOAD	= 8,
	SCHED_FEAT_START_DEBIT		= 16,
	SCHED_FEAT_SKIP_INITIAL		= 32,
};

unsigned int sysctl_sched_features __read_mostly =
		SCHED_FEAT_FAIR_SLEEPERS	*1 |
		SCHED_FEAT_SLEEPER_AVG		*0 |
		SCHED_FEAT_SLEEPER_LOAD_AVG	*1 |
		SCHED_FEAT_PRECISE_CPU_LOAD	*1 |
		SCHED_FEAT_START_DEBIT		*1 |
		SCHED_FEAT_SKIP_INITIAL		*0;

extern struct sched_class fair_sched_class;

/**************************************************************
 * CFS operations on generic schedulable entities:
 */

#ifdef CONFIG_FAIR_GROUP_SCHED

/* cpu runqueue to which this cfs_rq is attached */
static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
{
	return cfs_rq->rq;
}

/* currently running entity (if any) on this cfs_rq */
static inline struct sched_entity *cfs_rq_curr(struct cfs_rq *cfs_rq)
{
	return cfs_rq->curr;
}

/* An entity is a task if it doesn't "own" a runqueue */
#define entity_is_task(se)	(!se->my_q)

static inline void
set_cfs_rq_curr(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
	cfs_rq->curr = se;
}

#else	/* CONFIG_FAIR_GROUP_SCHED */

static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
{
	return container_of(cfs_rq, struct rq, cfs);
}

static inline struct sched_entity *cfs_rq_curr(struct cfs_rq *cfs_rq)
{
	struct rq *rq = rq_of(cfs_rq);

	if (unlikely(rq->curr->sched_class != &fair_sched_class))
		return NULL;

	return &rq->curr->se;
}

#define entity_is_task(se)	1

static inline void
set_cfs_rq_curr(struct cfs_rq *cfs_rq, struct sched_entity *se) { }

#endif	/* CONFIG_FAIR_GROUP_SCHED */

static inline struct task_struct *task_of(struct sched_entity *se)
{
	return container_of(se, struct task_struct, se);
}


/**************************************************************
 * Scheduling class tree data structure manipulation methods:
 */

/*
 * Enqueue an entity into the rb-tree:
 */
static inline void
__enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)