path: root/kernel/sched_rt.c

/*
 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
 * policies)
 */

#ifdef CONFIG_SMP

static inline int rt_overloaded(struct rq *rq)
{
	return atomic_read(&rq->rd->rto_count);
}

static inline void rt_set_overload(struct rq *rq)
{
	cpu_set(rq->cpu, rq->rd->rto_mask);
	/*
	 * Make sure the mask is visible before we set
	 * the overload count. That is checked to determine
	 * if we should look at the mask. It would be a shame
	 * if we looked at the mask, but the mask was not
	 * updated yet.
	 */
	wmb();
	atomic_inc(&rq->rd->rto_count);
}

static inline void rt_clear_overload(struct rq *rq)
{
	/* the order here really doesn't matter */
	atomic_dec(&rq->rd->rto_count);
	cpu_clear(rq->cpu, rq->rd->rto_mask);
}

static void update_rt_migration(struct rq *rq)
{
	if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) {
		if (!rq->rt.overloaded) {
			rt_set_overload(rq);
			rq->rt.overloaded = 1;
		}
	} else if (rq->rt.overloaded) {
		rt_clear_overload(rq);
		rq->rt.overloaded = 0;
	}
}
#endif /* CONFIG_SMP */

static int sched_rt_ratio_exceeded(struct rq *rq, struct rt_rq *rt_rq)
{
	u64 period, ratio;

	if (sysctl_sched_rt_ratio == SCHED_RT_FRAC)
		return 0;

	if (rt_rq->rt_throttled)
		return 1;

	period = (u64)sysctl_sched_rt_period * NSEC_PER_MSEC;
	ratio = (period * sysctl_sched_rt_ratio) >> SCHED_RT_FRAC_SHIFT;

	if (rt_rq->rt_time > ratio) {
		rt_rq->rt_throttled = rq->clock + period - rt_rq->rt_time;
		return 1;
	}

	return 0;
}

static void update_sched_rt_period(struct rq *rq)
{
	while (rq->clock > rq->rt_period_expire) {
		u64 period, ratio;

		period = (u64)sysctl_sched_rt_period * NSEC_PER_MSEC;
		ratio = (period * sysctl_sched_rt_ratio) >> SCHED_RT_FRAC_SHIFT;

		rq->rt.rt_time -= min(rq->rt.rt_time, ratio);
		rq->rt_period_expire += period;
	}

	/*
	 * When the rt throttle is expired, let them rip.
	 * (XXX: use hrtick when available)
	 */
	if (rq->rt.rt_throttled && rq->clock > rq->rt.rt_throttled) {
		rq->rt.rt_throttled = 0;
		if (!sched_rt_ratio_exceeded(rq, &rq->rt))
			resched_task(rq->curr);
	}
}

/*
 * Update the current task's runtime statistics. Skip current tasks that
 * are not in our scheduling class.
 */
static void update_curr_rt(struct rq *rq)
{
	struct task_struct *curr = rq->curr;
	u64 delta_exec;

	if (!task_has_rt_policy(curr))
		return;

	delta_exec = rq->clock - curr->se.exec_start;
	if (unlikely((s64)delta_exec < 0))
		delta_exec = 0;

	schedstat_set(curr->se.exec_max, max(curr->se.exec_max, delta_exec));

	curr->se.sum_exec_runtime += delta_exec;
	curr->se.exec_start = rq->clock;
	cpuacct_charge(curr, delta_exec);

	rq->rt.rt_time += delta_exec;
	update_sched_rt_period(rq);
	if (sched_rt_ratio_exceeded(rq, &rq->rt))
		resched_task(curr);
}

static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq)
{
	WARN_ON(!rt_task(p));
	rq->rt.rt_nr_running++;
#ifdef CONFIG_SMP
	if (p->prio < rq->rt.highest_prio)
		rq->rt.highest_prio = p->prio;
	if (p->nr_cpus_allowed > 1)
		rq->rt.rt_nr_migratory++;

	update_rt_migration(rq);
#endif /* CONFIG_SMP */
}

static inline void dec_rt_tasks(struct task_struct *p, struct rq *rq)
{
	WARN_ON(!rt_task(p));
	WARN_ON(!rq->rt.rt_nr_running);
	rq->rt.rt_nr_running--;
#ifdef CONFIG_SMP
	if (rq->rt.rt_nr_running) {
		struct rt_prio_array