aboutsummaryrefslogtreecommitdiff
path: root/arch/powerpc/platforms/cell/spufs/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/platforms/cell/spufs/sched.c')
-rw-r--r--arch/powerpc/platforms/cell/spufs/sched.c104
1 files changed, 76 insertions, 28 deletions
diff --git a/arch/powerpc/platforms/cell/spufs/sched.c b/arch/powerpc/platforms/cell/spufs/sched.c
index 34654743363..4a0a64fe25d 100644
--- a/arch/powerpc/platforms/cell/spufs/sched.c
+++ b/arch/powerpc/platforms/cell/spufs/sched.c
@@ -22,11 +22,12 @@
#undef DEBUG
-#include <linux/module.h>
#include <linux/errno.h>
#include <linux/sched.h>
+#include <linux/sched/rt.h>
#include <linux/kernel.h>
#include <linux/mm.h>
+#include <linux/slab.h>
#include <linux/completion.h>
#include <linux/vmalloc.h>
#include <linux/smp.h>
@@ -39,7 +40,6 @@
#include <linux/pid_namespace.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
-#include <linux/marker.h>
#include <asm/io.h>
#include <asm/mmu_context.h>
@@ -47,6 +47,8 @@
#include <asm/spu_csa.h>
#include <asm/spu_priv1.h>
#include "spufs.h"
+#define CREATE_TRACE_POINTS
+#include "sputrace.h"
struct spu_prio_array {
DECLARE_BITMAP(bitmap, MAX_PRIO);
@@ -81,7 +83,6 @@ static struct timer_list spuloadavg_timer;
#define MIN_SPU_TIMESLICE max(5 * HZ / (1000 * SPUSCHED_TICK), 1)
#define DEF_SPU_TIMESLICE (100 * HZ / (1000 * SPUSCHED_TICK))
-#define MAX_USER_PRIO (MAX_PRIO - MAX_RT_PRIO)
#define SCALE_PRIO(x, prio) \
max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_SPU_TIMESLICE)
@@ -139,7 +140,7 @@ void __spu_update_sched_info(struct spu_context *ctx)
* runqueue. The context will be rescheduled on the proper node
* if it is timesliced or preempted.
*/
- ctx->cpus_allowed = current->cpus_allowed;
+ cpumask_copy(&ctx->cpus_allowed, tsk_cpus_allowed(current));
/* Save the current cpu id for spu interrupt routing. */
ctx->last_ran = raw_smp_processor_id();
@@ -166,9 +167,9 @@ void spu_update_sched_info(struct spu_context *ctx)
static int __node_allowed(struct spu_context *ctx, int node)
{
if (nr_cpus_node(node)) {
- cpumask_t mask = node_to_cpumask(node);
+ const struct cpumask *mask = cpumask_of_node(node);
- if (cpus_intersects(mask, ctx->cpus_allowed))
+ if (cpumask_intersects(mask, &ctx->cpus_allowed))
return 1;
}
@@ -312,11 +313,35 @@ static struct spu *aff_ref_location(struct spu_context *ctx, int mem_aff,
*/
node = cpu_to_node(raw_smp_processor_id());
for (n = 0; n < MAX_NUMNODES; n++, node++) {
+ /*
+ * "available_spus" counts how many spus are not potentially
+ * going to be used by other affinity gangs whose reference
+ * context is already in place. Although this code seeks to
+ * avoid having affinity gangs with a summed amount of
+ * contexts bigger than the amount of spus in the node,
+ * this may happen sporadically. In this case, available_spus
+ * becomes negative, which is harmless.
+ */
+ int available_spus;
+
node = (node < MAX_NUMNODES) ? node : 0;
if (!node_allowed(ctx, node))
continue;
+
+ available_spus = 0;
mutex_lock(&cbe_spu_info[node].list_mutex);
list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) {
+ if (spu->ctx && spu->ctx->gang && !spu->ctx->aff_offset
+ && spu->ctx->gang->aff_ref_spu)
+ available_spus -= spu->ctx->gang->contexts;
+ available_spus++;
+ }
+ if (available_spus < ctx->gang->contexts) {
+ mutex_unlock(&cbe_spu_info[node].list_mutex);
+ continue;
+ }
+
+ list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) {
if ((!mem_aff || spu->has_mem_affinity) &&
sched_spu(spu)) {
mutex_unlock(&cbe_spu_info[node].list_mutex);
@@ -389,6 +414,9 @@ static int has_affinity(struct spu_context *ctx)
if (list_empty(&ctx->aff_list))
return 0;
+ if (atomic_read(&ctx->gang->aff_sched_count) == 0)
+ ctx->gang->aff_ref_spu = NULL;
+
if (!gang->aff_ref_spu) {
if (!(gang->aff_flags & AFF_MERGED))
aff_merge_remaining_ctxs(gang);
@@ -416,14 +444,13 @@ static void spu_unbind_context(struct spu *spu, struct spu_context *ctx)
if (spu->ctx->flags & SPU_CREATE_NOSCHED)
atomic_dec(&cbe_spu_info[spu->node].reserved_spus);
- if (ctx->gang){
- mutex_lock(&ctx->gang->aff_mutex);
- if (has_affinity(ctx)) {
- if (atomic_dec_and_test(&ctx->gang->aff_sched_count))
- ctx->gang->aff_ref_spu = NULL;
- }
- mutex_unlock(&ctx->gang->aff_mutex);
- }
+ if (ctx->gang)
+ /*
+ * If ctx->gang->aff_sched_count is positive, SPU affinity is
+ * being considered in this gang. Using atomic_dec_if_positive
+ * allow us to skip an explicit check for affinity in this gang
+ */
+ atomic_dec_if_positive(&ctx->gang->aff_sched_count);
spu_switch_notify(spu, NULL);
spu_unmap_mappings(ctx);
@@ -482,7 +509,7 @@ static void __spu_add_to_rq(struct spu_context *ctx)
list_add_tail(&ctx->rq, &spu_prio->runq[ctx->prio]);
set_bit(ctx->prio, spu_prio->bitmap);
if (!spu_prio->nr_waiting++)
- __mod_timer(&spusched_timer, jiffies + SPUSCHED_TICK);
+ mod_timer(&spusched_timer, jiffies + SPUSCHED_TICK);
}
}
@@ -562,10 +589,7 @@ static struct spu *spu_get_idle(struct spu_context *ctx)
goto found;
mutex_unlock(&cbe_spu_info[node].list_mutex);
- mutex_lock(&ctx->gang->aff_mutex);
- if (atomic_dec_and_test(&ctx->gang->aff_sched_count))
- ctx->gang->aff_ref_spu = NULL;
- mutex_unlock(&ctx->gang->aff_mutex);
+ atomic_dec(&ctx->gang->aff_sched_count);
goto not_found;
}
mutex_unlock(&ctx->gang->aff_mutex);
@@ -630,9 +654,12 @@ static struct spu *find_victim(struct spu_context *ctx)
if (tmp && tmp->prio > ctx->prio &&
!(tmp->flags & SPU_CREATE_NOSCHED) &&
- (!victim || tmp->prio > victim->prio))
+ (!victim || tmp->prio > victim->prio)) {
victim = spu->ctx;
+ }
}
+ if (victim)
+ get_spu_context(victim);
mutex_unlock(&cbe_spu_info[node].list_mutex);
if (victim) {
@@ -647,6 +674,7 @@ static struct spu *find_victim(struct spu_context *ctx)
* look at another context or give up after X retries.
*/
if (!mutex_trylock(&victim->state_mutex)) {
+ put_spu_context(victim);
victim = NULL;
goto restart;
}
@@ -659,6 +687,7 @@ static struct spu *find_victim(struct spu_context *ctx)
* restart the search.
*/
mutex_unlock(&victim->state_mutex);
+ put_spu_context(victim);
victim = NULL;
goto restart;
}
@@ -676,6 +705,7 @@ static struct spu *find_victim(struct spu_context *ctx)
spu_add_to_rq(victim);
mutex_unlock(&victim->state_mutex);
+ put_spu_context(victim);
return spu;
}
@@ -711,17 +741,33 @@ static void spu_schedule(struct spu *spu, struct spu_context *ctx)
/* not a candidate for interruptible because it's called either
from the scheduler thread or from spu_deactivate */
mutex_lock(&ctx->state_mutex);
- __spu_schedule(spu, ctx);
+ if (ctx->state == SPU_STATE_SAVED)
+ __spu_schedule(spu, ctx);
spu_release(ctx);
}
-static void spu_unschedule(struct spu *spu, struct spu_context *ctx)
+/**
+ * spu_unschedule - remove a context from a spu, and possibly release it.
+ * @spu: The SPU to unschedule from
+ * @ctx: The context currently scheduled on the SPU
+ * @free_spu Whether to free the SPU for other contexts
+ *
+ * Unbinds the context @ctx from the SPU @spu. If @free_spu is non-zero, the
+ * SPU is made available for other contexts (ie, may be returned by
+ * spu_get_idle). If this is zero, the caller is expected to schedule another
+ * context to this spu.
+ *
+ * Should be called with ctx->state_mutex held.
+ */
+static void spu_unschedule(struct spu *spu, struct spu_context *ctx,
+ int free_spu)
{
int node = spu->node;
mutex_lock(&cbe_spu_info[node].list_mutex);
cbe_spu_info[node].nr_active--;
- spu->alloc_state = SPU_FREE;
+ if (free_spu)
+ spu->alloc_state = SPU_FREE;
spu_unbind_context(spu, ctx);
ctx->stats.invol_ctx_switch++;
spu->stats.invol_ctx_switch++;
@@ -799,7 +845,7 @@ static struct spu_context *grab_runnable_context(int prio, int node)
struct list_head *rq = &spu_prio->runq[best];
list_for_each_entry(ctx, rq, rq) {
- /* XXX(hch): check for affinity here aswell */
+ /* XXX(hch): check for affinity here as well */
if (__node_allowed(ctx, node)) {
__spu_del_from_rq(ctx);
goto found;
@@ -821,7 +867,7 @@ static int __spu_deactivate(struct spu_context *ctx, int force, int max_prio)
if (spu) {
new = grab_runnable_context(max_prio, spu->node);
if (new || force) {
- spu_unschedule(spu, ctx);
+ spu_unschedule(spu, ctx, new == NULL);
if (new) {
if (new->flags & SPU_CREATE_NOSCHED)
wake_up(&new->stop_wq);
@@ -894,7 +940,7 @@ static noinline void spusched_tick(struct spu_context *ctx)
new = grab_runnable_context(ctx->prio + 1, spu->node);
if (new) {
- spu_unschedule(spu, ctx);
+ spu_unschedule(spu, ctx, 0);
if (test_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags))
spu_add_to_rq(ctx);
} else {
@@ -974,9 +1020,11 @@ static int spusched_thread(void *unused)
struct spu_context *ctx = spu->ctx;
if (ctx) {
+ get_spu_context(ctx);
mutex_unlock(mtx);
spusched_tick(ctx);
mutex_lock(mtx);
+ put_spu_context(ctx);
}
}
mutex_unlock(mtx);
@@ -1019,7 +1067,7 @@ void spuctx_switch_state(struct spu_context *ctx,
node = spu->node;
if (old_state == SPU_UTIL_USER)
atomic_dec(&cbe_spu_info[node].busy_spus);
- if (new_state == SPU_UTIL_USER);
+ if (new_state == SPU_UTIL_USER)
atomic_inc(&cbe_spu_info[node].busy_spus);
}
}
@@ -1046,7 +1094,7 @@ static int show_spu_loadavg(struct seq_file *s, void *private)
LOAD_INT(c), LOAD_FRAC(c),
count_active_contexts(),
atomic_read(&nr_spu_contexts),
- current->nsproxy->pid_ns->last_pid);
+ task_active_pid_ns(current)->last_pid);
return 0;
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-12-17 08:19:50 +0000