[PATCH 6/9] sched_ext: defer WARN console output under rq->lock
From: Rik van Riel
Date: Wed Jun 10 2026 - 22:24:31 EST
Convert the WARN*() calls that run under rq->lock or ->pi_lock to the
SCHED_WARN*() variants, so their console output is deferred to irq_work
instead of being emitted synchronously (which can deadlock via
console_unlock() -> up(&console_sem) -> try_to_wake_up() while the lock
is held).
This should prevent a deadlock if these warnings fire with a legacy
or boot console configured.
Signed-off-by: Rik van Riel <riel@xxxxxxxxxxx>
Assisted-by: Claude:claude-opus-4-8
---
kernel/sched/ext.c | 98 +++++++++++++++++++++++-----------------------
1 file changed, 49 insertions(+), 49 deletions(-)
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index f412c4bb21c3..149ca9920071 100644
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -518,7 +518,7 @@ do { \
*/
#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
do { \
- WARN_ON_ONCE(current->scx.kf_tasks[0]); \
+ SCHED_WARN_ON_ONCE(current->scx.kf_tasks[0]); \
current->scx.kf_tasks[0] = task; \
SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
current->scx.kf_tasks[0] = NULL; \
@@ -527,7 +527,7 @@ do { \
#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
({ \
__typeof__((sch)->ops.op(task, ##args)) __ret; \
- WARN_ON_ONCE(current->scx.kf_tasks[0]); \
+ SCHED_WARN_ON_ONCE(current->scx.kf_tasks[0]); \
current->scx.kf_tasks[0] = task; \
__ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
current->scx.kf_tasks[0] = NULL; \
@@ -537,7 +537,7 @@ do { \
#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
({ \
__typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
- WARN_ON_ONCE(current->scx.kf_tasks[0]); \
+ SCHED_WARN_ON_ONCE(current->scx.kf_tasks[0]); \
current->scx.kf_tasks[0] = task0; \
current->scx.kf_tasks[1] = task1; \
__ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
@@ -688,7 +688,7 @@ static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor,
return true;
/* if @p has stayed on @dsq, its rq couldn't have changed */
- if (WARN_ON_ONCE(rq != task_rq(p)))
+ if (SCHED_WARN_ON_ONCE(rq != task_rq(p)))
return true;
return false;
@@ -1225,7 +1225,7 @@ static void schedule_reenq_local(struct rq *rq, u64 reenq_flags)
{
struct scx_sched *root = rcu_dereference_sched(scx_root);
- if (WARN_ON_ONCE(!root))
+ if (SCHED_WARN_ON_ONCE(!root))
return;
schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq);
@@ -1322,7 +1322,7 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en
*/
if (enq_flags & SCX_ENQ_IMMED) {
if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
- WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK));
+ SCHED_WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK));
return;
}
p->scx.flags |= SCX_TASK_IMMED;
@@ -1331,7 +1331,7 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en
if (p->scx.flags & SCX_TASK_IMMED) {
struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
- if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
+ if (SCHED_WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
return;
rq->scx.nr_immed++;
@@ -1353,8 +1353,8 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p)
if (p->scx.flags & SCX_TASK_IMMED) {
struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
- if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) ||
- WARN_ON_ONCE(rq->scx.nr_immed <= 0))
+ if (SCHED_WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) ||
+ SCHED_WARN_ON_ONCE(rq->scx.nr_immed <= 0))
return;
rq->scx.nr_immed--;
@@ -1464,8 +1464,8 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
{
bool is_local = dsq->id == SCX_DSQ_LOCAL;
- WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
- WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
+ SCHED_WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
+ SCHED_WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
!RB_EMPTY_NODE(&p->scx.dsq_priq));
if (!is_local) {
@@ -1589,7 +1589,7 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
static void task_unlink_from_dsq(struct task_struct *p,
struct scx_dispatch_q *dsq)
{
- WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
+ SCHED_WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) {
rb_erase(&p->scx.dsq_priq, &dsq->priq);
@@ -1652,7 +1652,7 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p)
* holding_cpu which tells dispatch_to_local_dsq() that it lost
* the race.
*/
- WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
+ SCHED_WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
p->scx.holding_cpu = -1;
}
p->scx.dsq = NULL;
@@ -1730,8 +1730,8 @@ static void mark_direct_dispatch(struct scx_sched *sch,
return;
}
- WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID);
- WARN_ON_ONCE(p->scx.ddsp_enq_flags);
+ SCHED_WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID);
+ SCHED_WARN_ON_ONCE(p->scx.ddsp_enq_flags);
p->scx.ddsp_dsq_id = dsq_id;
p->scx.ddsp_enq_flags = enq_flags;
@@ -1792,13 +1792,13 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
break;
default:
- WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()",
+ SCHED_WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()",
p->comm, p->pid, opss);
atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
break;
}
- WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
+ SCHED_WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
list_add_tail(&p->scx.dsq_list.node,
&rq->scx.ddsp_deferred_locals);
schedule_deferred_locked(rq);
@@ -1831,7 +1831,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
struct scx_dispatch_q *dsq;
unsigned long qseq;
- WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
+ SCHED_WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
/* internal movements - rq migration / RESTORE */
if (sticky_cpu == cpu_of(rq))
@@ -1881,11 +1881,11 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
/* DSQ bypass didn't trigger, enqueue on the BPF scheduler */
qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
- WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
+ SCHED_WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq);
ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
- WARN_ON_ONCE(*ddsp_taskp);
+ SCHED_WARN_ON_ONCE(*ddsp_taskp);
*ddsp_taskp = p;
SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags);
@@ -1982,7 +1982,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
sticky_cpu = cpu_of(rq);
if (p->scx.flags & SCX_TASK_QUEUED) {
- WARN_ON_ONCE(!task_runnable(p));
+ SCHED_WARN_ON_ONCE(!task_runnable(p));
goto out;
}
@@ -2035,7 +2035,7 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
BUG();
case SCX_OPSS_QUEUED:
/* A queued task must always be in BPF scheduler's custody */
- WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_IN_CUSTODY));
+ SCHED_WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_IN_CUSTODY));
if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
SCX_OPSS_NONE))
break;
@@ -2089,7 +2089,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
deq_flags |= SCX_DEQ_SCHED_CHANGE;
if (!(p->scx.flags & SCX_TASK_QUEUED)) {
- WARN_ON_ONCE(task_runnable(p));
+ SCHED_WARN_ON_ONCE(task_runnable(p));
return true;
}
@@ -2186,7 +2186,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch,
lockdep_assert_held(&src_dsq->lock);
lockdep_assert_rq_held(dst_rq);
- WARN_ON_ONCE(p->scx.holding_cpu >= 0);
+ SCHED_WARN_ON_ONCE(p->scx.holding_cpu >= 0);
if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
list_add(&p->scx.dsq_list.node, &dst_dsq->list);
@@ -2229,8 +2229,8 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
* truncate the upper 32 bit. As we own @rq, we can pass them through
* @rq->scx.extra_enq_flags instead.
*/
- WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr));
- WARN_ON_ONCE(dst_rq->scx.extra_enq_flags);
+ SCHED_WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr));
+ SCHED_WARN_ON_ONCE(dst_rq->scx.extra_enq_flags);
dst_rq->scx.extra_enq_flags = enq_flags;
activate_task(dst_rq, p, 0);
dst_rq->scx.extra_enq_flags = 0;
@@ -2261,7 +2261,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
{
s32 cpu = cpu_of(rq);
- WARN_ON_ONCE(task_cpu(p) == cpu);
+ SCHED_WARN_ON_ONCE(task_cpu(p) == cpu);
/*
* If @p has migration disabled, @p->cpus_ptr is updated to contain only
@@ -2341,7 +2341,7 @@ static bool unlink_dsq_and_lock_src_rq(struct task_struct *p,
lockdep_assert_held(&dsq->lock);
- WARN_ON_ONCE(p->scx.holding_cpu >= 0);
+ SCHED_WARN_ON_ONCE(p->scx.holding_cpu >= 0);
task_unlink_from_dsq(p, dsq);
p->scx.holding_cpu = cpu;
@@ -2350,7 +2350,7 @@ static bool unlink_dsq_and_lock_src_rq(struct task_struct *p,
/* task_rq couldn't have changed if we're still the holding cpu */
return likely(p->scx.holding_cpu == cpu) &&
- !WARN_ON_ONCE(src_rq != task_rq(p));
+ !SCHED_WARN_ON_ONCE(src_rq != task_rq(p));
}
static bool consume_remote_task(struct rq *this_rq,
@@ -2560,7 +2560,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
/* task_rq couldn't have changed if we're still the holding cpu */
if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
- !WARN_ON_ONCE(src_rq != task_rq(p))) {
+ !SCHED_WARN_ON_ONCE(src_rq != task_rq(p))) {
/*
* If @p is staying on the same rq, there's no need to go
* through the full deactivate/activate cycle. Optimize by
@@ -3029,7 +3029,7 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
* which should trigger an explicit follow-up scheduling event.
*/
if (next && sched_class_above(&ext_sched_class, next->sched_class)) {
- WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST));
+ SCHED_WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST));
do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
} else {
do_enqueue_task(rq, p, 0, -1);
@@ -3131,7 +3131,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP;
if (unlikely(keep_prev &&
prev->sched_class != &ext_sched_class)) {
- WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED);
+ SCHED_WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED);
keep_prev = false;
}
@@ -3262,7 +3262,7 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
struct task_struct **ddsp_taskp;
ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
- WARN_ON_ONCE(*ddsp_taskp);
+ SCHED_WARN_ON_ONCE(*ddsp_taskp);
*ddsp_taskp = p;
this_rq()->scx.in_select_cpu = true;
@@ -3510,12 +3510,12 @@ static void scx_set_task_state(struct task_struct *p, u32 state)
warn = prev_state != SCX_TASK_READY;
break;
default:
- WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]",
+ SCHED_WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]",
prev_state, state, p->comm, p->pid);
return;
}
- WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]",
+ SCHED_WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]",
prev_state, state, p->comm, p->pid);
p->scx.flags &= ~SCX_TASK_STATE_MASK;
@@ -3601,7 +3601,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)
* transitions are consistent, the flag should always be clear
* here.
*/
- WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
+ SCHED_WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
/*
* Set the weight before calling ops.enable() so that the scheduler
@@ -3632,7 +3632,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
struct rq *rq = task_rq(p);
lockdep_assert_rq_held(rq);
- WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED);
+ SCHED_WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED);
clear_direct_dispatch(p);
@@ -3645,7 +3645,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
* transitions are consistent, the flag should always be clear
* here.
*/
- WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
+ SCHED_WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
}
static void __scx_disable_and_exit_task(struct scx_sched *sch,
@@ -3670,7 +3670,7 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch,
scx_disable_task(sch, p);
break;
default:
- WARN_ON_ONCE(true);
+ SCHED_WARN_ON_ONCE(true);
return;
}
@@ -3706,7 +3706,7 @@ static void scx_disable_and_exit_task(struct scx_sched *sch,
* it, so undo only init_task.
*/
if (p->scx.flags & SCX_TASK_SUB_INIT) {
- if (!WARN_ON_ONCE(!scx_enabling_sub_sched))
+ if (!SCHED_WARN_ON_ONCE(!scx_enabling_sub_sched))
scx_sub_init_cancel_task(scx_enabling_sub_sched, p);
p->scx.flags &= ~SCX_TASK_SUB_INIT;
}
@@ -3794,7 +3794,7 @@ void scx_cancel_fork(struct task_struct *p)
struct rq_flags rf;
rq = task_rq_lock(p, &rf);
- WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY);
+ SCHED_WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY);
scx_disable_and_exit_task(scx_task_sched(p), p);
task_rq_unlock(rq, p, &rf);
}
@@ -3941,7 +3941,7 @@ static void process_ddsp_deferred_locals(struct rq *rq)
clear_direct_dispatch(p);
dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
- if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
+ if (!SCHED_WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags);
}
}
@@ -3996,7 +3996,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
lockdep_assert_rq_held(rq);
- if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK))
+ if (SCHED_WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK))
reenq_flags &= ~__SCX_REENQ_TSR_MASK;
if (rq_is_open(rq, 0))
reenq_flags |= SCX_REENQ_TSR_RQ_OPEN;
@@ -4033,7 +4033,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
dispatch_dequeue(rq, p);
- if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
+ if (SCHED_WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
p->scx.flags |= reason;
@@ -4154,7 +4154,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
dispatch_dequeue_locked(p, dsq);
raw_spin_unlock(&dsq->lock);
- if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
+ if (SCHED_WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
p->scx.flags |= reason;
@@ -4361,7 +4361,7 @@ void scx_cgroup_move_task(struct task_struct *p)
* cgrp_moving_from set.
*/
if (SCX_HAS_OP(sch, cgroup_move) &&
- !WARN_ON_ONCE(!p->scx.cgrp_moving_from))
+ !SCHED_WARN_ON_ONCE(!p->scx.cgrp_moving_from))
SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p),
p, p->scx.cgrp_moving_from,
tg_cgrp(task_group(p)));
@@ -5711,7 +5711,7 @@ static void scx_sub_disable(struct scx_sched *sch)
* By the time control reaches here, all descendant schedulers
* should already have been disabled.
*/
- WARN_ON_ONCE(!scx_task_on_sched(sch, p));
+ SCHED_WARN_ON_ONCE(!scx_task_on_sched(sch, p));
/*
* If $p is about to be freed, nothing prevents $sch from
@@ -5913,7 +5913,7 @@ static void scx_root_disable(struct scx_sched *sch)
scoped_guard(rq_lock_irqsave, rq) {
update_rq_clock(rq);
if (was_switched_all) {
- if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server,
+ if (SCHED_WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server,
&rq->fair_server)))
pr_warn("failed to re-attach fair_server on CPU %d\n", cpu);
} else {
@@ -7068,7 +7068,7 @@ static bool assert_task_ready_or_enabled(struct task_struct *p)
case SCX_TASK_ENABLED:
return true;
default:
- WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched",
+ SCHED_WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched",
state, p->comm, p->pid);
return false;
}
--
2.53.0-Meta