[PATCH 1/2] io_uring: avoid activating drain for failed requests or links
From: Zhiqi Zhang
Date: Tue Nov 17 2026 - 03:50:53 EST
io_init_req() currently calls io_init_drain() and consumes
IO_RING_F_DRAIN_NEXT before request initialization and BPF filter
checks have completed. If io_init_req(), io_uring_run_bpf_filters(),
or a later request in the same link fails, io_submit_fail_init()
completes the request or link via io_req_defer_failed(), bypassing
io_drain_req().
This can leave IO_RING_F_DRAIN_ACTIVE or IO_RING_F_DRAIN_NEXT set on
the ring with no deferred drain request and ctx->drain_req == NULL:
1. P0 (e.g. POLL_ADD) is in flight
2. Invalid IOSQE_IO_DRAIN SQE sets IO_RING_F_DRAIN_ACTIVE, then fails
with -EINVAL and frees immediately
3. Subsequent non-drain SQE sees IO_RING_F_DRAIN_ACTIVE, is forced
through io_drain_req(), and blocks in ->defer_list behind P0
Similarly, a failed link containing an IOSQE_IO_DRAIN SQE leaks
IO_RING_F_DRAIN_ACTIVE and IO_RING_F_DRAIN_NEXT onto the next
unrelated request, or an -ENOMEM failure allocating io_defer_entry in
io_drain_req() fails the request with IO_RING_F_DRAIN_ACTIVE still set.
Do not touch IO_RING_F_DRAIN_ACTIVE or IO_RING_F_DRAIN_NEXT in
io_init_req(). During link assembly, only propagate REQ_F_IO_DRAIN to
a non-failing link head, and move drain flag activation into
io_drain_req() after struct io_defer_entry allocation succeeds. Also
clear state->link.head before flushing an unterminated link in
io_submit_state_end().
Fixes: 8fb7aee05591 ("io_uring: drain based on allocates reqs")
Cc: stable@xxxxxxxxxxxxxxx
Signed-off-by: Zhiqi Zhang <zhiqizhangg@xxxxxxxxx>
---
io_uring/io_uring.c | 100 +++++++++++++++++++++++++-------------------
1 file changed, 57 insertions(+), 43 deletions(-)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 5db57bf675f0..4e8b93a584b4 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1471,8 +1471,9 @@ static __cold void io_drain_req(struct io_kiocb *req)
__must_hold(&ctx->uring_lock)
{
struct io_ring_ctx *ctx = req->ctx;
- bool drain = req->flags & IOSQE_IO_DRAIN;
struct io_defer_entry *de;
+ bool drain_next = false;
+ bool drain;
de = kmalloc_obj(*de, GFP_KERNEL_ACCOUNT);
if (!de) {
@@ -1480,6 +1481,33 @@ static __cold void io_drain_req(struct io_kiocb *req)
return;
}
+ /*
+ * If a request in the middle or end of a link needs draining, drain
+ * the link head (propagated at link assembly) and the next request/link
+ * after the current link.
+ */
+ if (unlikely(req->link)) {
+ struct io_kiocb *cur;
+
+ io_for_each_link(cur, req->link) {
+ if (cur->flags & REQ_F_IO_DRAIN) {
+ drain_next = true;
+ break;
+ }
+ }
+ }
+ if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_NEXT)) {
+ ctx->int_flags &= ~IO_RING_F_DRAIN_NEXT;
+ ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
+ req->flags |= REQ_F_IO_DRAIN;
+ }
+ if (unlikely(req->flags & REQ_F_IO_DRAIN))
+ ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
+ if (unlikely(drain_next))
+ ctx->int_flags |= IO_RING_F_DRAIN_NEXT;
+ req->flags |= REQ_F_FORCE_ASYNC;
+ drain = req->flags & REQ_F_IO_DRAIN;
+
io_prep_async_link(req);
trace_io_uring_defer(req);
de->req = req;
@@ -1801,6 +1829,13 @@ static inline void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags)
io_queue_async(req, issue_flags, ret);
}
+static inline bool io_drain_needed(const struct io_ring_ctx *ctx,
+ const struct io_kiocb *req)
+{
+ return (ctx->int_flags & (IO_RING_F_DRAIN_ACTIVE | IO_RING_F_DRAIN_NEXT)) ||
+ (req->flags & REQ_F_IO_DRAIN);
+}
+
static void io_queue_sqe_fallback(struct io_kiocb *req)
__must_hold(&req->ctx->uring_lock)
{
@@ -1815,7 +1850,7 @@ static void io_queue_sqe_fallback(struct io_kiocb *req)
} else {
/* can't fail with IO_URING_F_INLINE */
io_req_sqe_copy(req, IO_URING_F_INLINE);
- if (unlikely(req->ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
+ if (unlikely(io_drain_needed(req->ctx, req)))
io_drain_req(req);
else
io_queue_iowq(req);
@@ -1847,24 +1882,6 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx,
return true;
}
-static void io_init_drain(struct io_ring_ctx *ctx)
-{
- struct io_kiocb *head = ctx->submit_state.link.head;
-
- ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
- if (head) {
- /*
- * If we need to drain a request in the middle of a link, drain
- * the head request and the next request/link after the current
- * link. Considering sequential execution of links,
- * REQ_F_IO_DRAIN will be maintained for every request of our
- * link.
- */
- head->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
- ctx->int_flags |= IO_RING_F_DRAIN_NEXT;
- }
-}
-
static __cold int io_init_fail_req(struct io_kiocb *req, int err)
{
/* ensure per-opcode data is cleared if we fail before prep */
@@ -1937,25 +1954,13 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
}
if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
ctx->int_flags |= IO_RING_F_DRAIN_DISABLED;
- if (sqe_flags & IOSQE_IO_DRAIN) {
- if (ctx->int_flags & IO_RING_F_DRAIN_DISABLED)
- return io_init_fail_req(req, -EOPNOTSUPP);
- io_init_drain(ctx);
- }
- }
- if (unlikely(ctx->int_flags & (IO_RING_F_OP_RESTRICTED | IO_RING_F_DRAIN_ACTIVE | IO_RING_F_DRAIN_NEXT))) {
- if (!io_check_restriction(ctx, req, sqe_flags))
- return io_init_fail_req(req, -EACCES);
- /* knock it to the slow queue path, will be drained there */
- if (ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)
- req->flags |= REQ_F_FORCE_ASYNC;
- /* if there is no link, we're at "next" request and need to drain */
- if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_NEXT) && !ctx->submit_state.link.head) {
- ctx->int_flags &= ~IO_RING_F_DRAIN_NEXT;
- ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
- req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
- }
+ if ((sqe_flags & IOSQE_IO_DRAIN) &&
+ (ctx->int_flags & IO_RING_F_DRAIN_DISABLED))
+ return io_init_fail_req(req, -EOPNOTSUPP);
}
+ if (unlikely(ctx->int_flags & IO_RING_F_OP_RESTRICTED) &&
+ !io_check_restriction(ctx, req, sqe_flags))
+ return io_init_fail_req(req, -EACCES);
if (!def->ioprio && sqe->ioprio)
return io_init_fail_req(req, -EINVAL);
@@ -2065,17 +2070,22 @@ static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
io_req_sqe_copy(req, IO_URING_F_INLINE);
link->last->link = req;
link->last = req;
+ if (unlikely((req->flags & REQ_F_IO_DRAIN) &&
+ !(link->head->flags & REQ_F_FAIL)))
+ link->head->flags |= REQ_F_IO_DRAIN;
if (req->flags & IO_REQ_LINK_FLAGS)
return 0;
/* last request of the link, flush it */
req = link->head;
link->head = NULL;
- if (req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL))
+ if ((req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)) ||
+ io_drain_needed(ctx, req))
goto fallback;
- } else if (unlikely(req->flags & (IO_REQ_LINK_FLAGS |
- REQ_F_FORCE_ASYNC | REQ_F_FAIL))) {
+ } else if (unlikely((req->flags & (IO_REQ_LINK_FLAGS | REQ_F_FORCE_ASYNC |
+ REQ_F_FAIL)) ||
+ io_drain_needed(ctx, req))) {
if (req->flags & IO_REQ_LINK_FLAGS) {
link->head = req;
link->last = req;
@@ -2097,8 +2107,12 @@ static void io_submit_state_end(struct io_ring_ctx *ctx)
{
struct io_submit_state *state = &ctx->submit_state;
- if (unlikely(state->link.head))
- io_queue_sqe_fallback(state->link.head);
+ if (unlikely(state->link.head)) {
+ struct io_kiocb *head = state->link.head;
+
+ state->link.head = NULL;
+ io_queue_sqe_fallback(head);
+ }
/* flush only after queuing links as they can generate completions */
io_submit_flush_completions(ctx);
if (state->plug_started)
base-commit: 666539eecdf4db623f5e3c3e55cf9f2dfad24aa4
--
2.43.0