[PATCH 07/15] io_uring: add uring_lock section depth tracking and blockable opdef flag
From: Jens Axboe
Date: Fri Sep 11 2026 - 12:14:44 EST
Prep patch for issuing requests inline in blocking mode and catching the
sleep when it happens, rather than punting to io-wq upfront because an
operation may block.
If a request blocks inline, uring_lock must be dropped on behalf of the
sleeping task, which is only safe outside the sections that rely on the
lock being held. Track those with a depth counter in
io_ring_submit_lock() and io_ring_submit_unlock().
Add a "blockable" flag to io_issue_def for opcodes whose issue path can
cope with blocking inline: read/write, the forced async fs ops, open,
close and splice/tee. uring_cmd is excluded for now, drivers may bind
state to the submitting task.
No functional changes in this patch.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
---
include/linux/io_uring_types.h | 5 +++++
io_uring/io_uring.h | 3 +++
io_uring/opdef.c | 29 +++++++++++++++++++++++++++++
io_uring/opdef.h | 2 ++
4 files changed, 39 insertions(+)
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 4af3d579ead6..50a4a0ad222f 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -352,6 +352,11 @@ struct io_ring_ctx {
/* submission data */
struct {
struct mutex uring_lock;
+ /*
+ * io_ring_submit_lock() nesting depth, non-zero means the
+ * issue path relies on the lock being held.
+ */
+ unsigned int submit_lock_depth;
/*
* Ring buffer of indices into array of io_uring_sqe, which is
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 896aab1ed026..870bb4dcc415 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -393,6 +393,8 @@ static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx,
unsigned issue_flags)
{
lockdep_assert_held(&ctx->uring_lock);
+ lockdep_assert(ctx->submit_lock_depth > 0);
+ ctx->submit_lock_depth--;
if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
mutex_unlock(&ctx->uring_lock);
}
@@ -409,6 +411,7 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx,
if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
mutex_lock(&ctx->uring_lock);
lockdep_assert_held(&ctx->uring_lock);
+ ctx->submit_lock_depth++;
}
static inline void io_commit_cqring(struct io_ring_ctx *ctx)
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index cf3aa2242cd7..fa07a2b94536 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -69,6 +69,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_readv,
.issue = io_read,
},
@@ -83,12 +84,14 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_writev,
.issue = io_write,
},
[IORING_OP_FSYNC] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_fsync_prep,
.issue = io_fsync,
},
@@ -101,6 +104,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_read_fixed,
.issue = io_read_fixed,
},
@@ -114,6 +118,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_write_fixed,
.issue = io_write_fixed,
},
@@ -132,6 +137,7 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_SYNC_FILE_RANGE] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_sfr_prep,
.issue = io_sync_file_range,
},
@@ -215,16 +221,19 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_FALLOCATE] = {
.needs_file = 1,
.hash_reg_file = 1,
+ .blockable = 1,
.prep = io_fallocate_prep,
.issue = io_fallocate,
},
[IORING_OP_OPENAT] = {
.filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
+ .blockable = 1,
.prep = io_openat_prep,
.issue = io_openat,
.filter_populate = io_openat_bpf_populate,
},
[IORING_OP_CLOSE] = {
+ .blockable = 1,
.prep = io_close_prep,
.issue = io_close,
},
@@ -236,6 +245,7 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_STATX] = {
.audit_skip = 1,
+ .blockable = 1,
.prep = io_statx_prep,
.issue = io_statx,
},
@@ -249,6 +259,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_read,
.issue = io_read,
},
@@ -262,17 +273,20 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_write,
.issue = io_write,
},
[IORING_OP_FADVISE] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_fadvise_prep,
.issue = io_fadvise,
},
[IORING_OP_MADVISE] = {
.audit_skip = 1,
+ .blockable = 1,
.prep = io_madvise_prep,
.issue = io_madvise,
},
@@ -308,6 +322,7 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_OPENAT2] = {
.filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
+ .blockable = 1,
.prep = io_openat2_prep,
.issue = io_openat2,
.filter_populate = io_openat_bpf_populate,
@@ -327,6 +342,7 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_splice_prep,
.issue = io_splice,
},
@@ -347,6 +363,7 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_tee_prep,
.issue = io_tee,
},
@@ -360,22 +377,27 @@ const struct io_issue_def io_issue_defs[] = {
#endif
},
[IORING_OP_RENAMEAT] = {
+ .blockable = 1,
.prep = io_renameat_prep,
.issue = io_renameat,
},
[IORING_OP_UNLINKAT] = {
+ .blockable = 1,
.prep = io_unlinkat_prep,
.issue = io_unlinkat,
},
[IORING_OP_MKDIRAT] = {
+ .blockable = 1,
.prep = io_mkdirat_prep,
.issue = io_mkdirat,
},
[IORING_OP_SYMLINKAT] = {
+ .blockable = 1,
.prep = io_symlinkat_prep,
.issue = io_symlinkat,
},
[IORING_OP_LINKAT] = {
+ .blockable = 1,
.prep = io_linkat_prep,
.issue = io_linkat,
},
@@ -387,19 +409,23 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_FSETXATTR] = {
.needs_file = 1,
+ .blockable = 1,
.prep = io_fsetxattr_prep,
.issue = io_fsetxattr,
},
[IORING_OP_SETXATTR] = {
+ .blockable = 1,
.prep = io_setxattr_prep,
.issue = io_setxattr,
},
[IORING_OP_FGETXATTR] = {
.needs_file = 1,
+ .blockable = 1,
.prep = io_fgetxattr_prep,
.issue = io_fgetxattr,
},
[IORING_OP_GETXATTR] = {
+ .blockable = 1,
.prep = io_getxattr_prep,
.issue = io_getxattr,
},
@@ -497,6 +523,7 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_FTRUNCATE] = {
.needs_file = 1,
.hash_reg_file = 1,
+ .blockable = 1,
.prep = io_ftruncate_prep,
.issue = io_ftruncate,
},
@@ -553,6 +580,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_readv_fixed,
.issue = io_read,
},
@@ -567,6 +595,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_writev_fixed,
.issue = io_write,
},
diff --git a/io_uring/opdef.h b/io_uring/opdef.h
index 667f981e63b0..45c2f77cf782 100644
--- a/io_uring/opdef.h
+++ b/io_uring/opdef.h
@@ -29,6 +29,8 @@ struct io_issue_def {
unsigned vectored : 1;
/* set to 1 if this opcode uses 128b sqes in a mixed sq */
unsigned is_128 : 1;
+ /* issue path is safe to run inline in blocking mode */
+ unsigned blockable : 1;
/* size of async data needed, if any */
unsigned short async_size;
--
2.55.0