[PATCH v6 5/6] IB/isert: do not build a new isert_device for a device being removed
From: Yehyeong Lee
Date: Tue Nov 03 2026 - 17:40:53 EST
isert_device_get() looks the device up by node_guid and has no reason to
refuse one. A CONNECT_REQUEST arriving while isert_remove_one() runs
therefore builds a new isert_device on an ib_device the core is about to
free, and nothing waits for its QP, CQ or PD. cma_process_remove() waits
only for id destruction, and the core takes no device reference for them,
so ib_dealloc_device() can run while they are still being released.
Let the ib_client data say whether a device may be used. isert_add_one()
stores a non-NULL value, isert_remove_one() clears it under
device_list_mutex before anything else, and isert_device_get() refuses a
device whose data is NULL. The clear and the lookup are in that one
critical section, so no isert_device can appear for the device after the
clear. The lookup also skips an isert_device being removed, so a device
with the same node_guid cannot raise its reference count again.
A test-only delay before isert_device_get() opens the window. It opened in
three of fifteen runs of patches 1 to 4 of this series, and all three built
a second isert_device and took ten or eleven warnings from the core. It
opened in three of fifteen runs with this patch, and all three built one
device and took no warning.
WARNING: drivers/infiniband/core/cq.c:370 at ib_cq_pool_cleanup+0x189/0x250, CPU#0: rdma/202
Modules linked in:
CPU: 0 UID: 0 PID: 202 Comm: rdma Not tainted 7.4.0-rc1-NW4-00006-ga65579195c15 #125 PREEMPT(full)
Hardware name: QEMU Ubuntu 24.04 PC v2 (i440FX + PIIX, arch_caps fix, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
RIP: 0010:ib_cq_pool_cleanup+0x189/0x250
Code: 85 19 ff ff ff 48 8b 04 24 49 83 c4 10 49 39 c4 0f 85 cd fe ff ff 48 83 c4 18 5b 5d 41 5c 41 5d 41 5e 41 5f c3 cc cc cc cc 90 <0f> 0b 90 e9 20 ff ff ff 48 89 44 24 08 e8 15 16 0b fe 48 8b 44 24
RSP: 0018:ffff88810c737308 EFLAGS: 00010206
RAX: ffff88810a03fc40 RBX: ffff888104b5c000 RCX: 0000000000000007
RDX: 000000000000010c RSI: 0000000000000000 RDI: ffff888104b5c02c
RBP: dffffc0000000000 R08: 0000000000000001 R09: 00000000000000bf
R10: 0000000000000003 R11: 00000000d3012b20 R12: ffff8881047f0ad0
R13: dead000000000122 R14: dead000000000100 R15: ffff88810a03fc00
FS: 00007f8b74df1740(0000) GS:ffff888189e5e000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9e81f41000 CR3: 0000000105f2b002 CR4: 0000000000770ef0
PKRU: 55555554
Call Trace:
<TASK>
? remove_client_context+0xb7/0xf0
disable_device+0x13c/0x240
? __pfx_disable_device+0x10/0x10
? __pfx_mutex_lock+0x10/0x10
__ib_unregister_device+0x226/0x3f0
? __pfx_mutex_unlock+0x10/0x10
ib_unregister_device_and_put+0x50/0x70
nldev_dellink+0x28e/0x3d0
? kernel_text_address+0xa7/0x130
? __pfx_nldev_dellink+0x10/0x10
? __pfx_down_read+0x10/0x10
rdma_nl_rcv_msg+0x29c/0x4e0
? __pfx_rdma_nl_rcv_msg+0x10/0x10
rdma_nl_rcv_skb.constprop.0.isra.0+0x21d/0x370
? __pfx_rdma_nl_rcv_skb.constprop.0.isra.0+0x10/0x10
netlink_unicast+0x68f/0x940
? __pfx_selinux_socket_getpeersec_dgram+0x10/0x10
? __pfx_netlink_unicast+0x10/0x10
netlink_sendmsg+0x810/0xf20
? __pfx_netlink_sendmsg+0x10/0x10
? __pfx__raw_spin_lock_irqsave+0x10/0x10
? selinux_socket_sendmsg+0x8c/0x270
? __sys_sendto+0x337/0x390
__sys_sendto+0x337/0x390
? __pfx___sys_sendto+0x10/0x10
? fdget+0x53/0x3b0
__x64_sys_sendto+0xdb/0x1b0
? fpregs_assert_state_consistent+0x56/0xe0
do_syscall_64+0xdd/0x4a0
entry_SYSCALL_64_after_hwframe+0x77/0x7f
RIP: 0033:0x7f8b74f200d7
Code: c7 c0 ff ff ff ff eb be 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa 80 3d 75 ef 0d 00 00 41 89 ca 74 10 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 69 c3 55 48 89 e5 53 48 83 ec 38 44 89 4d d0
RSP: 002b:00007ffd34f4ef18 EFLAGS: 00000202 ORIG_RAX: 000000000000002c
RAX: ffffffffffffffda RBX: 0000556b747c4390 RCX: 00007f8b74f200d7
RDX: 0000000000000018 RSI: 0000556b747c32a0 RDI: 0000000000000004
RBP: 00007ffd34f4ef50 R08: 00007f8b75017200 R09: 000000000000000c
R10: 0000000000000000 R11: 0000000000000202 R12: 00007ffd34f4f140
R13: 00007ffd34f4ef24 R14: 00007ffd34f4f160 R15: 00007ffd34f4f140
</TASK>
Fixes: 63b268d232b8 ("IB/isert: Properly release resources on DEVICE_REMOVAL")
Cc: stable@xxxxxxxxxxxxxxx
Signed-off-by: Yehyeong Lee <yhlee@xxxxxxxxxxxxxxxxxx>
---
drivers/infiniband/ulp/isert/ib_isert.c | 35 ++++++++++++++++++++-----
1 file changed, 28 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 712d6165150e3c..2057bb35af4cac 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -41,6 +41,11 @@ module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644);
MODULE_PARM_DESC(sg_tablesize,
"Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)");
+static struct ib_client isert_ib_client;
+
+/* Non-NULL client data on a device means it may be used. */
+static char isert_device_enabled;
+
static DEFINE_MUTEX(device_list_mutex);
static LIST_HEAD(device_list);
static DEFINE_MUTEX(isert_conn_mutex);
@@ -275,14 +280,19 @@ isert_device_get(struct rdma_cm_id *cma_id)
int ret;
mutex_lock(&device_list_mutex);
+ if (!ib_get_client_data(cma_id->device, &isert_ib_client)) {
+ mutex_unlock(&device_list_mutex);
+ return ERR_PTR(-ENODEV);
+ }
list_for_each_entry(device, &device_list, dev_node) {
- if (device->ib_device->node_guid == cma_id->device->node_guid) {
- device->refcount++;
- isert_info("Found iser device %p refcount %d\n",
- device, device->refcount);
- mutex_unlock(&device_list_mutex);
- return device;
- }
+ if (device->removing ||
+ device->ib_device->node_guid != cma_id->device->node_guid)
+ continue;
+ device->refcount++;
+ isert_info("Found iser device %p refcount %d\n",
+ device, device->refcount);
+ mutex_unlock(&device_list_mutex);
+ return device;
}
device = kzalloc_obj(struct isert_device);
@@ -2791,6 +2801,7 @@ isert_remove_one(struct ib_device *ib_device, void *client_data)
LIST_HEAD(removal_list);
mutex_lock(&device_list_mutex);
+ ib_set_client_data(ib_device, &isert_ib_client, NULL);
list_for_each_entry(device, &device_list, dev_node) {
if (device->ib_device != ib_device)
continue;
@@ -2805,7 +2816,9 @@ isert_remove_one(struct ib_device *ib_device, void *client_data)
return;
mutex_lock(&isert_conn_mutex);
+ mutex_lock(&device_list_mutex);
found->removing = true;
+ mutex_unlock(&device_list_mutex);
list_for_each_entry(isert_conn, &isert_conn_list, conn_list) {
if (isert_conn->device != found)
continue;
@@ -2828,8 +2841,16 @@ isert_remove_one(struct ib_device *ib_device, void *client_data)
wait_for_completion(&removal_comp);
}
+static int isert_add_one(struct ib_device *ib_device)
+{
+ ib_set_client_data(ib_device, &isert_ib_client, &isert_device_enabled);
+
+ return 0;
+}
+
static struct ib_client isert_ib_client = {
.name = "isert",
+ .add = isert_add_one,
.remove = isert_remove_one,
};
--
2.43.0