[PATCH net V3 4/8] net/mlx5e: Fix E-Switch races in uplink unbind
From: Tariq Toukan
Date: Thu Nov 05 2026 - 16:19:44 EST
From: Cosmin Ratiu <cratiu@xxxxxxxxxx>
Unbinding the uplink Ethernet auxiliary driver in switchdev mode with
offloaded IPsec policies deadlocks on the esw->mode_lock semaphore:
mlx5_eswitch_safe_aux_devs_remove()
mlx5_esw_hold()
down_read_trylock(&esw->mode_lock) (*)
mlx5_core_reps_aux_devs_remove()
...
mlx5e_vport_uplink_rep_unload()
unregister_netdev()
xfrm_dev_unregister()
xfrm_dev_policy_flush()
xfrm_policy_kill()
xfrm_dev_policy_delete()
mlx5e_xfrm_del_policy()
mlx5e_accel_ipsec_fs_del_pol()
rx_ft_put_policy()
rx_put()
mlx5_eswitch_unblock_mode()
down_write(&esw->mode_lock) (BOOM)
Another deadlock with IPsec states:
mlx5_eswitch_safe_aux_devs_remove()
mlx5_esw_hold()
down_read_trylock(&esw->mode_lock) (*)
mlx5_core_reps_aux_devs_remove()
...
mlx5e_vport_uplink_rep_unload()
unregister_netdev()
xfrm_dev_unregister()
xfrm_dev_state_flush()
xfrm_dev_state_free()
mlx5e_xfrm_free_state()
mlx5e_accel_ipsec_fs_del_rule()
rx_ft_put()
rx_put()
rx_destroy()
mlx5_eswitch_unblock_encap()
down_write(&esw->mode_lock) (BOOM)
To fix these, first refactor initiators of eswitch operations so they no
longer step on each other's toes.
The active/inactive toggle and setting the inline/encap mode now use the
new mlx5_esw_lock_wait(), which waits for an operation in progress to
finish. They all hold devl_lock, so the only operation they can find in
progress is an uplink unbind, which doesn't need devl_lock.
Setting the esw mode keeps using mlx5_esw_try_lock(), which also rejects
the change while eswitch users exist and bails out with -EBUSY.
mlx5_eswitch_safe_aux_devs_remove() now also waits for an operation in
progress with the same mlx5_esw_lock_wait() before initiating one,
dropping mode_lock and then removing representors.
The deadlocks are avoided.
Finishing an operation in progress now requires mlx5_esw_unlock() to
wake up any waiters instead of plain up_write(&mode_lock).
As a side effect, the timeout in mlx5_eswitch_safe_aux_devs_remove() is
gone, which fixes another scary corner case, of representor devs not
being removed on timeout but leaked instead. The timeout hid the SD
deadlock fixed in the previous patch. Now that that's gone, no reason
for the timeout to stay.
This was found by internal testing and reproduced on HW.
It was independently found by Sashiko during analysis of another patch
in this series.
Fixes: 2ae8c7edea87 ("net/mlx5: Fix Unbinding uplink-netdev in switchdev mode")
Signed-off-by: Cosmin Ratiu <cratiu@xxxxxxxxxx>
Reviewed-by: Jianbo Liu <jianbol@xxxxxxxxxx>
Signed-off-by: Tariq Toukan <tariqt@xxxxxxxxxx>
---
.../net/ethernet/mellanox/mlx5/core/eswitch.c | 24 +++++++++
.../net/ethernet/mellanox/mlx5/core/eswitch.h | 1 +
.../mellanox/mlx5/core/eswitch_offloads.c | 50 +++++++++----------
3 files changed, 48 insertions(+), 27 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 2d289fb1353b..f50993cfcf7a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -3051,12 +3051,36 @@ int mlx5_esw_lock(struct mlx5_eswitch *esw)
return 0;
}
+static void esw_mode_lock(struct mlx5_eswitch *esw)
+{
+ down_write(&esw->mode_lock);
+}
+
+static void esw_mode_unlock(struct mlx5_eswitch *esw)
+{
+ up_write(&esw->mode_lock);
+}
+
+/* Lock the eswitch and wait for an operation in progress to finish. */
+void mlx5_esw_lock_wait(struct mlx5_eswitch *esw)
+{
+ down_write(&esw->mode_lock);
+ wait_var_event_any_lock(&esw->eswitch_operation_in_progress,
+ !esw->eswitch_operation_in_progress,
+ esw, esw_mode, TASK_UNINTERRUPTIBLE);
+}
+
/**
* mlx5_esw_unlock() - Release write lock on esw mode lock
* @esw: eswitch device.
+ *
+ * Also wakes up mlx5_esw_lock_wait() waiters, so it must be used to release
+ * mode_lock after clearing eswitch_operation_in_progress.
*/
void mlx5_esw_unlock(struct mlx5_eswitch *esw)
{
+ wake_up_var_locked(&esw->eswitch_operation_in_progress,
+ &esw->mode_lock);
up_write(&esw->mode_lock);
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 928b20e73d37..1fb67fc6d49a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -948,6 +948,7 @@ void mlx5_esw_get(struct mlx5_core_dev *dev);
void mlx5_esw_put(struct mlx5_core_dev *dev);
int mlx5_esw_try_lock(struct mlx5_eswitch *esw);
int mlx5_esw_lock(struct mlx5_eswitch *esw);
+void mlx5_esw_lock_wait(struct mlx5_eswitch *esw);
void mlx5_esw_unlock(struct mlx5_eswitch *esw);
void esw_vport_change_handle_locked(struct mlx5_vport *vport);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 4939e10336c2..4da89fcf215b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -4441,7 +4441,7 @@ static bool mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch *esw,
!esw->offloads_inactive)
return false;
- down_write(&esw->mode_lock);
+ mlx5_esw_lock_wait(esw);
esw->offloads_inactive = !esw->offloads_inactive;
esw->eswitch_operation_in_progress = true;
up_write(&esw->mode_lock);
@@ -4453,34 +4453,30 @@ static bool mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch *esw,
down_write(&esw->mode_lock);
esw->eswitch_operation_in_progress = false;
- up_write(&esw->mode_lock);
+ mlx5_esw_unlock(esw);
return true;
}
-#define MLX5_ESW_HOLD_TIMEOUT_MS 7000
-#define MLX5_ESW_HOLD_RETRY_DELAY_MS 500
-
void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev)
{
- unsigned long timeout;
- bool hold_esw = true;
+ struct mlx5_eswitch *esw = dev->priv.eswitch;
- /* Wait for any concurrent eswitch mode transition to complete. */
- if (!mlx5_esw_hold(dev)) {
- timeout = jiffies + msecs_to_jiffies(MLX5_ESW_HOLD_TIMEOUT_MS);
- while (!mlx5_esw_hold(dev)) {
- if (!time_before(jiffies, timeout)) {
- hold_esw = false;
- break;
- }
- msleep(MLX5_ESW_HOLD_RETRY_DELAY_MS);
- }
- }
- if (hold_esw) {
- if (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS)
- mlx5_core_reps_aux_devs_remove(dev);
- mlx5_esw_release(dev);
- }
+ if (!mlx5_esw_allowed(esw))
+ return;
+
+ /* Wait for any concurrent eswitch mode transition to complete.
+ * Deadlock risk if ever called from within an operation in progress.
+ */
+ mlx5_esw_lock_wait(esw);
+ esw->eswitch_operation_in_progress = true;
+ up_write(&esw->mode_lock);
+
+ if (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS)
+ mlx5_core_reps_aux_devs_remove(dev);
+
+ down_write(&esw->mode_lock);
+ esw->eswitch_operation_in_progress = false;
+ mlx5_esw_unlock(esw);
}
int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
@@ -4656,7 +4652,7 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
if (IS_ERR(esw))
return PTR_ERR(esw);
- down_write(&esw->mode_lock);
+ mlx5_esw_lock_wait(esw);
switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
@@ -4694,7 +4690,7 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
down_write(&esw->mode_lock);
esw->eswitch_operation_in_progress = false;
- up_write(&esw->mode_lock);
+ mlx5_esw_unlock(esw);
return 0;
out:
@@ -4764,7 +4760,7 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
if (IS_ERR(esw))
return PTR_ERR(esw);
- down_write(&esw->mode_lock);
+ mlx5_esw_lock_wait(esw);
if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
(!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
@@ -4820,7 +4816,7 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
esw->eswitch_operation_in_progress = false;
unlock:
- up_write(&esw->mode_lock);
+ mlx5_esw_unlock(esw);
return err;
}
--
2.44.0