[PATCH mlx5-next 08/10] RDMA/mlx5: Consolidate data direct state into one object
From: Tariq Toukan
Date: Thu Sep 17 2026 - 07:31:40 EST
From: Dragos Tatulea <dtatulea@xxxxxxxxxx>
There are three per device data direct objects: mlx5_data_direct_dev,
mlx5_data_direct_resources and mlx5_data_direct_registration with
separate allocations. This patch consolidates all of them into the
registration object.
For this, a few things needed to happen:
- mlx5_ib_dev keeps a pointer to mlx5_data_direct.
- mlx5_data_direct_dev is now accessible from mlx5_data_direct.
mlx5_data_direct_get_dev() is added as a helper to access
the device pointer.
- Since mlx5_data_direct is part of mlx5_data_direct_registration,
getting to the registration is a simple container_of. No need
for searching the list.
Since mlx5_data_direct_resources no longer has an accurate name,
this patch renames it.
This is a preparation for the upcoming move of data_direct to
mlx5_core.
This patch has no functional changes.
Signed-off-by: Dragos Tatulea <dtatulea@xxxxxxxxxx>
Signed-off-by: Tariq Toukan <tariqt@xxxxxxxxxx>
---
drivers/infiniband/hw/mlx5/data_direct.c | 73 ++++++++++--------------
drivers/infiniband/hw/mlx5/data_direct.h | 7 ---
drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +-
drivers/infiniband/hw/mlx5/mr.c | 6 +-
drivers/infiniband/hw/mlx5/odp.c | 2 +-
drivers/infiniband/hw/mlx5/std_types.c | 2 +-
drivers/infiniband/hw/mlx5/umr.c | 11 ++--
include/linux/mlx5/data_direct.h | 17 ++++++
8 files changed, 59 insertions(+), 62 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index 16afacdd3601..f58a144d95e3 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -18,10 +18,10 @@ static LIST_HEAD(mlx5_data_direct_reg_list);
static DEFINE_MUTEX(mlx5_data_direct_mutex);
struct mlx5_data_direct_registration {
- struct mlx5_ib_dev *ibdev;
char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];
struct list_head list;
struct blocking_notifier_head users;
+ struct mlx5_data_direct dd;
};
static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev,
@@ -139,8 +139,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
if (err)
goto err_mkey;
- dev->ddr.mkey = mkey;
- dev->ddr.pdn = pdn;
+ dev->data_direct->mkey = mkey;
+ dev->data_direct->pdn = pdn;
/* create another mkey with RO support */
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
@@ -157,8 +157,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
/* RO is defined as best effort */
if (!err) {
- dev->ddr.mkey_ro = mkey;
- dev->ddr.mkey_ro_valid = true;
+ dev->data_direct->mkey_ro = mkey;
+ dev->data_direct->mkey_ro_valid = true;
}
}
@@ -174,38 +174,25 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev)
void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev)
{
- if (dev->ddr.mkey_ro_valid)
- mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro);
+ if (dev->data_direct->mkey_ro_valid)
+ mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro);
- mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
- mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
+ mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey);
+ mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn);
- memset(&dev->ddr, 0, sizeof(dev->ddr));
+ memset(dev->data_direct, 0, sizeof(*dev->data_direct));
}
-static struct mlx5_data_direct_registration *
-mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev)
-{
- struct mlx5_data_direct_registration *reg;
-
- list_for_each_entry(reg, &mlx5_data_direct_reg_list, list)
- if (reg->ibdev == ibdev)
- return reg;
- return NULL;
-}
-
-static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev,
+static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg,
struct mlx5_data_direct_dev *dev)
{
- WRITE_ONCE(ibdev->data_direct_dev, dev);
+ WRITE_ONCE(reg->dd.dev, dev);
}
static void
mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg)
{
- struct mlx5_ib_dev *ibdev = reg->ibdev;
-
- WRITE_ONCE(ibdev->data_direct_dev, NULL);
+ WRITE_ONCE(reg->dd.dev, NULL);
blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND,
NULL);
}
@@ -223,7 +210,6 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
if (!reg)
return -ENOMEM;
- reg->ibdev = ibdev;
BLOCKING_INIT_NOTIFIER_HEAD(®->users);
err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid);
@@ -234,10 +220,12 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev)
return err;
}
+ ibdev->data_direct = ®->dd;
+
mutex_lock(&mlx5_data_direct_mutex);
list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
if (strcmp(dev->vuid, reg->vuid) == 0) {
- mlx5_data_direct_bind(ibdev, dev);
+ mlx5_data_direct_bind(reg, dev);
break;
}
}
@@ -257,14 +245,14 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev)
if (!mlx5_data_direct_supported(ibdev->mdev))
return;
+ reg = container_of(ibdev->data_direct,
+ struct mlx5_data_direct_registration, dd);
mutex_lock(&mlx5_data_direct_mutex);
- reg = mlx5_data_direct_get_reg(ibdev);
- if (reg) {
- list_del(®->list);
- mlx5_data_direct_do_unbind(reg);
- }
+ list_del(®->list);
+ mlx5_data_direct_do_unbind(reg);
mutex_unlock(&mlx5_data_direct_mutex);
+ ibdev->data_direct = NULL;
kfree(reg);
}
@@ -276,11 +264,10 @@ int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev,
if (!mlx5_data_direct_supported(ibdev->mdev))
return 0;
- mutex_lock(&mlx5_data_direct_mutex);
- reg = mlx5_data_direct_get_reg(ibdev);
- if (reg)
- blocking_notifier_chain_register(®->users, nb);
- mutex_unlock(&mlx5_data_direct_mutex);
+ reg = container_of(ibdev->data_direct,
+ struct mlx5_data_direct_registration, dd);
+ blocking_notifier_chain_register(®->users, nb);
+
return 0;
}
EXPORT_SYMBOL_GPL(mlx5_data_direct_register);
@@ -293,11 +280,9 @@ void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev,
if (!mlx5_data_direct_supported(ibdev->mdev))
return;
- mutex_lock(&mlx5_data_direct_mutex);
- reg = mlx5_data_direct_get_reg(ibdev);
- if (reg)
- blocking_notifier_chain_unregister(®->users, nb);
- mutex_unlock(&mlx5_data_direct_mutex);
+ reg = container_of(ibdev->data_direct,
+ struct mlx5_data_direct_registration, dd);
+ blocking_notifier_chain_unregister(®->users, nb);
}
EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister);
@@ -308,7 +293,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
mutex_lock(&mlx5_data_direct_mutex);
list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
if (strcmp(dev->vuid, reg->vuid) == 0)
- mlx5_data_direct_bind(reg->ibdev, dev);
+ mlx5_data_direct_bind(reg, dev);
}
/* Add the data direct device to the global list, further IB devices may
diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h
index f207239461bf..de2730f00852 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.h
+++ b/drivers/infiniband/hw/mlx5/data_direct.h
@@ -23,13 +23,6 @@ struct mlx5_data_direct_dev {
struct list_head list;
};
-struct mlx5_data_direct_resources {
- u32 pdn;
- u32 mkey;
- u32 mkey_ro;
- u8 mkey_ro_valid :1;
-};
-
int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev);
void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev);
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 8792e10340c5..6952170f472b 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1091,7 +1091,7 @@ struct mlx5_macsec {
struct mlx5_ib_dev {
struct ib_device ib_dev;
struct mlx5_core_dev *mdev;
- struct mlx5_data_direct_dev *data_direct_dev;
+ struct mlx5_data_direct *data_direct;
/* Protects data_direct_mr_list and serializes mr
* registration/deregistration with data direct device unbind.
*/
@@ -1152,7 +1152,6 @@ struct mlx5_ib_dev {
u16 pkey_table_len;
u8 lag_ports;
struct mlx5_special_mkeys mkeys;
- struct mlx5_data_direct_resources ddr;
#ifdef CONFIG_MLX5_MACSEC
struct mlx5_macsec macsec;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 1fc61a680f8a..5efe78cb34c1 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -587,7 +587,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
populate ? pd : dev->umrc.pd);
/* In case a data direct flow, overwrite the pdn field by its internal kernel PD */
if (umem->is_dmabuf && ksm_mode)
- MLX5_SET(mkc, mkc, pd, dev->ddr.pdn);
+ MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn);
MLX5_SET(mkc, mkc, free, !populate);
MLX5_SET(mkc, mkc, access_mode_1_0, access_mode);
@@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
return ERR_PTR(-EOPNOTSUPP);
mutex_lock(&dev->data_direct_lock);
- data_direct_dev = READ_ONCE(dev->data_direct_dev);
+ data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
if (!data_direct_dev) {
ret = -EINVAL;
goto end;
@@ -1006,7 +1006,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
/* If no device's 'data direct mkey' with RO flags exists
* mask it out accordingly.
*/
- if (!dev->ddr.mkey_ro_valid)
+ if (!dev->data_direct->mkey_ro_valid)
access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev,
offset, length, virt_addr, fd,
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 1badec9bf527..7b9f73b76d0d 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd)
int ret;
if (mr->data_direct)
- pdn = dev->ddr.pdn;
+ pdn = dev->data_direct->pdn;
else
pdn = to_mpd(pd)->pdn;
diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c
index df949fa634c9..3828100fdc6c 100644
--- a/drivers/infiniband/hw/mlx5/std_types.c
+++ b/drivers/infiniband/hw/mlx5/std_types.c
@@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)(
return PTR_ERR(c);
dev = to_mdev(c->ibucontext.device);
mutex_lock(&dev->data_direct_lock);
- data_direct_dev = READ_ONCE(dev->data_direct_dev);
+ data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct);
if (!data_direct_dev) {
ret = -ENODEV;
goto end;
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 48cae5cc1c1b..1ca83d2d8a0b 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -760,12 +760,14 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags,
}
if (dd) {
+ struct mlx5_data_direct *mdd = dev->data_direct;
+
cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter));
if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING &&
- dev->ddr.mkey_ro_valid)
- cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro);
+ mdd->mkey_ro_valid)
+ cur_ksm->key = cpu_to_be32(mdd->mkey_ro);
else
- cur_ksm->key = cpu_to_be32(dev->ddr.mkey);
+ cur_ksm->key = cpu_to_be32(mdd->mkey);
if (mr->umem->is_dmabuf &&
(flags & MLX5_IB_UPD_XLT_ZAP)) {
cur_ksm->va = 0;
@@ -810,7 +812,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr,
!(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ)))
return -EINVAL;
- return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn,
+ return _mlx5r_umr_update_mr_pas(mr, flags,
+ mr_to_mdev(mr)->data_direct->pdn,
true, start_block, nblocks);
}
diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h
index 71f47547aebb..eaa2dc6bc607 100644
--- a/include/linux/mlx5/data_direct.h
+++ b/include/linux/mlx5/data_direct.h
@@ -6,10 +6,27 @@
#ifndef _MLX5_DATA_DIRECT_H
#define _MLX5_DATA_DIRECT_H
+#include <linux/compiler.h>
+
struct mlx5_core_dev;
+struct mlx5_data_direct_dev;
struct mlx5_ib_dev;
struct notifier_block;
+struct mlx5_data_direct {
+ struct mlx5_data_direct_dev *dev;
+ u32 pdn;
+ u32 mkey;
+ u32 mkey_ro;
+ u8 mkey_ro_valid :1;
+};
+
+static inline struct mlx5_data_direct_dev *
+mlx5_data_direct_get_dev(struct mlx5_data_direct *dd)
+{
+ return dd ? READ_ONCE(dd->dev) : NULL;
+}
+
int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev);
void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev);
--
2.44.0