[PATCH v1 07/44] dmaengine: sdxi: implement synchronous admin operations
From: Wei Huang
Date: Thu Nov 19 2026 - 15:26:32 EST
Move context start and stop submission out of existing context
management code into new admin management files, and build the final
AdminGrp helper layer on the existing submission queue and completion
queue. Implement AKey and RKey sync helpers as part of AdminGrp. To
support vendor modules that will be introduced later, a discovery
operations plus a generic exported descriptor path are also added.
Co-developed-by: Nathan Lynch <nathan.lynch@xxxxxxx>
Signed-off-by: Nathan Lynch <nathan.lynch@xxxxxxx>
Signed-off-by: Wei Huang <wei.huang2@xxxxxxx>
---
drivers/dma/sdxi/Makefile | 1 +
drivers/dma/sdxi/admin.c | 281 +++++++++++++++++++++++++++++++++++++
drivers/dma/sdxi/admin.h | 45 ++++++
drivers/dma/sdxi/context.c | 136 +-----------------
drivers/dma/sdxi/context.h | 13 --
drivers/dma/sdxi/device.c | 33 +++++
drivers/dma/sdxi/dma.c | 22 ++-
drivers/dma/sdxi/pci.c | 16 +++
drivers/dma/sdxi/sdxi.h | 8 ++
9 files changed, 402 insertions(+), 153 deletions(-)
create mode 100644 drivers/dma/sdxi/admin.c
create mode 100644 drivers/dma/sdxi/admin.h
diff --git a/drivers/dma/sdxi/Makefile b/drivers/dma/sdxi/Makefile
index 3a1d36997b44..bddd1278096c 100644
--- a/drivers/dma/sdxi/Makefile
+++ b/drivers/dma/sdxi/Makefile
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_SDXI_CORE) += sdxi-core.o
sdxi-core-y := \
+ admin.o \
completion.o \
context.o \
descriptor.o \
diff --git a/drivers/dma/sdxi/admin.c b/drivers/dma/sdxi/admin.c
new file mode 100644
index 000000000000..ed3890de8cc1
--- /dev/null
+++ b/drivers/dma/sdxi/admin.c
@@ -0,0 +1,281 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * SDXI administrative command submission.
+ *
+ * Copyright Advanced Micro Devices, Inc.
+ */
+
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+#include "admin.h"
+#include "completion.h"
+#include "descriptor.h"
+#include "ring.h"
+
+#define SDXI_ADMIN_DEFAULT_TIMEOUT_MS 5000U
+#define SDXI_ADMIN_DRAIN_TIMEOUT_MS 1000U
+
+static int sdxi_adm_submit(struct sdxi_cxt *adm,
+ const struct sdxi_desc *descs, size_t nr_descs,
+ unsigned int timeout_ms, u32 *cst_flags,
+ bool interruptible)
+{
+ struct sdxi_ring_resv resv;
+ struct sdxi_completion comp;
+ struct sdxi_slot slot;
+ bool completed = false;
+ int wait_ret;
+ size_t i;
+ int ret;
+
+ if (!adm || !sdxi_cxt_is_admin(adm) || !adm->ring_state)
+ return -ENODEV;
+ if (!nr_descs)
+ return -EINVAL;
+
+ /* Keep reservations from becoming visible before they are populated. */
+ mutex_lock(&adm->sdxi->admin_lock);
+ if (adm->sdxi->admin_dead) {
+ ret = -EIO;
+ goto out_unlock;
+ }
+ ret = sdxi_cst_pool_get_wait(adm->cst_pool, &comp);
+ if (ret)
+ goto out_unlock;
+ ret = sdxi_ring_reserve(adm->ring_state, nr_descs, &resv);
+ if (ret) {
+ sdxi_cst_pool_put(adm->cst_pool, comp.cst_blk);
+ goto out_unlock;
+ }
+
+ for (i = 0; i < nr_descs; i++) {
+ slot = sdxi_ring_resv_slot(&resv, i);
+ *sdxi_slot_desc(slot) = descs[i];
+ if (i + 1 == nr_descs)
+ sdxi_cxt_submit(adm, slot, &comp, SDXI_CXT_SUBMIT_KICK);
+ else
+ sdxi_cxt_submit(adm, slot, NULL, 0);
+ }
+
+ ret = sdxi_cst_blk_wait_timeout(comp.cst_blk, timeout_ms, interruptible);
+ if (ret == -ETIMEDOUT || ret == -ERESTARTSYS) {
+ /*
+ * The descriptor can still access caller-owned DMA memory after
+ * a timed or interrupted wait. Drain it before returning so the
+ * caller can safely release any associated mappings.
+ */
+ wait_ret = sdxi_cst_blk_wait_timeout(comp.cst_blk,
+ SDXI_ADMIN_DRAIN_TIMEOUT_MS,
+ false);
+ if (wait_ret == 0 || wait_ret == -EIO) {
+ completed = true;
+ if (cst_flags)
+ *cst_flags = sdxi_cst_blk_flags(comp.cst_blk);
+ }
+ if (wait_ret == -ETIMEDOUT) {
+ adm->sdxi->admin_dead = true;
+ dev_err(adm->sdxi->dev,
+ "admin command did not drain; quiescing function\n");
+ sdxi_dev_force_quiesce(adm->sdxi);
+ }
+ }
+ if (!ret || ret == -EIO) {
+ completed = true;
+ if (cst_flags)
+ *cst_flags = sdxi_cst_blk_flags(comp.cst_blk);
+ }
+ if (completed)
+ sdxi_cst_pool_put(adm->cst_pool, comp.cst_blk);
+ sdxi_ring_wake_up(adm->ring_state);
+
+out_unlock:
+ mutex_unlock(&adm->sdxi->admin_lock);
+ return ret;
+}
+
+int sdxi_adm_update_cxt(struct sdxi_cxt *cxt)
+{
+ struct sdxi_desc descs[2] = {};
+ int ret;
+
+ if (!cxt || sdxi_cxt_is_admin(cxt))
+ return -EINVAL;
+
+ ret = sdxi_encode_cxt_upd(&descs[0], &(const struct sdxi_cxt_upd) {
+ .level = SDXI_CXT_UPD_LVL_L1,
+ .range = sdxi_cxt_range_single(cxt->id),
+ });
+ if (ret)
+ return ret;
+
+ ret = sdxi_encode_sync(&descs[1], &(const struct sdxi_sync) {
+ .filter = SDXI_SYNC_FLT_CXT,
+ .range = sdxi_cxt_range_single(cxt->id),
+ });
+ if (ret)
+ return ret;
+
+ return sdxi_adm_submit(to_admin_cxt(cxt), descs, ARRAY_SIZE(descs),
+ SDXI_ADMIN_DEFAULT_TIMEOUT_MS, NULL, false);
+}
+
+int sdxi_adm_start_cxt_sync(struct sdxi_cxt *cxt, unsigned int timeout_ms,
+ u32 *cst_flags, bool interruptible)
+{
+ struct sdxi_desc descs[2] = {};
+ int ret;
+
+ if (!cxt || sdxi_cxt_is_admin(cxt))
+ return -EINVAL;
+
+ ret = sdxi_encode_cxt_start(&descs[0], &(const struct sdxi_cxt_start) {
+ .range = sdxi_cxt_range_single(cxt->id),
+ });
+ if (ret)
+ return ret;
+
+ ret = sdxi_encode_sync(&descs[1], &(const struct sdxi_sync) {
+ .filter = SDXI_SYNC_FLT_CXT,
+ .range = sdxi_cxt_range_single(cxt->id),
+ });
+ if (ret)
+ return ret;
+
+ return sdxi_adm_submit(to_admin_cxt(cxt), descs, ARRAY_SIZE(descs),
+ timeout_ms, cst_flags, interruptible);
+}
+
+int sdxi_adm_stop_cxt_sync(struct sdxi_cxt *cxt, unsigned int timeout_ms,
+ u32 *cst_flags, bool interruptible)
+{
+ struct sdxi_desc descs[2] = {};
+ int ret;
+
+ if (!cxt || sdxi_cxt_is_admin(cxt))
+ return -EINVAL;
+
+ ret = sdxi_encode_cxt_stop(&descs[0], &(const struct sdxi_cxt_stop) {
+ .range = sdxi_cxt_range_single(cxt->id),
+ });
+ if (ret)
+ return ret;
+
+ ret = sdxi_encode_sync(&descs[1], &(const struct sdxi_sync) {
+ .filter = SDXI_SYNC_FLT_STOP,
+ .range = sdxi_cxt_range_single(cxt->id),
+ });
+ if (ret)
+ return ret;
+
+ return sdxi_adm_submit(to_admin_cxt(cxt), descs, ARRAY_SIZE(descs),
+ timeout_ms, cst_flags, interruptible);
+}
+
+int sdxi_adm_rkey_update_sync(struct sdxi_dev *sdxi, u16 rkey_start,
+ u16 rkey_end, unsigned int timeout_ms,
+ u32 *cst_flags)
+{
+ struct sdxi_desc descs[2] = {};
+ int ret;
+
+ if (!sdxi)
+ return -ENODEV;
+
+ ret = sdxi_encode_rkey_upd(&descs[0], &(const struct sdxi_rkey_upd) {
+ .rkey_start = rkey_start,
+ .rkey_end = rkey_end,
+ });
+ if (ret)
+ return ret;
+
+ ret = sdxi_encode_sync(&descs[1], &(const struct sdxi_sync) {
+ .filter = SDXI_SYNC_FLT_RKEY,
+ .key_start = rkey_start,
+ .key_end = rkey_end,
+ });
+ if (ret)
+ return ret;
+
+ return sdxi_adm_submit(sdxi->admin_cxt, descs, ARRAY_SIZE(descs),
+ timeout_ms, cst_flags, false);
+}
+
+int sdxi_adm_akey_update_sync(struct sdxi_dev *sdxi, u16 cxt_start,
+ u16 cxt_end, u16 akey_start, u16 akey_end,
+ unsigned int timeout_ms, u32 *cst_flags)
+{
+ struct sdxi_cxt_range range = {
+ .cxt_start = cxt_start,
+ .cxt_end = cxt_end,
+ };
+ struct sdxi_desc descs[2] = {};
+ int ret;
+
+ if (!sdxi)
+ return -ENODEV;
+
+ ret = sdxi_encode_akey_upd(&descs[0], &(const struct sdxi_akey_upd) {
+ .range = range,
+ .akey_start = akey_start,
+ .akey_end = akey_end,
+ });
+ if (ret)
+ return ret;
+
+ ret = sdxi_encode_sync(&descs[1], &(const struct sdxi_sync) {
+ .filter = SDXI_SYNC_FLT_AKEY,
+ .range = range,
+ .key_start = akey_start,
+ .key_end = akey_end,
+ });
+ if (ret)
+ return ret;
+
+ return sdxi_adm_submit(sdxi->admin_cxt, descs, ARRAY_SIZE(descs),
+ timeout_ms, cst_flags, false);
+}
+
+int sdxi_adm_disc(struct sdxi_dev *sdxi, u16 index_start, u16 index_end,
+ dma_addr_t addr, unsigned int timeout_ms, u32 *cst_flags)
+{
+ struct sdxi_desc desc = {};
+ int ret;
+
+ if (!sdxi)
+ return -ENODEV;
+
+ ret = sdxi_encode_disc(&desc, &(const struct sdxi_disc) {
+ .index_start = index_start,
+ .index_end = index_end,
+ .addr = addr,
+ });
+ if (ret)
+ return ret;
+
+ return sdxi_adm_submit(sdxi->admin_cxt, &desc, 1, timeout_ms,
+ cst_flags, false);
+}
+
+int sdxi_adm_submit_desc(struct sdxi_cxt *adm, const struct sdxi_desc *desc,
+ unsigned int timeout_ms, u32 *cst_flags)
+{
+ if (!desc)
+ return -EINVAL;
+
+ return sdxi_adm_submit(adm, desc, 1, timeout_ms, cst_flags, false);
+}
+EXPORT_SYMBOL_NS_GPL(sdxi_adm_submit_desc, "SDXI");
+
+int sdxi_adm_start_cxt(struct sdxi_cxt *cxt)
+{
+ return sdxi_adm_start_cxt_sync(cxt, SDXI_ADMIN_DEFAULT_TIMEOUT_MS,
+ NULL, false);
+}
+
+int sdxi_adm_stop_cxt(struct sdxi_cxt *cxt)
+{
+ return sdxi_adm_stop_cxt_sync(cxt, SDXI_ADMIN_DEFAULT_TIMEOUT_MS,
+ NULL, false);
+}
diff --git a/drivers/dma/sdxi/admin.h b/drivers/dma/sdxi/admin.h
new file mode 100644
index 000000000000..175f8c28c757
--- /dev/null
+++ b/drivers/dma/sdxi/admin.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * SDXI administrative command helpers.
+ *
+ * Copyright Advanced Micro Devices, Inc.
+ */
+
+#ifndef DMA_SDXI_ADMIN_H
+#define DMA_SDXI_ADMIN_H
+
+#include <linux/dma-mapping.h>
+#include <linux/types.h>
+
+#include "context.h"
+
+static inline struct sdxi_cxt *to_admin_cxt(const struct sdxi_cxt *cxt)
+{
+ return cxt->sdxi->admin_cxt;
+}
+
+static inline bool sdxi_cxt_is_admin(const struct sdxi_cxt *cxt)
+{
+ return cxt == to_admin_cxt(cxt);
+}
+
+int sdxi_adm_update_cxt(struct sdxi_cxt *cxt);
+int sdxi_adm_start_cxt_sync(struct sdxi_cxt *cxt, unsigned int timeout_ms,
+ u32 *cst_flags, bool interruptible);
+int sdxi_adm_stop_cxt_sync(struct sdxi_cxt *cxt, unsigned int timeout_ms,
+ u32 *cst_flags, bool interruptible);
+int sdxi_adm_rkey_update_sync(struct sdxi_dev *sdxi, u16 rkey_start,
+ u16 rkey_end, unsigned int timeout_ms,
+ u32 *cst_flags);
+int sdxi_adm_akey_update_sync(struct sdxi_dev *sdxi, u16 cxt_start,
+ u16 cxt_end, u16 akey_start, u16 akey_end,
+ unsigned int timeout_ms, u32 *cst_flags);
+int sdxi_adm_disc(struct sdxi_dev *sdxi, u16 index_start, u16 index_end,
+ dma_addr_t addr, unsigned int timeout_ms, u32 *cst_flags);
+int sdxi_adm_submit_desc(struct sdxi_cxt *adm, const struct sdxi_desc *desc,
+ unsigned int timeout_ms, u32 *cst_flags);
+
+int sdxi_adm_start_cxt(struct sdxi_cxt *cxt);
+int sdxi_adm_stop_cxt(struct sdxi_cxt *cxt);
+
+#endif /* DMA_SDXI_ADMIN_H */
diff --git a/drivers/dma/sdxi/context.c b/drivers/dma/sdxi/context.c
index d75668e89e61..a8d065b57bac 100644
--- a/drivers/dma/sdxi/context.c
+++ b/drivers/dma/sdxi/context.c
@@ -22,6 +22,7 @@
#include <asm/barrier.h>
#include <asm/rwonce.h>
+#include "admin.h"
#include "completion.h"
#include "context.h"
#include "descriptor.h"
@@ -106,78 +107,6 @@ static struct sdxi_cxt *sdxi_alloc_cxt(struct sdxi_dev *sdxi)
return_ptr(cxt);
}
-/* Claim completion storage before making a ring reservation irrevocable. */
-static int sdxi_cxt_reserve(struct sdxi_cxt *cxt, size_t nr,
- struct sdxi_ring_resv *resv,
- struct sdxi_completion *comp)
-{
- int err;
-
- err = sdxi_cst_pool_get_wait(cxt->cst_pool, comp);
- if (err)
- return err;
-
- err = sdxi_ring_reserve(cxt->ring_state, nr, resv);
- if (err)
- sdxi_cst_pool_put(cxt->cst_pool, comp->cst_blk);
-
- return err;
-}
-
-static int sdxi_cxt_poll_completion(struct sdxi_cxt *cxt,
- const struct sdxi_completion *comp)
-{
- int err = sdxi_cst_blk_poll(comp->cst_blk);
-
- /*
- * A timeout does not end hardware ownership. Keep these administrative
- * blocks claimed until context teardown, even if they complete later.
- * Repeated timeouts can exhaust the pool; allocation then times out too.
- */
- if (err != -ETIMEDOUT)
- sdxi_cst_pool_put(cxt->cst_pool, comp->cst_blk);
- sdxi_ring_wake_up(cxt->ring_state);
-
- return err;
-}
-
-/*
- * Signal the function that a client context's control structures have
- * been modified so it discards any privately cached copies of them.
- * DSC_SYNC.CXT is required before the operation can be considered
- * complete.
- */
-static int sdxi_cxt_update(const struct sdxi_cxt *cxt)
-{
- struct sdxi_cxt *adm = to_admin_cxt(cxt);
- struct sdxi_slot upd, sync;
- struct sdxi_ring_resv resv;
- struct sdxi_completion comp;
- int err;
-
- might_sleep();
-
- err = sdxi_cxt_reserve(adm, 2, &resv, &comp);
- if (err)
- return err;
-
- upd = sdxi_ring_resv_slot(&resv, 0);
- sync = sdxi_ring_resv_slot(&resv, 1);
-
- sdxi_encode_cxt_upd(sdxi_slot_desc(upd), &(const struct sdxi_cxt_upd) {
- .level = SDXI_CXT_UPD_LVL_L1,
- .range = sdxi_cxt_range_single(cxt->id),
- });
- sdxi_encode_sync(sdxi_slot_desc(sync), &(const struct sdxi_sync) {
- .filter = SDXI_SYNC_FLT_CXT,
- .range = sdxi_cxt_range_single(cxt->id),
- });
- sdxi_cxt_submit(adm, upd, NULL, 0);
- sdxi_cxt_submit(adm, sync, &comp, SDXI_CXT_SUBMIT_KICK);
-
- return sdxi_cxt_poll_completion(adm, &comp);
-}
-
struct sdxi_cxt_ctl_cfg {
dma_addr_t ds_ring_ptr;
dma_addr_t cxt_sts_ptr;
@@ -417,67 +346,6 @@ int sdxi_admin_cxt_init(struct sdxi_dev *sdxi)
return devm_add_action_or_reset(sdxi->dev, free_admin_cxt, sdxi);
}
-int sdxi_start_cxt(struct sdxi_cxt *cxt)
-{
- struct sdxi_cxt *adm = to_admin_cxt(cxt);
- struct sdxi_ring_resv resv;
- struct sdxi_completion comp;
- struct sdxi_slot start;
- int err;
-
- might_sleep();
-
- /* Administrative context activation is controlled by the function state. */
- if (WARN_ON(adm == cxt))
- return -EINVAL;
-
- err = sdxi_cxt_reserve(adm, 1, &resv, &comp);
- if (err)
- return err;
-
- start = sdxi_ring_resv_slot(&resv, 0);
- sdxi_encode_cxt_start(sdxi_slot_desc(start), &(const struct sdxi_cxt_start) {
- .range = sdxi_cxt_range_single(cxt->id),
- });
- sdxi_cxt_submit(adm, start, &comp, SDXI_CXT_SUBMIT_KICK);
-
- return sdxi_cxt_poll_completion(adm, &comp);
-}
-
-void sdxi_stop_cxt(struct sdxi_cxt *cxt)
-{
- struct sdxi_cxt *adm = to_admin_cxt(cxt);
- struct sdxi_slot stop, sync;
- struct sdxi_ring_resv resv;
- struct sdxi_completion comp;
- int err;
-
- might_sleep();
-
- /* Administrative context stopping is controlled by the function state. */
- if (WARN_ON(adm == cxt))
- return;
-
- err = sdxi_cxt_reserve(adm, 2, &resv, &comp);
- if (WARN_ON_ONCE(err))
- return;
-
- stop = sdxi_ring_resv_slot(&resv, 0);
- sync = sdxi_ring_resv_slot(&resv, 1);
-
- sdxi_encode_cxt_stop(sdxi_slot_desc(stop), &(const struct sdxi_cxt_stop) {
- .range = sdxi_cxt_range_single(cxt->id),
- });
- sdxi_encode_sync(sdxi_slot_desc(sync), &(const struct sdxi_sync) {
- .filter = SDXI_SYNC_FLT_STOP,
- .range = sdxi_cxt_range_single(cxt->id),
- });
- sdxi_cxt_submit(adm, stop, NULL, 0);
- sdxi_cxt_submit(adm, sync, &comp, SDXI_CXT_SUBMIT_KICK);
-
- WARN_ON(sdxi_cxt_poll_completion(adm, &comp));
-}
-
/*
* Temporary owner for context id until it can be assigned to a
* context object; enables scope-based cleanup.
@@ -637,7 +505,7 @@ int sdxi_cxt_exit(struct sdxi_cxt *cxt)
invalidate_cxt_ctl(cxt->cxt_ctl);
mutex_unlock(&sdxi->cxt_lock);
- err = sdxi_cxt_update(cxt);
+ err = sdxi_adm_update_cxt(cxt);
if (err) {
dev_err(sdxi->dev,
"context %u release failed (%d), retaining resources until function shutdown\n",
diff --git a/drivers/dma/sdxi/context.h b/drivers/dma/sdxi/context.h
index acf41709c4de..196207cbca94 100644
--- a/drivers/dma/sdxi/context.h
+++ b/drivers/dma/sdxi/context.h
@@ -60,9 +60,6 @@ struct sdxi_cxt *sdxi_cxt_new(struct sdxi_dev *sdxi);
int sdxi_cxt_exit(struct sdxi_cxt *cxt);
void sdxi_client_cxts_exit(struct sdxi_dev *sdxi);
-int sdxi_start_cxt(struct sdxi_cxt *cxt);
-void sdxi_stop_cxt(struct sdxi_cxt *cxt);
-
enum sdxi_submit_flags {
SDXI_CXT_SUBMIT_KICK = BIT(0),
SDXI_CXT_SUBMIT_FENCE = BIT(1),
@@ -73,16 +70,6 @@ void sdxi_cxt_attach_cst(struct sdxi_slot slot,
void sdxi_cxt_submit(struct sdxi_cxt *cxt, struct sdxi_slot slot,
const struct sdxi_completion *comp, enum sdxi_submit_flags flags);
-static inline struct sdxi_cxt *to_admin_cxt(const struct sdxi_cxt *cxt)
-{
- return cxt->sdxi->admin_cxt;
-}
-
-static inline bool sdxi_cxt_is_admin(const struct sdxi_cxt *cxt)
-{
- return cxt == to_admin_cxt(cxt);
-}
-
static inline void sdxi_cxt_kick(struct sdxi_cxt *cxt)
{
/*
diff --git a/drivers/dma/sdxi/device.c b/drivers/dma/sdxi/device.c
index c5b5b39a7fe5..9b6a89e469ad 100644
--- a/drivers/dma/sdxi/device.c
+++ b/drivers/dma/sdxi/device.c
@@ -245,6 +245,38 @@ static int sdxi_dev_start(struct sdxi_dev *sdxi)
return 0;
}
+void sdxi_dev_force_quiesce(struct sdxi_dev *sdxi)
+{
+ enum sdxi_fn_gsv status = sdxi_dev_gsv(sdxi);
+ int ret;
+
+ if (status == SDXI_GSV_STOP)
+ return;
+
+ if (status != SDXI_GSV_ERROR) {
+ sdxi_write_fn_gsr(sdxi, SDXI_GSRV_STOP_HD);
+ ret = sdxi_dev_gsv_poll(sdxi, status,
+ status == SDXI_GSV_STOP ||
+ status == SDXI_GSV_ERROR);
+ if (!ret && status == SDXI_GSV_STOP)
+ return;
+ } else {
+ ret = 0;
+ }
+
+ if (!ret && status == SDXI_GSV_ERROR) {
+ sdxi_write_fn_gsr(sdxi, SDXI_GSRV_RESET);
+ ret = sdxi_dev_gsv_poll(sdxi, status,
+ status == SDXI_GSV_STOP);
+ if (!ret)
+ return;
+ }
+
+ dev_err(sdxi->dev, "failed to quiesce function; disabling bus DMA\n");
+ if (sdxi->bus_ops->force_quiesce)
+ sdxi->bus_ops->force_quiesce(sdxi);
+}
+
/* Stop the function and discard inherited MMIO configuration. */
static int sdxi_fn_prepare(struct sdxi_dev *sdxi)
{
@@ -425,6 +457,7 @@ int sdxi_register(struct device *dev, const struct sdxi_bus_ops *ops)
ida_init(&sdxi->vectors);
xa_init_flags(&sdxi->client_cxts, XA_FLAGS_ALLOC1);
mutex_init(&sdxi->cxt_lock);
+ mutex_init(&sdxi->admin_lock);
dev_set_drvdata(dev, sdxi);
err = sdxi->bus_ops->init(sdxi);
diff --git a/drivers/dma/sdxi/dma.c b/drivers/dma/sdxi/dma.c
index 512855c83da2..a4b254a3080d 100644
--- a/drivers/dma/sdxi/dma.c
+++ b/drivers/dma/sdxi/dma.c
@@ -20,6 +20,7 @@
#include "../dmaengine.h"
#include "../virt-dma.h"
+#include "admin.h"
#include "completion.h"
#include "context.h"
#include "descriptor.h"
@@ -416,9 +417,12 @@ static int sdxi_dma_alloc_chan_resources(struct dma_chan *dma_chan)
if (err)
goto free_akey;
- err = sdxi_start_cxt(sdchan->cxt);
- if (err)
+ err = sdxi_adm_start_cxt(sdchan->cxt);
+ if (err) {
+ sdxi->admin_dead = true;
+ sdxi_dev_force_quiesce(sdxi);
goto free_irq;
+ }
return 0;
free_irq:
@@ -428,25 +432,31 @@ static int sdxi_dma_alloc_chan_resources(struct dma_chan *dma_chan)
free_vector:
sdxi_free_vector(sdxi, vector);
exit_cxt:
- sdxi_cxt_exit(sdchan->cxt);
+ WARN_ON(sdxi_cxt_exit(sdchan->cxt));
return err;
}
static void sdxi_dma_free_chan_resources(struct dma_chan *dma_chan)
{
struct sdxi_dma_chan *sdchan = to_sdxi_dma_chan(dma_chan);
+ struct sdxi_dev *sdxi = sdchan->cxt->sdxi;
LIST_HEAD(head);
+ int err;
- sdxi_stop_cxt(sdchan->cxt);
+ err = sdxi_adm_stop_cxt(sdchan->cxt);
+ if (WARN_ON(err)) {
+ sdxi->admin_dead = true;
+ sdxi_dev_force_quiesce(sdxi);
+ }
free_irq(sdchan->irq, sdchan);
- sdxi_free_vector(sdchan->cxt->sdxi, sdchan->vector);
+ sdxi_free_vector(sdxi, sdchan->vector);
sdxi_free_akey(sdchan->cxt, sdchan->akey);
/* virt-dma does not know about the driver's pending list. */
scoped_guard(spinlock_irqsave, &sdchan->vchan.lock)
list_splice_tail_init(&sdchan->pending, &head);
vchan_dma_desc_free_list(&sdchan->vchan, &head);
vchan_free_chan_resources(to_virt_chan(dma_chan));
- sdxi_cxt_exit(sdchan->cxt);
+ WARN_ON(sdxi_cxt_exit(sdchan->cxt));
}
int sdxi_dma_register(struct sdxi_dev *sdxi)
diff --git a/drivers/dma/sdxi/pci.c b/drivers/dma/sdxi/pci.c
index ef978ca22db1..f9f75ff5e179 100644
--- a/drivers/dma/sdxi/pci.c
+++ b/drivers/dma/sdxi/pci.c
@@ -113,10 +113,26 @@ static int sdxi_pci_get_irq(struct sdxi_dev *sdxi, unsigned int nr)
return pci_irq_vector(sdxi_to_pci_dev(sdxi), nr);
}
+static void sdxi_pci_force_quiesce(struct sdxi_dev *sdxi)
+{
+ struct pci_dev *pdev = sdxi_to_pci_dev(sdxi);
+ int ret;
+
+ pci_clear_master(pdev);
+ ret = pci_try_reset_function(pdev);
+ if (!ret)
+ return;
+
+ dev_err(&pdev->dev, "function reset failed: %d\n", ret);
+ if (!pci_wait_for_pending_transaction(pdev))
+ dev_err(&pdev->dev, "pending PCI transactions did not drain\n");
+}
+
static const struct sdxi_bus_ops sdxi_pci_ops = {
.init = sdxi_pci_init,
.reset = sdxi_pci_reset,
.get_irq = sdxi_pci_get_irq,
+ .force_quiesce = sdxi_pci_force_quiesce,
};
static int sdxi_pci_probe(struct pci_dev *pdev,
diff --git a/drivers/dma/sdxi/sdxi.h b/drivers/dma/sdxi/sdxi.h
index 6d2b9ba88182..7b3cad785442 100644
--- a/drivers/dma/sdxi/sdxi.h
+++ b/drivers/dma/sdxi/sdxi.h
@@ -63,6 +63,11 @@ struct sdxi_bus_ops {
* @get_irq: Map device interrupt index to Linux IRQ number.
*/
int (*get_irq)(struct sdxi_dev *sdxi, unsigned int index);
+ /**
+ * @force_quiesce: Permanently prevent further bus-mastering DMA after
+ * device recovery fails.
+ */
+ void (*force_quiesce)(struct sdxi_dev *sdxi);
};
struct sdxi_dev {
@@ -91,6 +96,8 @@ struct sdxi_dev {
struct ida vectors;
struct sdxi_cxt *admin_cxt;
+ struct mutex admin_lock; /* admin submission serialization */
+ bool admin_dead;
struct xarray client_cxts; /* context id -> (struct sdxi_cxt *) */
bool has_mailbox;
@@ -138,6 +145,7 @@ static inline int sdxi_vector_to_irq(struct sdxi_dev *sdxi, unsigned int nr)
int sdxi_register(struct device *dev, const struct sdxi_bus_ops *ops);
void sdxi_unregister(struct device *dev);
+void sdxi_dev_force_quiesce(struct sdxi_dev *sdxi);
static inline u64 sdxi_read64(const struct sdxi_dev *sdxi, enum sdxi_reg reg)
{
--
2.56.1