[PATCH 2/4] RDMA/ionic: segregate rq related fields from ionic_qp into a new ionic_rq struct
From: Abhijit Gangurde
Date: Tue Jul 07 2026 - 05:57:00 EST
In preparation for SRQ support, segregate rq related fields from ionic_qp
into a new ionic_rq struct so that it can leverage the code in and around
ionic_qp_rq_init/_destroy. ionic_rq would enable both non-srq and srq
code to use the same rq init/destroy infra.
Signed-off-by: Abhijit Gangurde <abhijit.gangurde@xxxxxxx>
---
.../infiniband/hw/ionic/ionic_controlpath.c | 306 +++++++++---------
drivers/infiniband/hw/ionic/ionic_datapath.c | 80 ++---
drivers/infiniband/hw/ionic/ionic_ibdev.h | 42 ++-
3 files changed, 220 insertions(+), 208 deletions(-)
diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c
index ea12d9b8e125..a5045c08915e 100644
--- a/drivers/infiniband/hw/ionic/ionic_controlpath.c
+++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c
@@ -1327,8 +1327,8 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject);
const u32 flags = to_ionic_qp_flags(0, 0,
qp->sq_cmb & IONIC_CMB_ENABLE,
- qp->rq_cmb & IONIC_CMB_ENABLE,
- qp->sq_spec, qp->rq_spec,
+ qp->rq.cmb & IONIC_CMB_ENABLE,
+ qp->sq_spec, qp->rq.spec,
pd->flags & IONIC_QPF_PRIVILEGED,
pd_remote_privileged(&pd->ibpd));
struct ionic_admin_wr wr = {
@@ -1362,8 +1362,8 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
if (qp->has_rq) {
wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid);
- wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.depth_log2;
- wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.stride_log2;
+ wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2;
+ wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2;
wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2;
wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0);
wr.wqe.cmd.create_qp.rq_map_count =
@@ -1385,8 +1385,8 @@ static int ionic_modify_qp_cmd(struct ionic_ibdev *dev,
const u32 flags = to_ionic_qp_flags(attr->qp_access_flags,
attr->en_sqd_async_notify,
qp->sq_cmb & IONIC_CMB_ENABLE,
- qp->rq_cmb & IONIC_CMB_ENABLE,
- qp->sq_spec, qp->rq_spec,
+ qp->rq.cmb & IONIC_CMB_ENABLE,
+ qp->sq_spec, qp->rq.spec,
pd->flags & IONIC_QPF_PRIVILEGED,
pd_remote_privileged(qp->ibqp.pd));
const u8 state = to_ionic_qp_modify_state(attr->qp_state,
@@ -1556,9 +1556,9 @@ static int ionic_query_qp_cmd(struct ionic_ibdev *dev,
if (qp->has_rq) {
attr->cap.max_recv_sge =
- ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2,
- qp->rq_spec,
- qp->rq_cmb & IONIC_CMB_EXPDB);
+ ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2,
+ qp->rq.spec,
+ qp->rq.cmb & IONIC_CMB_EXPDB);
}
query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
@@ -1930,99 +1930,88 @@ static void ionic_qp_sq_destroy(struct ionic_ibdev *dev,
ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev);
}
-static void ionic_qp_rq_init_cmb(struct ionic_ibdev *dev,
- struct ionic_qp *qp,
- struct ib_udata *udata)
+static void ionic_rq_init_cmb(struct ionic_ibdev *dev,
+ struct ionic_rq *rq,
+ struct ib_udata *udata)
{
u8 expdb_stride_log2 = 0;
bool expdb;
int rc;
- if (!(qp->rq_cmb & IONIC_CMB_ENABLE))
+ if (!(rq->cmb & IONIC_CMB_ENABLE))
goto not_in_cmb;
- if (qp->rq_cmb & ~IONIC_CMB_SUPPORTED) {
- if (qp->rq_cmb & IONIC_CMB_REQUIRE)
+ if (rq->cmb & ~IONIC_CMB_SUPPORTED) {
+ if (rq->cmb & IONIC_CMB_REQUIRE)
goto not_in_cmb;
- qp->rq_cmb &= IONIC_CMB_SUPPORTED;
+ rq->cmb &= IONIC_CMB_SUPPORTED;
}
- if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) {
- if (qp->rq_cmb & IONIC_CMB_REQUIRE)
+ if ((rq->cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) {
+ if (rq->cmb & IONIC_CMB_REQUIRE)
goto not_in_cmb;
- qp->rq_cmb &= ~IONIC_CMB_EXPDB;
+ rq->cmb &= ~IONIC_CMB_EXPDB;
}
- qp->rq_cmb_order = order_base_2(qp->rq.size / PAGE_SIZE);
+ rq->cmb_order = order_base_2(rq->q.size / PAGE_SIZE);
- if (qp->rq_cmb_order >= IONIC_RQCMB_ORDER)
+ if (rq->cmb_order >= IONIC_RQCMB_ORDER)
goto not_in_cmb;
- if (qp->rq_cmb & IONIC_CMB_EXPDB)
- expdb_stride_log2 = qp->rq.stride_log2;
+ if (rq->cmb & IONIC_CMB_EXPDB)
+ expdb_stride_log2 = rq->q.stride_log2;
- rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->rq_cmb_pgid,
- &qp->rq_cmb_addr, qp->rq_cmb_order,
+ rc = ionic_get_cmb(dev->lif_cfg.lif, &rq->cmb_pgid,
+ &rq->cmb_addr, rq->cmb_order,
expdb_stride_log2, &expdb);
if (rc)
goto not_in_cmb;
- if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !expdb) {
- if (qp->rq_cmb & IONIC_CMB_REQUIRE)
+ if ((rq->cmb & IONIC_CMB_EXPDB) && !expdb) {
+ if (rq->cmb & IONIC_CMB_REQUIRE)
goto err_map;
- qp->rq_cmb &= ~IONIC_CMB_EXPDB;
+ rq->cmb &= ~IONIC_CMB_EXPDB;
}
return;
err_map:
- ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order);
+ ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order);
not_in_cmb:
- if (qp->rq_cmb & IONIC_CMB_REQUIRE)
+ if (rq->cmb & IONIC_CMB_REQUIRE)
ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n");
- qp->rq_cmb = 0;
- qp->rq_cmb_order = IONIC_RES_INVALID;
- qp->rq_cmb_pgid = 0;
- qp->rq_cmb_addr = 0;
+ rq->cmb = 0;
+ rq->cmb_order = IONIC_RES_INVALID;
+ rq->cmb_pgid = 0;
+ rq->cmb_addr = 0;
}
-static void ionic_qp_rq_destroy_cmb(struct ionic_ibdev *dev,
- struct ionic_ctx *ctx,
- struct ionic_qp *qp)
+static void ionic_rq_destroy_cmb(struct ionic_ibdev *dev,
+ struct ionic_ctx *ctx,
+ struct ionic_rq *rq)
{
- if (!(qp->rq_cmb & IONIC_CMB_ENABLE))
+ if (!(rq->cmb & IONIC_CMB_ENABLE))
return;
if (ctx)
- rdma_user_mmap_entry_remove(qp->mmap_rq_cmb);
+ rdma_user_mmap_entry_remove(rq->mmap_cmb);
- ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order);
+ ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order);
}
-static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
- struct ionic_qp *qp, struct ionic_qdesc *rq,
- struct ionic_tbl_buf *buf, int max_wr, int max_sge,
- int rq_spec, struct ib_udata *udata)
+static int ionic_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
+ struct ionic_rq *rq, struct ionic_qdesc *qdesc,
+ struct ionic_tbl_buf *buf, int max_wr, int max_sge,
+ int rq_spec, struct ib_udata *udata, int access)
{
int rc = 0, i;
u32 wqe_size;
- if (!qp->has_rq) {
- if (buf) {
- buf->tbl_buf = NULL;
- buf->tbl_limit = 0;
- buf->tbl_pages = 0;
- }
- if (udata)
- rc = ionic_validate_qdesc_zero(rq);
-
- return rc;
- }
-
+ rq->cmb_prod = 0;
rc = -EINVAL;
if (max_wr < 0 || max_wr > 0xffff)
@@ -2036,107 +2025,104 @@ static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
return rc;
if (udata) {
- rc = ionic_validate_qdesc(rq);
+ rc = ionic_validate_qdesc(qdesc);
if (rc)
return rc;
- qp->rq_spec = rq_spec;
+ rq->spec = rq_spec;
- qp->rq.ptr = NULL;
- qp->rq.size = rq->size;
- qp->rq.mask = rq->mask;
- qp->rq.depth_log2 = rq->depth_log2;
- qp->rq.stride_log2 = rq->stride_log2;
+ rq->q.ptr = NULL;
+ rq->q.size = qdesc->size;
+ rq->q.mask = qdesc->mask;
+ rq->q.depth_log2 = qdesc->depth_log2;
+ rq->q.stride_log2 = qdesc->stride_log2;
- qp->rq_meta = NULL;
+ rq->meta = NULL;
- qp->rq_umem = ib_umem_get(&dev->ibdev, rq->addr, rq->size, 0);
- if (IS_ERR(qp->rq_umem))
- return PTR_ERR(qp->rq_umem);
+ rq->umem = ib_umem_get(&dev->ibdev, qdesc->addr, qdesc->size, 0);
+ if (IS_ERR(rq->umem))
+ return PTR_ERR(rq->umem);
} else {
- qp->rq_umem = NULL;
+ rq->umem = NULL;
- qp->rq_spec = ionic_v1_use_spec_sge(max_sge, rq_spec);
- if (rq_spec && !qp->rq_spec)
+ rq->spec = ionic_v1_use_spec_sge(max_sge, rq_spec);
+ if (rq_spec && !rq->spec)
ibdev_dbg(&dev->ibdev,
"init rq: max_sge %u disables spec\n",
max_sge);
- if (qp->rq_cmb & IONIC_CMB_EXPDB) {
+ if (rq->cmb & IONIC_CMB_EXPDB) {
wqe_size = ionic_v1_recv_wqe_min_size(max_sge,
- qp->rq_spec,
+ rq->spec,
true);
if (!ionic_expdb_wqe_size_supported(dev, wqe_size))
- qp->rq_cmb &= ~IONIC_CMB_EXPDB;
+ rq->cmb &= ~IONIC_CMB_EXPDB;
}
- if (!(qp->rq_cmb & IONIC_CMB_EXPDB))
+ if (!(rq->cmb & IONIC_CMB_EXPDB))
wqe_size = ionic_v1_recv_wqe_min_size(max_sge,
- qp->rq_spec,
+ rq->spec,
false);
- rc = ionic_queue_init(&qp->rq, dev->lif_cfg.hwdev,
+ rc = ionic_queue_init(&rq->q, dev->lif_cfg.hwdev,
max_wr, wqe_size);
if (rc)
return rc;
- ionic_queue_dbell_init(&qp->rq, qp->qpid);
+ ionic_queue_dbell_init(&rq->q, rq->qid);
- qp->rq_meta = kmalloc_array((u32)qp->rq.mask + 1,
- sizeof(*qp->rq_meta),
- GFP_KERNEL);
- if (!qp->rq_meta) {
+ rq->meta = kmalloc_array((u32)rq->q.mask + 1,
+ sizeof(*rq->meta),
+ GFP_KERNEL);
+ if (!rq->meta) {
rc = -ENOMEM;
goto err_rq_meta;
}
- for (i = 0; i < qp->rq.mask; ++i)
- qp->rq_meta[i].next = &qp->rq_meta[i + 1];
- qp->rq_meta[i].next = IONIC_META_LAST;
- qp->rq_meta_head = &qp->rq_meta[0];
+ for (i = 0; i < rq->q.mask; ++i)
+ rq->meta[i].next = &rq->meta[i + 1];
+ rq->meta[i].next = IONIC_META_LAST;
+ rq->meta_head = &rq->meta[0];
}
- ionic_qp_rq_init_cmb(dev, qp, udata);
+ ionic_rq_init_cmb(dev, rq, udata);
- if (qp->rq_cmb & IONIC_CMB_ENABLE)
+ if (rq->cmb & IONIC_CMB_ENABLE)
rc = ionic_pgtbl_init(dev, buf, NULL,
- (u64)qp->rq_cmb_pgid << PAGE_SHIFT,
+ (u64)rq->cmb_pgid << PAGE_SHIFT,
1, PAGE_SIZE);
else
- rc = ionic_pgtbl_init(dev, buf,
- qp->rq_umem, qp->rq.dma, 1, PAGE_SIZE);
+ rc = ionic_pgtbl_init(dev, buf, rq->umem,
+ rq->q.dma, 1, PAGE_SIZE);
if (rc)
goto err_rq_tbl;
return 0;
err_rq_tbl:
- ionic_qp_rq_destroy_cmb(dev, ctx, qp);
- kfree(qp->rq_meta);
+ ionic_rq_destroy_cmb(dev, ctx, rq);
+ kfree(rq->meta);
err_rq_meta:
- if (qp->rq_umem)
- ib_umem_release(qp->rq_umem);
+ if (rq->umem)
+ ib_umem_release(rq->umem);
else
- ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev);
+ ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev);
return rc;
}
-static void ionic_qp_rq_destroy(struct ionic_ibdev *dev,
- struct ionic_ctx *ctx,
- struct ionic_qp *qp)
+static void ionic_rq_destroy(struct ionic_ibdev *dev,
+ struct ionic_ctx *ctx,
+ struct ionic_rq *rq)
{
- if (!qp->has_rq)
- return;
+ ionic_rq_destroy_cmb(dev, ctx, rq);
- ionic_qp_rq_destroy_cmb(dev, ctx, qp);
+ kfree(rq->meta);
- kfree(qp->rq_meta);
-
- if (qp->rq_umem)
- ib_umem_release(qp->rq_umem);
+ if (rq->umem)
+ ib_umem_release(rq->umem);
else
- ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev);
+ ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev);
}
int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
@@ -2174,7 +2160,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
INIT_LIST_HEAD(&qp->cq_flush_rq);
spin_lock_init(&qp->sq_lock);
- spin_lock_init(&qp->rq_lock);
+ spin_lock_init(&qp->rq.lock);
qp->has_sq = 1;
qp->has_rq = 1;
@@ -2218,7 +2204,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
if (udata) {
if (req.rq_cmb & IONIC_CMB_ENABLE)
- qp->rq_cmb = req.rq_cmb;
+ qp->rq.cmb = req.rq_cmb;
if (req.sq_cmb & IONIC_CMB_ENABLE)
qp->sq_cmb = req.sq_cmb;
@@ -2230,11 +2216,24 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
if (rc)
goto err_sq;
- rc = ionic_qp_rq_init(dev, ctx, qp, &req.rq, &rq_buf,
- attr->cap.max_recv_wr, attr->cap.max_recv_sge,
- req.rq_spec, udata);
- if (rc)
- goto err_rq;
+ if (qp->has_rq) {
+ /* for non-srq qps, rq qid is same as qpid */
+ qp->rq.qid = qp->qpid;
+ rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf,
+ attr->cap.max_recv_wr, attr->cap.max_recv_sge,
+ req.rq_spec, udata,
+ (qp->ibqp.qp_type == IB_QPT_DRIVER) ?
+ IB_ACCESS_LOCAL_WRITE : 0);
+ if (rc)
+ goto err_rq;
+ } else {
+ rq_buf.tbl_buf = NULL;
+ rq_buf.tbl_limit = 0;
+ rq_buf.tbl_pages = 0;
+
+ if (udata)
+ rc = ionic_validate_qdesc_zero(&req.rq);
+ }
rc = ionic_create_qp_cmd(dev, pd,
to_ionic_vcq_cq(attr->send_cq, qp->udma_idx),
@@ -2280,41 +2279,41 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
resp.sq_cmb = qp->sq_cmb;
}
- if (qp->rq_cmb & IONIC_CMB_ENABLE) {
+ if (qp->rq.cmb & IONIC_CMB_ENABLE) {
bool wc;
- if ((qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) ==
+ if ((qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) ==
(IONIC_CMB_WC | IONIC_CMB_UC)) {
ibdev_dbg(&dev->ibdev,
- "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n");
- qp->rq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC);
+ "Both rq.cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n");
+ qp->rq.cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC);
}
- if (qp->rq_cmb & IONIC_CMB_EXPDB)
- wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
+ if (qp->rq.cmb & IONIC_CMB_EXPDB)
+ wc = (qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
== IONIC_CMB_WC;
else
- wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
+ wc = (qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
!= IONIC_CMB_UC;
/* let userspace know the mapping */
if (wc)
- qp->rq_cmb |= IONIC_CMB_WC;
+ qp->rq.cmb |= IONIC_CMB_WC;
else
- qp->rq_cmb |= IONIC_CMB_UC;
+ qp->rq.cmb |= IONIC_CMB_UC;
- qp->mmap_rq_cmb =
+ qp->rq.mmap_cmb =
ionic_mmap_entry_insert(ctx,
- qp->rq.size,
- PHYS_PFN(qp->rq_cmb_addr),
+ qp->rq.q.size,
+ PHYS_PFN(qp->rq.cmb_addr),
wc ? IONIC_MMAP_WC : 0,
&resp.rq_cmb_offset);
- if (!qp->mmap_rq_cmb) {
+ if (!qp->rq.mmap_cmb) {
rc = -ENOMEM;
goto err_mmap_rq;
}
- resp.rq_cmb = qp->rq_cmb;
+ resp.rq_cmb = qp->rq.cmb;
}
rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
@@ -2358,19 +2357,19 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
if (qp->has_rq) {
cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx);
- attr->cap.max_recv_wr = qp->rq.mask;
+ attr->cap.max_recv_wr = qp->rq.q.mask;
attr->cap.max_recv_sge =
- ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2,
- qp->rq_spec,
- qp->rq_cmb & IONIC_CMB_EXPDB);
- qp->rq_cqid = cq->cqid;
+ ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2,
+ qp->rq.spec,
+ qp->rq.cmb & IONIC_CMB_EXPDB);
+ qp->rq.cqid = cq->cqid;
}
return 0;
err_resp:
- if (udata && (qp->rq_cmb & IONIC_CMB_ENABLE))
- rdma_user_mmap_entry_remove(qp->mmap_rq_cmb);
+ if (udata && (qp->rq.cmb & IONIC_CMB_ENABLE))
+ rdma_user_mmap_entry_remove(qp->rq.mmap_cmb);
err_mmap_rq:
if (udata && (qp->sq_cmb & IONIC_CMB_ENABLE))
rdma_user_mmap_entry_remove(qp->mmap_sq_cmb);
@@ -2378,7 +2377,8 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
ionic_destroy_qp_cmd(dev, qp->qpid);
err_cmd:
ionic_pgtbl_unbuf(dev, &rq_buf);
- ionic_qp_rq_destroy(dev, ctx, qp);
+ if (qp->has_rq)
+ ionic_rq_destroy(dev, ctx, &qp->rq);
err_rq:
ionic_pgtbl_unbuf(dev, &sq_buf);
ionic_qp_sq_destroy(dev, ctx, qp);
@@ -2432,15 +2432,15 @@ void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)
if (qp->ibqp.recv_cq) {
cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
- /* Hold the CQ lock and QP rq_lock to set up flush */
+ /* Hold the CQ lock and QP rq.lock to set up flush */
spin_lock_irqsave(&cq->lock, irqflags);
- spin_lock(&qp->rq_lock);
- qp->rq_flush = true;
- if (!ionic_queue_empty(&qp->rq)) {
+ spin_lock(&qp->rq.lock);
+ qp->rq.flush = true;
+ if (!ionic_queue_empty(&qp->rq.q)) {
cq->flush = true;
list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
}
- spin_unlock(&qp->rq_lock);
+ spin_unlock(&qp->rq.lock);
spin_unlock_irqrestore(&cq->lock, irqflags);
}
}
@@ -2506,17 +2506,17 @@ static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)
}
if (qp->has_rq) {
- spin_lock(&qp->rq_lock);
- qp->rq_flush = false;
- qp->rq.prod = 0;
- qp->rq.cons = 0;
- if (qp->rq_meta) {
- for (i = 0; i < qp->rq.mask; ++i)
- qp->rq_meta[i].next = &qp->rq_meta[i + 1];
- qp->rq_meta[i].next = IONIC_META_LAST;
+ spin_lock(&qp->rq.lock);
+ qp->rq.flush = false;
+ qp->rq.q.prod = 0;
+ qp->rq.q.cons = 0;
+ if (qp->rq.meta) {
+ for (i = 0; i < qp->rq.q.mask; ++i)
+ qp->rq.meta[i].next = &qp->rq.meta[i + 1];
+ qp->rq.meta[i].next = IONIC_META_LAST;
}
- qp->rq_meta_head = &qp->rq_meta[0];
- spin_unlock(&qp->rq_lock);
+ qp->rq.meta_head = &qp->rq.meta[0];
+ spin_unlock(&qp->rq.lock);
}
local_irq_restore(irqflags);
@@ -2611,7 +2611,7 @@ int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
attr->cap.max_send_wr = qp->sq.mask;
if (qp->has_rq)
- attr->cap.max_recv_wr = qp->rq.mask;
+ attr->cap.max_recv_wr = qp->rq.q.mask;
init_attr->event_handler = ibqp->event_handler;
init_attr->qp_context = ibqp->qp_context;
@@ -2667,7 +2667,9 @@ int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
spin_unlock_irqrestore(&cq->lock, irqflags);
}
- ionic_qp_rq_destroy(dev, ctx, qp);
+ if (qp->has_rq)
+ ionic_rq_destroy(dev, ctx, &qp->rq);
+
ionic_qp_sq_destroy(dev, ctx, qp);
if (qp->has_ah) {
ionic_put_ahid(dev, qp->ahid);
diff --git a/drivers/infiniband/hw/ionic/ionic_datapath.c b/drivers/infiniband/hw/ionic/ionic_datapath.c
index aa2944887f23..b8fa6b6e5f33 100644
--- a/drivers/infiniband/hw/ionic/ionic_datapath.c
+++ b/drivers/infiniband/hw/ionic/ionic_datapath.c
@@ -33,16 +33,16 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
struct ionic_rq_meta *meta;
struct ionic_v1_wqe *wqe;
- if (!qp->rq_flush)
+ if (!qp->rq.flush)
return 0;
- if (ionic_queue_empty(&qp->rq))
+ if (ionic_queue_empty(&qp->rq.q))
return 0;
- wqe = ionic_queue_at_cons(&qp->rq);
+ wqe = ionic_queue_at_cons(&qp->rq.q);
/* wqe_id must be a valid queue index */
- if (unlikely(wqe->base.wqe_id >> qp->rq.depth_log2)) {
+ if (unlikely(wqe->base.wqe_id >> qp->rq.q.depth_log2)) {
ibdev_warn(qp->ibqp.device,
"flush qp %u recv index %llu invalid\n",
qp->qpid, (unsigned long long)wqe->base.wqe_id);
@@ -50,7 +50,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
}
/* wqe_id must indicate a request that is outstanding */
- meta = &qp->rq_meta[wqe->base.wqe_id];
+ meta = &qp->rq.meta[wqe->base.wqe_id];
if (unlikely(meta->next != IONIC_META_POSTED)) {
ibdev_warn(qp->ibqp.device,
"flush qp %u recv index %llu not posted\n",
@@ -58,7 +58,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
return -EIO;
}
- ionic_queue_consume(&qp->rq);
+ ionic_queue_consume(&qp->rq.q);
memset(wc, 0, sizeof(*wc));
@@ -66,8 +66,8 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
wc->wr_id = meta->wrid;
wc->qp = &qp->ibqp;
- meta->next = qp->rq_meta_head;
- qp->rq_meta_head = meta;
+ meta->next = qp->rq.meta_head;
+ qp->rq.meta_head = meta;
return 1;
}
@@ -137,7 +137,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
u16 vlan_tag;
u8 op;
- if (cqe_qp->rq_flush)
+ if (cqe_qp->rq.flush)
return 0;
qp = cqe_qp;
@@ -146,7 +146,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
/* ignore wqe_id in case of flush error */
if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) {
- cqe_qp->rq_flush = true;
+ cqe_qp->rq.flush = true;
cq->flush = true;
list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
@@ -155,13 +155,13 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
}
/* there had better be something in the recv queue to complete */
- if (ionic_queue_empty(&qp->rq)) {
+ if (ionic_queue_empty(&qp->rq.q)) {
ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid);
return -EIO;
}
/* wqe_id must be a valid queue index */
- if (unlikely(cqe->recv.wqe_id >> qp->rq.depth_log2)) {
+ if (unlikely(cqe->recv.wqe_id >> qp->rq.q.depth_log2)) {
ibdev_warn(&dev->ibdev,
"qp %u recv index %llu invalid\n",
qp->qpid, (unsigned long long)cqe->recv.wqe_id);
@@ -169,7 +169,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
}
/* wqe_id must indicate a request that is outstanding */
- meta = &qp->rq_meta[cqe->recv.wqe_id];
+ meta = &qp->rq.meta[cqe->recv.wqe_id];
if (unlikely(meta->next != IONIC_META_POSTED)) {
ibdev_warn(&dev->ibdev,
"qp %u recv index %llu not posted\n",
@@ -177,8 +177,8 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
return -EIO;
}
- meta->next = qp->rq_meta_head;
- qp->rq_meta_head = meta;
+ meta->next = qp->rq.meta_head;
+ qp->rq.meta_head = meta;
memset(wc, 0, sizeof(*wc));
@@ -190,7 +190,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
wc->vendor_err = st_len;
wc->status = ionic_to_ib_status(st_len);
- cqe_qp->rq_flush = true;
+ cqe_qp->rq.flush = true;
cq->flush = true;
list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
@@ -254,7 +254,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
wc->port_num = 1;
out:
- ionic_queue_consume(&qp->rq);
+ ionic_queue_consume(&qp->rq.q);
return 1;
}
@@ -525,9 +525,9 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev,
switch (type) {
case IONIC_V1_CQE_TYPE_RECV:
- spin_lock(&qp->rq_lock);
+ spin_lock(&qp->rq.lock);
rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled);
- spin_unlock(&qp->rq_lock);
+ spin_unlock(&qp->rq.lock);
if (rc < 0)
goto out;
@@ -615,9 +615,9 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev,
if (npolled == nwc)
goto out;
- spin_lock(&qp->rq_lock);
+ spin_lock(&qp->rq.lock);
rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled);
- spin_unlock(&qp->rq_lock);
+ spin_unlock(&qp->rq.lock);
if (rc > 0)
npolled += rc;
@@ -880,7 +880,7 @@ static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe)
static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe)
{
- memset(wqe, 0, 1u << qp->rq.stride_log2);
+ memset(wqe, 0, 1u << qp->rq.q.stride_log2);
}
static int ionic_prep_send(struct ionic_qp *qp,
@@ -1182,38 +1182,38 @@ static int ionic_prep_recv(struct ionic_qp *qp,
s64 signed_len;
u32 mval;
- wqe = ionic_queue_at_prod(&qp->rq);
+ wqe = ionic_queue_at_prod(&qp->rq.q);
/* if wqe is owned by device, caller can try posting again soon */
if (wqe->base.flags & cpu_to_be16(IONIC_V1_FLAG_FENCE))
return -EAGAIN;
- meta = qp->rq_meta_head;
+ meta = qp->rq.meta_head;
if (unlikely(meta == IONIC_META_LAST) ||
unlikely(meta == IONIC_META_POSTED))
return -EIO;
ionic_prep_rq_wqe(qp, wqe);
- mval = ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, qp->rq_spec,
+ mval = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec,
false);
signed_len = ionic_prep_pld(wqe, &wqe->recv.pld,
- qp->rq_spec, mval,
+ qp->rq.spec, mval,
wr->sg_list, wr->num_sge);
if (signed_len < 0)
return signed_len;
meta->wrid = wr->wr_id;
- wqe->base.wqe_id = meta - qp->rq_meta;
+ wqe->base.wqe_id = meta - qp->rq.meta;
wqe->base.num_sge_key = wr->num_sge;
/* total length for recv goes in base imm_data_key */
wqe->base.imm_data_key = cpu_to_be32(signed_len);
- ionic_queue_produce(&qp->rq);
+ ionic_queue_produce(&qp->rq.q);
- qp->rq_meta_head = meta->next;
+ qp->rq.meta_head = meta->next;
meta->next = IONIC_META_POSTED;
return 0;
@@ -1320,10 +1320,10 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev,
return -EINVAL;
}
- spin_lock_irqsave(&qp->rq_lock, irqflags);
+ spin_lock_irqsave(&qp->rq.lock, irqflags);
while (wr) {
- if (ionic_queue_full(&qp->rq)) {
+ if (ionic_queue_full(&qp->rq.q)) {
ibdev_dbg(&dev->ibdev, "queue full");
rc = -ENOMEM;
goto out;
@@ -1338,32 +1338,32 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev,
out:
if (!cq) {
- spin_unlock_irqrestore(&qp->rq_lock, irqflags);
+ spin_unlock_irqrestore(&qp->rq.lock, irqflags);
goto out_unlocked;
}
- spin_unlock_irqrestore(&qp->rq_lock, irqflags);
+ spin_unlock_irqrestore(&qp->rq.lock, irqflags);
spin_lock_irqsave(&cq->lock, irqflags);
- spin_lock(&qp->rq_lock);
+ spin_lock(&qp->rq.lock);
- if (likely(qp->rq.prod != qp->rq_old_prod)) {
+ if (likely(qp->rq.q.prod != qp->rq.old_prod)) {
/* ring cq doorbell just in time */
- spend = (qp->rq.prod - qp->rq_old_prod) & qp->rq.mask;
+ spend = (qp->rq.q.prod - qp->rq.old_prod) & qp->rq.q.mask;
ionic_reserve_cq(dev, cq, spend);
- qp->rq_old_prod = qp->rq.prod;
+ qp->rq.old_prod = qp->rq.q.prod;
ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.rq_qtype,
- ionic_queue_dbell_val(&qp->rq));
+ ionic_queue_dbell_val(&qp->rq.q));
}
- if (qp->rq_flush) {
+ if (qp->rq.flush) {
notify = true;
cq->flush = true;
list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
}
- spin_unlock(&qp->rq_lock);
+ spin_unlock(&qp->rq.lock);
spin_unlock_irqrestore(&cq->lock, irqflags);
if (notify && vcq->ibcq.comp_handler)
diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h
index 82fda1e3cdb6..5517638d4ced 100644
--- a/drivers/infiniband/hw/ionic/ionic_ibdev.h
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h
@@ -240,6 +240,31 @@ struct ionic_rq_meta {
u64 wrid;
};
+struct ionic_rq {
+ struct ionic_queue q;
+ u32 qid;
+ u32 cqid;
+ struct list_head cq_flush;
+ spinlock_t lock; /* for posting and polling */
+
+ phys_addr_t cmb_addr;
+ void __iomem *cmb_ptr;
+ struct rdma_user_mmap_entry *mmap_cmb;
+
+ struct ionic_rq_meta *meta;
+ struct ionic_rq_meta *meta_head;
+ u16 *meta_idx;
+ struct ib_umem *umem;
+
+ int spec;
+ int cmb_order;
+ u32 cmb_pgid;
+ u16 cmb_prod;
+ u16 old_prod;
+ u8 cmb;
+ bool flush;
+};
+
struct ionic_qp {
struct ib_qp ibqp;
enum ib_qp_state state;
@@ -247,7 +272,6 @@ struct ionic_qp {
u32 qpid;
u32 ahid;
u32 sq_cqid;
- u32 rq_cqid;
u8 udma_idx;
u8 has_ah:1;
u8 has_sq:1;
@@ -273,14 +297,7 @@ struct ionic_qp {
bool sq_flush;
bool sq_flush_rcvd;
- spinlock_t rq_lock; /* for posting and polling */
- struct ionic_queue rq;
- struct ionic_rq_meta *rq_meta;
- struct ionic_rq_meta *rq_meta_head;
- int rq_spec;
- u16 rq_old_prod;
- u8 rq_cmb;
- bool rq_flush;
+ struct ionic_rq rq;
struct kref qp_kref;
struct completion qp_rel_comp;
@@ -294,13 +311,6 @@ struct ionic_qp {
struct ib_umem *sq_umem;
- int rq_cmb_order;
- u32 rq_cmb_pgid;
- phys_addr_t rq_cmb_addr;
- struct rdma_user_mmap_entry *mmap_rq_cmb;
-
- struct ib_umem *rq_umem;
-
int dcqcn_profile;
struct ib_ud_header *hdr;
--
2.43.0