[PATCH 07/11] RDMA/cxi: add VNI-isolated queue pairs

[Doug Ledford]

Add queue pairs for the RU transport.  A QP is a portal table entry
plus the command and event queues behind it, created in the INIT
state and enabled on the transition to RTR.  Messages are matched on
the packet VNI as well as the key, so two jobs sharing a NIC cannot
match each other's receive buffers or memory regions.  The VNI match
is enabled only on Cassini 2 with the default VNI matching selector;
other configurations are rejected.

modify_qp() follows the UET v2 flow: an RU QP is created ready to send
and the initial modify goes through the v2 path.

If copying the response to user space fails on INIT to RTR after the
PTE has been mapped, the PTE is unmapped and its map index reset.
Without that a retried transition mapped the PTE a second time and
leaked the first slot in the portal list.

Responses use ib_respond_udata().

checkpatch --strict (100 columns) still reports the following in this
patch. They need coordination with the internal repositories this code
is maintained in and will be fixed in V2:
- CHECK: PARENTHESIS_ALIGNMENT (6), PREFER_KERNEL_TYPES (1)

Co-developed-by: Mamila Santosh <santosh-kumar.mamila@xxxxxxx>
Signed-off-by: Mamila Santosh <santosh-kumar.mamila@xxxxxxx>
Co-developed-by: Iraimani P <iraimani.pavadai@xxxxxxx>
Signed-off-by: Iraimani P <iraimani.pavadai@xxxxxxx>
Signed-off-by: Doug Ledford <doug.ledford@xxxxxxx>
---
 drivers/infiniband/hw/cxi/cxiverbs_qp.c | 949 ++++++++++++++++++++++++++++++++
 1 file changed, 949 insertions(+)

diff --git a/drivers/infiniband/hw/cxi/cxiverbs_qp.c b/drivers/infiniband/hw/cxi/cxiverbs_qp.c
new file mode 100644
index 000000000000..3875a5de0a98
--- /dev/null
+++ b/drivers/infiniband/hw/cxi/cxiverbs_qp.c
@@ -0,0 +1,949 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CXI InfiniBand RDMA driver — Queue Pair lifecycle (Phase 4).
+ * Copyright 2025 Hewlett Packard Enterprise Development LP
+ *
+ * Each UEC/CXI Queue Pair allocates the following hardware resources:
+ *
+ *   cxi_domain   — VNI + PID  (fabric endpoint address)
+ *   cxi_cp       — Communication Profile (maps to LCID / traffic class)
+ *   cxi_cq (TXQ) — transmit command queue (initiator path)
+ *   cxi_cq (TGQ) — target command queue   (receive path)
+ *   cxi_pte      — Portal Table Entry     (receive matching)
+ *
+ * QPN encoding:
+ *   bit  31     : addr_mode (1 = absolute, 0 = job-relative)
+ *   bits [23:12]: Resource Index (PTE pid_offset within the PID slice)
+ *   bits [11:0] : PIDonFEP (process-endpoint address)
+ *
+ * QP state machine:
+ *   RESET  -> INIT  : validate parameters; allocate CXI resources
+ *   INIT   -> RTR   : map PTE + insert catch-all receive LE
+ *   RTR    -> RTS   : arm transmit queue
+ *   RTS/RTR -> ERROR: disable PTE and drain TXQ
+ *   any    -> RESET : free all sub-resources
+ */
+
+#include <linux/slab.h>
+#include <linux/gfp.h>
+#include <linux/mm.h>
+#include <linux/random.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/uverbs_ioctl.h>
+
+#include "cxiverbs.h"
+
+/* Work request queue depth limits */
+#define CXI_MAX_SEND_WR    1024
+#define CXI_MAX_RECV_WR    1024
+#define CXI_MAX_SEND_SGE   1
+#define CXI_MAX_RECV_SGE   1
+#define CXI_MAX_INLINE     192   /* IDC fast-path maximum bytes */
+
+/* TXQ ring buffer entry count (each entry = C_CQ_CMD_SIZE = 64 bytes) */
+#define CXI_TXQ_DEFAULT_ENTRIES 512
+
+int cxiverbs_query_qp_semantics(struct ib_device *device,
+			       enum ib_qp_type type,
+			       struct ib_qp_semantics *sem)
+{
+	if (type != IB_QPT_RU)
+		return -EOPNOTSUPP;
+	memset(sem, 0, sizeof(*sem));
+	sem->comp_mask = IB_UVERBS_QP_SEMANTICS_MASK_MSG_ORDER |
+		IB_UVERBS_QP_SEMANTICS_MASK_RAW |
+		IB_UVERBS_QP_SEMANTICS_MASK_WAR |
+		IB_UVERBS_QP_SEMANTICS_MASK_WAW |
+		IB_UVERBS_QP_SEMANTICS_MASK_PDU |
+		IB_UVERBS_QP_SEMANTICS_MASK_IMM |
+		IB_UVERBS_QP_SEMANTICS_MASK_USAGE;
+	sem->max_pdu = ib_mtu_enum_to_int(IB_MTU_4096);
+	sem->imm_data_size = 64;
+	sem->usage_flags = IB_UVERBS_QP_USAGE_IMM_DATA_RQ;
+	return 0;
+}
+
+/* =======================================================================
+ * Resource teardown helper
+ * =======================================================================
+ */
+
+struct cxiverbs_qp_resources {
+	refcount_t refs;
+	struct cxi_adev *cadev;
+	struct ib_pd *pd;
+	struct ib_cq *send_cq;
+	struct ib_cq *recv_cq;
+	struct cxi_domain *domain;
+	struct cxi_cp *cp;
+	struct cxi_cq *txq;
+	struct cxi_cq *tgq;
+	struct cxi_pte *pte;
+	int pt_map_index;
+	u32 ri;
+};
+
+static void cxiverbs_qp_resources_put(void *owner)
+{
+	struct cxiverbs_qp_resources *resources = owner;
+	struct cxi_adev *cadev = resources->cadev;
+	struct cxiverbs_pd *pd = to_cxiverbs_pd(resources->pd);
+
+	if (!refcount_dec_and_test(&resources->refs))
+		return;
+
+	cadev->pte_unmap(resources->pte, resources->domain,
+			 resources->pt_map_index);
+	cadev->cxi_pte_free(resources->pte);
+	cadev->cq_free(resources->tgq);
+	cadev->cq_free(resources->txq);
+	cadev->cp_free(resources->cp);
+	cadev->domain_free(resources->domain);
+	xa_erase(&pd->qpns, resources->ri);
+	atomic_dec(&resources->send_cq->usecnt);
+	if (resources->recv_cq)
+		atomic_dec(&resources->recv_cq->usecnt);
+	atomic_dec(&resources->pd->usecnt);
+	kfree(resources);
+}
+
+static struct cxiverbs_mmap_entry *
+cxiverbs_qp_mmap(struct cxiverbs_qp *qp, struct ib_ucontext *ibctx,
+		 bool is_io, phys_addr_t phys, struct page *first_page,
+		 size_t size, u64 *offset)
+{
+	struct cxiverbs_mmap_entry *entry;
+
+	refcount_inc(&qp->resources->refs);
+	entry = cxiverbs_mmap_entry_alloc_owned(ibctx, is_io, phys, first_page,
+						size, offset, qp->resources,
+						cxiverbs_qp_resources_put);
+	if (IS_ERR(entry))
+		cxiverbs_qp_resources_put(qp->resources);
+	return entry;
+}
+
+static void cxiverbs_qp_free_resources(struct cxiverbs_qp *qp)
+{
+	struct cxiverbs_dev *dev = to_cxiverbs_dev(qp->ibqp.device);
+
+	/* Remove user mmap entries (they hold references) */
+	if (qp->tgq_db_mmap) {
+		rdma_user_mmap_entry_remove(&qp->tgq_db_mmap->rdma_entry);
+		qp->tgq_db_mmap = NULL;
+	}
+	if (qp->tgq_cmds_mmap) {
+		rdma_user_mmap_entry_remove(&qp->tgq_cmds_mmap->rdma_entry);
+		qp->tgq_cmds_mmap = NULL;
+	}
+	if (qp->txq_db_mmap) {
+		rdma_user_mmap_entry_remove(&qp->txq_db_mmap->rdma_entry);
+		qp->txq_db_mmap = NULL;
+	}
+	if (qp->txq_cmds_mmap) {
+		rdma_user_mmap_entry_remove(&qp->txq_cmds_mmap->rdma_entry);
+		qp->txq_cmds_mmap = NULL;
+	}
+
+	if (qp->resources) {
+		cxiverbs_qp_resources_put(qp->resources);
+		qp->resources = NULL;
+		qp->pte = NULL;
+		qp->tgq = NULL;
+		qp->txq = NULL;
+		qp->cp = NULL;
+		qp->domain = NULL;
+		qp->lni = NULL;
+		qp->identity_reserved = false;
+		return;
+	}
+
+	/* Unmap PTE from domain before freeing */
+	if (qp->pte) {
+		if (qp->pt_map_index >= 0 && qp->domain)
+			dev->cadev->pte_unmap(qp->pte, qp->domain,
+					      qp->pt_map_index);
+		dev->cadev->cxi_pte_free(qp->pte);
+		qp->pte = NULL;
+	}
+
+	if (qp->tgq) {
+		dev->cadev->cq_free(qp->tgq);
+		qp->tgq = NULL;
+	}
+	if (qp->txq) {
+		dev->cadev->cq_free(qp->txq);
+		qp->txq = NULL;
+	}
+	if (qp->cp) {
+		dev->cadev->cp_free(qp->cp);
+		qp->cp = NULL;
+	}
+	if (qp->domain) {
+		dev->cadev->domain_free(qp->domain);
+		qp->domain = NULL;
+	}
+	qp->lni = NULL;
+	if (qp->identity_reserved) {
+		struct cxiverbs_pd *pd = to_cxiverbs_pd(qp->ibqp.pd);
+
+		xa_erase(&pd->qpns, qp->ri);
+		qp->identity_reserved = false;
+	}
+}
+
+/* =======================================================================
+ * create_qp
+ * =======================================================================
+ */
+
+int cxiverbs_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
+		      struct ib_udata *udata)
+{
+	struct ib_pd *ibpd             = ibqp->pd;
+	struct cxiverbs_pd *pd;
+	struct cxiverbs_dev *dev;
+	struct cxiverbs_cq *send_cq;
+	struct cxiverbs_qp *qp           = to_cxiverbs_qp(ibqp);
+	struct cxi_ibv_create_qp_cmd cmd = {};
+	struct cxi_ibv_create_qp_resp resp = {};
+	struct cxi_cq_alloc_opts txq_opts = {};
+	struct cxi_cq_alloc_opts tgq_opts = {};
+	struct cxi_pt_alloc_opts pte_opts = {};
+	size_t txq_cmds_size;
+	struct page *txq_cmds_pages = NULL;
+	phys_addr_t txq_db_phys;
+	size_t txq_db_size;
+	size_t tgq_cmds_size;
+	struct page *tgq_cmds_pages = NULL;
+	phys_addr_t tgq_db_phys;
+	size_t tgq_db_size;
+	u32 requested_pid = init_attr->source_qpn & CXI_QPN_PID_MASK;
+	u32 requested_ri = FIELD_GET(CXI_QPN_RI_MASK, init_attr->source_qpn);
+	u32 requested_addr_mode = init_attr->source_qpn & CXI_QPN_ADDR_MODE_MASK;
+	u32 svc_id;
+	u32 resource_index;
+	u32 ri;
+	u32 tc = CXI_TC_BEST_EFFORT;
+	int pid;
+	int ret;
+	unsigned int pt_index;
+
+	if (!udata || !ibqp->uobject || !ibpd || !ibpd->uobject)
+		return -EOPNOTSUPP;
+
+	pd = to_cxiverbs_pd(ibpd);
+	dev = to_cxiverbs_dev(ibpd->device);
+	send_cq = to_cxiverbs_cq(init_attr->send_cq);
+
+	if (!(dev->cdev->prop.cassini_version & CASSINI_2) ||
+	    dev->cdev->prop.vni_matching != 0)
+		return -EOPNOTSUPP;
+
+	if (!pd->lni_ref || !pd->pidonfep_valid || !init_attr->jkey ||
+	    init_attr->jkey->pd != ibpd)
+		return -EINVAL;
+	if (send_cq->lni_ref != pd->lni_ref ||
+	    (init_attr->recv_cq &&
+	     to_cxiverbs_cq(init_attr->recv_cq)->lni_ref != pd->lni_ref))
+		return -EINVAL;
+
+	if (init_attr->qp_type != IB_QPT_RU)
+		return -EOPNOTSUPP;
+	if (init_attr->qp_semantics) {
+		const struct ib_qp_semantics *sem = init_attr->qp_semantics;
+
+		if (sem->msg_order || sem->max_rdma_raw_size ||
+		    sem->max_rdma_war_size || sem->max_rdma_waw_size ||
+		    (sem->max_pdu &&
+		     sem->max_pdu != ib_mtu_enum_to_int(IB_MTU_4096)) ||
+		    sem->imm_data_size > 64 ||
+		    (sem->usage_flags & ~IB_UVERBS_QP_USAGE_IMM_DATA_RQ))
+			return -EOPNOTSUPP;
+	}
+
+	if (udata->inlen >= sizeof(cmd)) {
+		ret = ib_copy_from_udata(&cmd, udata, sizeof(cmd));
+		if (ret)
+			return ret;
+		if (cmd.comp_mask & ~(CXI_IBV_CREATE_QP_SVC_ID |
+				      CXI_IBV_CREATE_QP_TC))
+			return -EOPNOTSUPP;
+		if ((cmd.comp_mask & CXI_IBV_CREATE_QP_SVC_ID) &&
+		    (!cmd.svc_id || cmd.svc_id > CXI_IBV_SERVICE_ID_MAX))
+			return -EINVAL;
+		if (cmd.comp_mask & CXI_IBV_CREATE_QP_TC)
+			tc = cmd.tc;
+	}
+
+	if (init_attr->create_flags & ~IB_QP_CREATE_SOURCE_QPN)
+		return -EOPNOTSUPP;
+
+	if (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN) {
+		if (init_attr->source_qpn &
+		    ~(CXI_QPN_ADDR_MODE_MASK | CXI_QPN_RI_MASK | CXI_QPN_PID_MASK))
+			return -EINVAL;
+		if (requested_addr_mode != pd->qpn_addr_mode)
+			return -EINVAL;
+		if (requested_pid != pd->pidonfep)
+			return -EINVAL;
+	}
+
+	/*
+	 * The Resource Index selects which receive bucket of this PD's PID
+	 * slice remote peers address (the PTE pid_offset). It is chosen by the
+	 * application and is independent of the service ID, which only names
+	 * the resource-limit pool the LNI draws from.
+	 */
+	ri = (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN) ?
+		requested_ri : 0;
+	if (dev->pid_granule && ri >= dev->pid_granule)
+		return -EINVAL;
+
+	svc_id = (cmd.comp_mask & CXI_IBV_CREATE_QP_SVC_ID) ? cmd.svc_id :
+		 pd->preferred_svc_id;
+	if (!svc_id)
+		svc_id = pd->lni_ref->svc_id;
+	if (svc_id != pd->lni_ref->svc_id)
+		return -EINVAL;
+	qp->lni = pd->lni_ref->lni;
+
+	spin_lock_init(&qp->lock);
+	mutex_init(&qp->modify_lock);
+	qp->state           = IB_QPS_RESET;
+	qp->pt_map_index    = -1;
+	qp->max_send_wr     = min_t(u32, init_attr->cap.max_send_wr, CXI_MAX_SEND_WR);
+	qp->max_recv_wr     = min_t(u32, init_attr->cap.max_recv_wr, CXI_MAX_RECV_WR);
+	qp->max_send_sge    = min_t(u32, init_attr->cap.max_send_sge, CXI_MAX_SEND_SGE);
+	qp->max_recv_sge    = min_t(u32, init_attr->cap.max_recv_sge, CXI_MAX_RECV_SGE);
+	qp->max_inline_data = min_t(u32, init_attr->cap.max_inline_data, CXI_MAX_INLINE);
+	qp->vni             = init_attr->jkey->job->job_id;
+	qp->svc_id          = svc_id;
+	qp->ri              = ri;
+	qp->tc              = tc;
+
+	mutex_lock(&pd->resource_lock);
+	if (pd->job_id && pd->job_id != qp->vni) {
+		ret = -EINVAL;
+		goto err_unlock_pd;
+	}
+	if (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN)
+		ret = xa_insert(&pd->qpns, ri, qp, GFP_KERNEL);
+	else
+		ret = xa_alloc(&pd->qpns, &ri, qp,
+			       XA_LIMIT(0, dev->pid_granule - 1), GFP_KERNEL);
+	if (ret)
+		goto err_unlock_pd;
+	pd->job_id = qp->vni;
+	qp->ri = ri;
+	qp->identity_reserved = true;
+	pid = pd->pidonfep;
+	resource_index = FIELD_PREP(CXI_QPN_RI_MASK, ri);
+	mutex_unlock(&pd->resource_lock);
+
+	qp->domain = dev->cadev->domain_alloc(qp->lni, qp->vni, (unsigned int)pid);
+	if (IS_ERR(qp->domain)) {
+		ret = PTR_ERR(qp->domain);
+		qp->domain = NULL;
+		goto err_identity;
+	}
+
+	/* Step 2: Allocate a Communication Profile (LCID + traffic class) */
+	qp->cp = dev->cadev->cp_alloc(qp->lni, qp->vni, qp->tc, CXI_TC_TYPE_DEFAULT);
+	if (IS_ERR(qp->cp)) {
+		ret = PTR_ERR(qp->cp);
+		qp->cp = NULL;
+		goto err_free_domain;
+	}
+	qp->lcid = qp->cp->lcid;
+
+	/* Step 3: Allocate transmit (initiator) CQ.
+	 * cxi-driver internally allocates the ring buffer for user CQs.
+	 */
+	txq_opts.count = CXI_TXQ_DEFAULT_ENTRIES;
+	txq_opts.flags = CXI_CQ_IS_TX | CXI_CQ_USER;
+	txq_opts.lcid  = qp->lcid;
+
+	qp->txq = dev->cadev->cq_alloc(qp->lni, send_cq->eq, &txq_opts, NUMA_NO_NODE);
+	if (IS_ERR(qp->txq)) {
+		ret = PTR_ERR(qp->txq);
+		qp->txq = NULL;
+		goto err_free_cp;
+	}
+
+	/* Step 4: Allocate target (receive) CQ */
+	tgq_opts.count = CXI_TXQ_DEFAULT_ENTRIES;
+	tgq_opts.flags = CXI_CQ_USER;
+
+	if (init_attr->recv_cq) {
+		struct cxiverbs_cq *recv_cq = to_cxiverbs_cq(init_attr->recv_cq);
+
+		qp->tgq = dev->cadev->cq_alloc(qp->lni, recv_cq->eq, &tgq_opts,
+					NUMA_NO_NODE);
+	} else {
+		qp->tgq = dev->cadev->cq_alloc(qp->lni, send_cq->eq, &tgq_opts,
+					NUMA_NO_NODE);
+	}
+	if (IS_ERR(qp->tgq)) {
+		ret = PTR_ERR(qp->tgq);
+		qp->tgq = NULL;
+		goto err_free_txq;
+	}
+
+	/* Step 5: Allocate the Portal Table Entry */
+	/*
+	 * Matching, so an RMA operation can select a region by its rkey. SEND
+	 * uses match bits of zero, which no rkey takes.
+	 */
+	pte_opts.is_matching    = 1;
+	pte_opts.en_match_on_vni = 1;
+	pte_opts.use_long_event = 0;
+
+	qp->pte = dev->cadev->cxi_pte_alloc(qp->lni, send_cq->eq, &pte_opts);
+	if (IS_ERR(qp->pte)) {
+		ret = PTR_ERR(qp->pte);
+		qp->pte = NULL;
+		goto err_free_tgq;
+	}
+	ret = dev->cadev->pte_map(qp->pte, qp->domain, qp->ri,
+				false, &pt_index);
+	if (ret)
+		goto err_free_pte;
+	qp->pt_map_index = pt_index;
+
+	/* Step 6: Get TXQ mmap info from cxi-driver */
+	ret = dev->cadev->cq_user_info(qp->txq, &txq_cmds_size, &txq_cmds_pages,
+			       &txq_db_phys, &txq_db_size);
+	if (ret)
+		goto err_free_pte;
+
+	/* Step 7: Get TGQ mmap info from cxi-driver */
+	ret = dev->cadev->cq_user_info(qp->tgq, &tgq_cmds_size, &tgq_cmds_pages,
+			       &tgq_db_phys, &tgq_db_size);
+	if (ret)
+		goto err_free_pte;
+
+	/* Register mmap entries for user-space direct submission */
+	qp->resources = kzalloc_obj(*qp->resources);
+	if (!qp->resources) {
+		ret = -ENOMEM;
+		goto err_free_pte;
+	}
+	refcount_set(&qp->resources->refs, 1);
+	qp->resources->cadev = dev->cadev;
+	qp->resources->pd = ibpd;
+	qp->resources->send_cq = init_attr->send_cq;
+	qp->resources->recv_cq = init_attr->recv_cq;
+	qp->resources->domain = qp->domain;
+	qp->resources->cp = qp->cp;
+	qp->resources->txq = qp->txq;
+	qp->resources->tgq = qp->tgq;
+	qp->resources->pte = qp->pte;
+	qp->resources->pt_map_index = qp->pt_map_index;
+	qp->resources->ri = qp->ri;
+	atomic_inc(&ibpd->usecnt);
+	atomic_inc(&init_attr->send_cq->usecnt);
+	if (init_attr->recv_cq)
+		atomic_inc(&init_attr->recv_cq->usecnt);
+
+	{
+		struct ib_ucontext *ibctx = ibqp->pd->uobject->context;
+
+		qp->txq_cmds_mmap = cxiverbs_qp_mmap(qp, ibctx, false, 0,
+						     txq_cmds_pages, txq_cmds_size,
+						     &resp.txq_cmds_offset);
+		if (IS_ERR(qp->txq_cmds_mmap)) {
+			ret = PTR_ERR(qp->txq_cmds_mmap);
+			qp->txq_cmds_mmap = NULL;
+			goto err_free_resources;
+		}
+
+		qp->txq_db_mmap = cxiverbs_qp_mmap(qp, ibctx, true, txq_db_phys,
+						   NULL, txq_db_size, &resp.txq_db_offset);
+		if (IS_ERR(qp->txq_db_mmap)) {
+			ret = PTR_ERR(qp->txq_db_mmap);
+			qp->txq_db_mmap = NULL;
+			goto err_free_resources;
+		}
+
+		qp->tgq_cmds_mmap = cxiverbs_qp_mmap(qp, ibctx, false, 0,
+						     tgq_cmds_pages, tgq_cmds_size,
+						     &resp.tgq_cmds_offset);
+		if (IS_ERR(qp->tgq_cmds_mmap)) {
+			ret = PTR_ERR(qp->tgq_cmds_mmap);
+			qp->tgq_cmds_mmap = NULL;
+			goto err_free_resources;
+		}
+
+		qp->tgq_db_mmap = cxiverbs_qp_mmap(qp, ibctx, true, tgq_db_phys,
+						   NULL, tgq_db_size, &resp.tgq_db_offset);
+		if (IS_ERR(qp->tgq_db_mmap)) {
+			ret = PTR_ERR(qp->tgq_db_mmap);
+			qp->tgq_db_mmap = NULL;
+			goto err_free_resources;
+		}
+	}
+
+	/*
+	 * QPN encoding:
+	 *   bits [11:0]  = PIDonFEP   (filled now)
+	 *   bits [23:12] = Resource Index (PTE pid_offset)
+	 *   bit  31      = addr_mode  (0 = job-relative by default)
+	 */
+	qp->qpn_base = (pd->pidonfep & CXI_QPN_PID_MASK) |
+			resource_index |
+			pd->qpn_addr_mode;
+	ibqp->qp_num = qp->qpn_base;
+
+	resp.txq_cmds_size = txq_cmds_size;
+	resp.txq_db_size   = txq_db_size;
+	resp.txq_idx       = cxi_cq_get_cqn(qp->txq);
+	resp.lcid          = qp->lcid;
+	resp.tgq_cmds_size = tgq_cmds_size;
+	resp.tgq_db_size   = tgq_db_size;
+	resp.tgq_idx       = cxi_cq_get_cqn(qp->tgq);
+	resp.job_id        = qp->vni;
+	resp.pte_index     = qp->pte->id;
+
+	ret = ib_respond_udata(udata, resp);
+	if (ret)
+		goto err_free_resources;
+
+	return 0;
+
+err_free_resources:
+	cxiverbs_qp_free_resources(qp);
+	return ret;
+err_free_pte:
+	if (qp->pt_map_index >= 0) {
+		dev->cadev->pte_unmap(qp->pte, qp->domain, qp->pt_map_index);
+		qp->pt_map_index = -1;
+	}
+	dev->cadev->cxi_pte_free(qp->pte);
+	qp->pte = NULL;
+err_free_tgq:
+	dev->cadev->cq_free(qp->tgq);
+	qp->tgq = NULL;
+err_free_txq:
+	dev->cadev->cq_free(qp->txq);
+	qp->txq = NULL;
+err_free_cp:
+	dev->cadev->cp_free(qp->cp);
+	qp->cp = NULL;
+err_free_domain:
+	dev->cadev->domain_free(qp->domain);
+	qp->domain = NULL;
+err_identity:
+	xa_erase(&pd->qpns, qp->ri);
+	qp->identity_reserved = false;
+	return ret;
+err_unlock_pd:
+	mutex_unlock(&pd->resource_lock);
+	return ret;
+}
+
+/* =======================================================================
+ * destroy_qp
+ * =======================================================================
+ */
+
+int cxiverbs_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
+{
+	struct cxiverbs_qp *qp = to_cxiverbs_qp(ibqp);
+
+	cxiverbs_qp_free_resources(qp);
+	mutex_destroy(&qp->modify_lock);
+	return 0;
+}
+
+/* =======================================================================
+ * modify_qp  —  UEC-aligned state machine
+ * =======================================================================
+ */
+
+int cxiverbs_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		      int attr_mask, struct ib_udata *udata)
+{
+	struct cxiverbs_qp *qp = to_cxiverbs_qp(ibqp);
+	struct cxiverbs_dev *dev = to_cxiverbs_dev(ibqp->device);
+	enum ib_qp_state cur_state, new_state;
+	int ret = -EINVAL;
+
+	if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_PORT |
+			  IB_QP_PKEY_INDEX | IB_QP_ACCESS_FLAGS | IB_QP_PATH_MTU))
+		return -EOPNOTSUPP;
+	if (((attr_mask & IB_QP_PORT) && attr->port_num != 1) ||
+	    ((attr_mask & IB_QP_PKEY_INDEX) && attr->pkey_index))
+		return -EINVAL;
+	if (((attr_mask & IB_QP_PATH_MTU) && attr->path_mtu != IB_MTU_4096) ||
+	    ((attr_mask & IB_QP_ACCESS_FLAGS) && attr->qp_access_flags))
+		return -EOPNOTSUPP;
+
+	/*
+	 * uverbs only takes a read reference on the QP for modify, so several
+	 * modifies can run at once. Hold modify_lock for the whole transition:
+	 * the state read below then stays valid until it is published, and the
+	 * portal table map step, which sleeps and so cannot run under the
+	 * spinlock, is not repeated or raced. qp->lock only publishes the state
+	 * to post_send(), post_recv() and query_qp().
+	 */
+	mutex_lock(&qp->modify_lock);
+
+	spin_lock_irq(&qp->lock);
+	cur_state = qp->state;
+	spin_unlock_irq(&qp->lock);
+	new_state = (attr_mask & IB_QP_STATE) ? attr->qp_state : cur_state;
+	if ((attr_mask & IB_QP_CUR_STATE) && attr->cur_qp_state != cur_state) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	switch (cur_state) {
+	case IB_QPS_RESET:
+		if (new_state == IB_QPS_RESET || new_state == IB_QPS_INIT ||
+		    new_state == IB_QPS_RTS) {
+			spin_lock_irq(&qp->lock);
+			qp->state = new_state;
+			spin_unlock_irq(&qp->lock);
+			ret = 0;
+		}
+		break;
+
+	case IB_QPS_INIT:
+		if (new_state == IB_QPS_RTR) {
+			struct cxi_ibv_modify_qp_resp resp = {};
+			unsigned int pt_index;
+			bool mapped = false;
+
+			if (qp->pt_map_index < 0) {
+				ret = dev->cadev->pte_map(qp->pte, qp->domain,
+							qp->ri, false, &pt_index);
+				if (ret)
+					goto out;
+				qp->pt_map_index = (int)pt_index;
+				mapped = true;
+			}
+			pt_index = qp->pt_map_index;
+
+			ibqp->qp_num = qp->qpn_base;
+
+			ibdev_info(&dev->ib_dev,
+				   "QP RTR: qpn=0x%06x ri=%u pid=%u vni=%u pte=%u map_slot=%u\n",
+				   qp->qpn_base, qp->ri,
+				   qp->domain ? qp->domain->pid : 0, qp->vni,
+				   qp->pte->id, pt_index);
+
+			/*
+			 * No list entry is appended here: an LE must be bound to
+			 * a registered MR, which only post_recv has. Until one is
+			 * posted the portal matches nothing and inbound traffic is
+			 * dropped, which is what an empty receive queue means.
+			 */
+
+			/*
+			 * pte_map() hands back an RMU portal-list slot, not the
+			 * PtlTE number that target commands address.
+			 */
+			resp.pte_index = qp->pte->id;
+			if (udata && udata->outlen >= sizeof(resp)) {
+				ret = ib_respond_udata(udata, resp);
+				if (ret) {
+					if (mapped) {
+						dev->cadev->pte_unmap(qp->pte,
+							qp->domain, qp->pt_map_index);
+						qp->pt_map_index = -1;
+					}
+					goto out;
+				}
+			}
+
+			spin_lock_irq(&qp->lock);
+			qp->state = IB_QPS_RTR;
+			spin_unlock_irq(&qp->lock);
+			ret = 0;
+		}
+		break;
+
+	case IB_QPS_RTR:
+		if (new_state == IB_QPS_RTS) {
+			spin_lock_irq(&qp->lock);
+			qp->state = IB_QPS_RTS;
+			spin_unlock_irq(&qp->lock);
+			ret = 0;
+		}
+		break;
+
+	case IB_QPS_RTS:
+	case IB_QPS_ERR:
+		if (new_state == cur_state || new_state == IB_QPS_ERR ||
+		    new_state == IB_QPS_RESET) {
+			spin_lock_irq(&qp->lock);
+			qp->state = new_state;
+			spin_unlock_irq(&qp->lock);
+			ret = 0;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+out:
+	mutex_unlock(&qp->modify_lock);
+	return ret;
+}
+
+/* =======================================================================
+ * query_qp
+ * =======================================================================
+ */
+
+int cxiverbs_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		     int attr_mask, struct ib_qp_init_attr *init_attr)
+{
+	struct cxiverbs_qp *qp = to_cxiverbs_qp(ibqp);
+	unsigned long flags;
+
+	memset(attr, 0, sizeof(*attr));
+	memset(init_attr, 0, sizeof(*init_attr));
+
+	spin_lock_irqsave(&qp->lock, flags);
+	attr->qp_state     = qp->state;
+	attr->cur_qp_state = qp->state;
+	spin_unlock_irqrestore(&qp->lock, flags);
+	attr->pkey_index   = 0;
+	attr->port_num     = 1;
+	attr->path_mtu     = IB_MTU_4096;
+
+	attr->cap.max_send_wr     = qp->max_send_wr;
+	attr->cap.max_recv_wr     = qp->max_recv_wr;
+	attr->cap.max_send_sge    = qp->max_send_sge;
+	attr->cap.max_recv_sge    = qp->max_recv_sge;
+	attr->cap.max_inline_data = qp->max_inline_data;
+
+	init_attr->cap = attr->cap;
+
+	return 0;
+}
+
+/* =======================================================================
+ * post_send  —  translate IB work requests to CXI DMA commands
+ * =======================================================================
+ */
+
+int cxiverbs_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+		      const struct ib_send_wr **bad_wr)
+{
+	struct cxiverbs_qp  *qp  = to_cxiverbs_qp(ibqp);
+	struct cxiverbs_dev *dev = to_cxiverbs_dev(ibqp->device);
+	struct cxiverbs_cq  *scq = to_cxiverbs_cq(ibqp->send_cq);
+	int ret = 0;
+
+	spin_lock(&qp->lock);
+	if (qp->state != IB_QPS_RTS && qp->state != IB_QPS_RTR) {
+		spin_unlock(&qp->lock);
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	for (; wr; wr = wr->next) {
+		union c_fab_addr dfa;
+		uint8_t dfa_ext = 0;
+
+		switch (wr->opcode) {
+		case IB_WR_SEND: {
+			const struct ib_ud_wr *ud_wr =
+				container_of(wr, const struct ib_ud_wr, wr);
+			struct cxiverbs_ah *ah = to_cxiverbs_ah(ud_wr->ah);
+			struct c_full_dma_cmd cmd = {};
+			u32 remote_pid;
+			u32 remote_ri;
+
+			if (!wr->num_sge || !wr->sg_list)
+				goto bad;
+
+			/* No CXI command carries a gather list. */
+			if (wr->num_sge > 1) {
+				ret = -EINVAL;
+				goto bad;
+			}
+
+			remote_pid = ud_wr->remote_qpn & CXI_QPN_PID_MASK;
+			remote_ri = FIELD_GET(CXI_QPN_RI_MASK,
+					      ud_wr->remote_qpn);
+			cxi_build_dfa(ah->nid, remote_pid, dev->pid_bits,
+				      remote_ri, &dfa, &dfa_ext);
+
+			/*
+			 * cxi_cq_emit_nomatch_dma() forces restricted=1, which
+			 * suppresses the target PUT event the receiver needs to
+			 * complete its recv WR, and c_nomatch_dma_cmd has no
+			 * user_ptr to carry wr_id. Use an unrestricted full DMA
+			 * put instead; the destination PtlTE is non-matching, so
+			 * match_bits are ignored.
+			 */
+			cmd.command.opcode = C_CMD_PUT;
+			cmd.lac            = wr->sg_list[0].lkey & CXI_LKEY64_LAC_MASK;
+			cmd.dfa            = dfa;
+			cmd.index_ext      = dfa_ext;
+			cmd.remote_offset  = 0;
+			cmd.local_addr     = wr->sg_list[0].addr;
+			cmd.request_len    = wr->sg_list[0].length;
+			cmd.user_ptr       = wr->wr_id;
+			cmd.eq             = scq->eq->eqn;
+			/* One completion per WR: ACK only, not SEND. */
+			cmd.event_send_disable = 1;
+
+			ret = cxi_cq_emit_dma(qp->txq, &cmd);
+			if (ret)
+				goto bad;
+			break;
+		}
+
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_READ: {
+			const struct ib_rdma_wr *rdma_wr =
+				container_of(wr, const struct ib_rdma_wr, wr);
+			struct c_full_dma_cmd cmd = {};
+
+			if (!wr->num_sge || !wr->sg_list)
+				goto bad;
+
+			if (wr->num_sge > 1) {
+				ret = -EINVAL;
+				goto bad;
+			}
+
+			/* For RC, remote NID is encoded in the QP's domain/AH.
+			 * Here we use a simplified direct NID from the rkey.
+			 */
+			cxi_build_dfa(0, 0, dev->pid_bits, 0, &dfa, &dfa_ext);
+
+			cmd.command.opcode = (wr->opcode == IB_WR_RDMA_READ) ?
+					     C_CMD_GET : C_CMD_PUT;
+			cmd.lac            = wr->sg_list[0].lkey & CXI_LKEY64_LAC_MASK;
+			cmd.dfa            = dfa;
+			cmd.index_ext      = dfa_ext;
+			cmd.remote_offset  = rdma_wr->remote_addr;
+			cmd.local_addr     = wr->sg_list[0].addr;
+			cmd.request_len    = wr->sg_list[0].length;
+			cmd.match_bits     = (u64)rdma_wr->rkey;
+			cmd.user_ptr       = wr->wr_id;
+			cmd.eq             = scq->eq->eqn;
+
+			ret = cxi_cq_emit_dma(qp->txq, &cmd);
+			if (ret)
+				goto bad;
+			break;
+		}
+
+		case IB_WR_ATOMIC_FETCH_AND_ADD:
+		case IB_WR_ATOMIC_CMP_AND_SWP: {
+			const struct ib_atomic_wr *atomic_wr =
+				container_of(wr, const struct ib_atomic_wr, wr);
+			const struct ib_rdma_wr *rdma_wr =
+				container_of(wr, const struct ib_rdma_wr, wr);
+			struct c_dma_amo_cmd cmd = {};
+			bool fetching = (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD);
+
+			cxi_build_dfa(0, 0, dev->pid_bits, 0, &dfa, &dfa_ext);
+
+			cmd.command.opcode  = C_CMD_ATOMIC;
+			cmd.lac             = 0;
+			cmd.dfa             = dfa;
+			cmd.index_ext       = dfa_ext;
+			cmd.remote_offset   = rdma_wr->remote_addr;
+			cmd.request_len     = 8;
+			cmd.match_bits      = (u64)rdma_wr->rkey;
+			cmd.user_ptr        = wr->wr_id;
+			cmd.eq              = scq->eq->eqn;
+			cmd.atomic_op       = fetching ? C_AMO_OP_SUM : C_AMO_OP_CSWAP;
+			cmd.atomic_type     = C_AMO_TYPE_UINT64_T;
+			cmd.op2_word1       = atomic_wr->compare_add;
+			if (!fetching)
+				cmd.op2_word2 = atomic_wr->swap;
+
+			ret = cxi_cq_emit_dma_amo(qp->txq, &cmd, fetching);
+			if (ret)
+				goto bad;
+			break;
+		}
+
+		default:
+			ret = -EOPNOTSUPP;
+			goto bad;
+		}
+
+		cxi_cq_ring(qp->txq);
+		continue;
+
+bad:
+		*bad_wr = wr;
+		spin_unlock(&qp->lock);
+		return ret ? ret : -EIO;
+	}
+
+	spin_unlock(&qp->lock);
+	return 0;
+}
+
+/* =======================================================================
+ * post_recv  —  append a receive LE to the portal table
+ * =======================================================================
+ */
+
+int cxiverbs_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
+		      const struct ib_recv_wr **bad_wr)
+{
+	struct cxiverbs_qp *qp = to_cxiverbs_qp(ibqp);
+	int ret = 0;
+
+	spin_lock(&qp->lock);
+	if (qp->state < IB_QPS_RTR) {
+		spin_unlock(&qp->lock);
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	for (; wr; wr = wr->next) {
+		union c_cmdu cmd = {};
+
+		if (!wr->num_sge || !wr->sg_list) {
+			ret = -EINVAL;
+			*bad_wr = wr;
+			break;
+		}
+
+		/* A list entry describes one buffer. */
+		if (wr->num_sge > 1) {
+			ret = -EINVAL;
+			*bad_wr = wr;
+			break;
+		}
+
+		cmd.command.opcode     = C_CMD_TGT_APPEND;
+		cmd.target.ptlte_index = (u16)qp->pte->id;
+		cmd.target.ptl_list    = C_PTL_LIST_PRIORITY;
+		cmd.target.op_put      = 1;
+		cmd.target.use_once    = 1;
+		cmd.target.lac         = wr->sg_list[0].lkey & CXI_LKEY64_LAC_MASK;
+		cmd.target.start       = wr->sg_list[0].addr;
+		cmd.target.length      = wr->sg_list[0].length;
+		/* SEND carries no key; match exactly the zero an rkey never is. */
+		cmd.target.match_bits  = 0;
+		cmd.target.ignore_bits = 0;
+
+		ret = cxi_cq_emit_target(qp->tgq, &cmd);
+		if (ret) {
+			*bad_wr = wr;
+			break;
+		}
+		cxi_cq_ring(qp->tgq);
+	}
+
+	spin_unlock(&qp->lock);
+	return ret;
+}

-- 
2.56.0