RE: [PATCH rdma-next v2 3/3] RDMA/mana_ib: fixed send wqe size support for RC QPs

[Long Li]

> From: Konstantin Taranov <kotaranov@xxxxxxxxxxxxx>
> 
> Support fixed-size send WQEs for RC QPs when requested by userspace.
> Calculate the WQE size from the maximum send SGE count and the 32-byte
> inline OOB size, round it up to a power of two, and pass the size to hardware in
> basic units.
> 
> When supported by the adapter, also enable placing the message sequence
> number in the WQE. Preserve the existing variable-size WQE behavior when
> userspace does not request fixed-size WQEs.
> 
> Signed-off-by: Konstantin Taranov <kotaranov@xxxxxxxxxxxxx>

Reviewed-by: Long Li <longli@xxxxxxxxxxxxx>




> ---
>  drivers/infiniband/hw/mana/main.c    |  3 +++
>  drivers/infiniband/hw/mana/mana_ib.h |  5 ++++-
>  drivers/infiniband/hw/mana/qp.c      | 14 ++++++++++++++
>  include/net/mana/gdma.h              |  1 +
>  4 files changed, 22 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/infiniband/hw/mana/main.c
> b/drivers/infiniband/hw/mana/main.c
> index 9d043f48e..57c6d9faa 100644
> --- a/drivers/infiniband/hw/mana/main.c
> +++ b/drivers/infiniband/hw/mana/main.c
> @@ -1060,6 +1060,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev
> *mdev, struct mana_ib_qp *qp,
>  	req.max_recv_sge = attr->cap.max_recv_sge;
>  	req.flags = flags;
> 
> +	if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE)
> +		req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu;
> +
>  	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),
> &resp);
>  	if (err)
>  		return err;
> diff --git a/drivers/infiniband/hw/mana/mana_ib.h
> b/drivers/infiniband/hw/mana/mana_ib.h
> index 547ddf9dd..828ca75fd 100644
> --- a/drivers/infiniband/hw/mana/mana_ib.h
> +++ b/drivers/infiniband/hw/mana/mana_ib.h
> @@ -180,6 +180,7 @@ enum mana_rc_queue_type {
> 
>  struct mana_ib_rc_qp {
>  	struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
> +	u32 wqe_size_in_bu;
>  };
> 
>  enum mana_uc_queue_type {
> @@ -262,6 +263,7 @@ enum mana_ib_adapter_features {
>  	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
>  	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
>  	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
> +	MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
>  };
> 
>  struct mana_ib_query_adapter_caps_resp { @@ -391,7 +393,8 @@ struct
> mana_rnic_create_qp_req {
>  	u32 max_recv_wr;
>  	u32 max_send_sge;
>  	u32 max_recv_sge;
> -	u32 reserved;
> +	u8 wqe_size_in_bu;
> +	u8 reserved[3];
>  }; /* HW Data */
> 
>  struct mana_rnic_create_qp_resp {
> diff --git a/drivers/infiniband/hw/mana/qp.c
> b/drivers/infiniband/hw/mana/qp.c index c6821ae5f..29bfee9ac 100644
> --- a/drivers/infiniband/hw/mana/qp.c
> +++ b/drivers/infiniband/hw/mana/qp.c
> @@ -422,6 +422,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
>  	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);  }
> 
> +static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size) {
> +	return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size)); }
> +
>  static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)  {
>  	u32 queue_size;
> @@ -576,6 +581,15 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp,
> struct ib_pd *ibpd,
>  		j++;
>  	}
> 
> +	if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) {
> +		u32 wqe_size = mana_ib_fixed_wqe_size(attr-
> >cap.max_send_sge,
> +
> INLINE_OOB_EXTRA_LARGE_SIZE);
> +		flags |= MANA_RC_FLAG_FIXED_SIZE_WQE;
> +		if (mdev->adapter_caps.feature_flags &
> MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT)
> +			flags |= MANA_RC_FLAG_MSN_IN_WQE;
> +		qp->rc_qp.wqe_size_in_bu = wqe_size /
> GDMA_WQE_BU_SIZE;
> +	}
> +
>  	err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
>  	if (err) {
>  		ibdev_err(&mdev->ib_dev, "Failed to create rc qp  %d\n", err);
> diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index
> de17c9bba..f3601ca78 100644
> --- a/include/net/mana/gdma.h
> +++ b/include/net/mana/gdma.h
> @@ -522,6 +522,7 @@ struct gdma_wqe {
> 
>  #define INLINE_OOB_SMALL_SIZE 8
>  #define INLINE_OOB_LARGE_SIZE 24
> +#define INLINE_OOB_EXTRA_LARGE_SIZE 32
> 
>  #define MANA_MAX_TX_WQE_SGL_ENTRIES 30
> 
> --
> 2.43.0