Re: [PATCH RFC 3/3] RDMA/rtrs: Post client read requests inline

[Jinpu Wang]

Hi Andrei,

The change looks fine to me, and I did some testing on it. However,
the data doesn't seem to show any clear performance gain from posting
inline requests.

Best regards,

Jinpu

On Thu, Oct 15, 2026 at 2:51 PM Andrei-Alexandru Bleortu
<me@xxxxxxxxxxxx> wrote:
>
> The RTRS client sends a read request as an RDMA Write with Immediate of
> a small message: the user message, the RTRS header and one memory
> descriptor, about 60 bytes for RNBD. The message is posted by reference
> and the QP asks for no inline data, so the HCA has to DMA-read it from
> host memory before sending. That read is on the latency path of every
> read.
>
> Let rtrs_cq_qp_create() take the amount of inline data to ask for.
> The client asks for RTRS_CLT_MAX_INLINE_DATA, enough for a read
> request with one descriptor and up to 64 bytes of user message, and
> the server asks for none. If the device refuses, the QP is created
> without inline data. The client posts a read request with
> IB_SEND_INLINE when it fits in the inline size the device granted.
> Write requests, whose message travels behind the data in the same
> WR, are not changed.
>
> Providers copy inline data from sg_list[].addr as a kernel virtual
> address, so the inline path puts req->iu->buf in the on-stack SGE.
>
> Assisted-by: LLM
> Signed-off-by: Andrei-Alexandru Bleortu <me@xxxxxxxxxxxx>
> ---
>  drivers/infiniband/ulp/rtrs/rtrs-clt.c | 17 +++++++++++++++--
>  drivers/infiniband/ulp/rtrs/rtrs-pri.h | 10 +++++++++-
>  drivers/infiniband/ulp/rtrs/rtrs-srv.c |  2 +-
>  drivers/infiniband/ulp/rtrs/rtrs.c     | 22 +++++++++++++++++-----
>  4 files changed, 42 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
> index bf3a0377da3f..4a552e0459b2 100644
> --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
> +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
> @@ -489,6 +489,17 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con,
>                                       req->iu->dma_addr,
>                                       req->sg_size, DMA_TO_DEVICE);
>
> +       /*
> +        * Post the request inline if it fits in the granted inline size, so
> +        * the HCA does not have to DMA-read it from host memory.  The CPU
> +        * copies inline data from a kernel virtual address, so put
> +        * req->iu->buf in the SGE.
> +        */
> +       if (req->sg_size <= con->c.max_inline_data) {
> +               sge.addr = (uintptr_t)req->iu->buf;
> +               flags |= IB_SEND_INLINE;
> +       }
> +
>         return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1,
>                                             rbuf->rkey, rbuf->addr + off,
>                                             imm, flags, wr, NULL);
> @@ -1714,11 +1725,13 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
>         if (con->c.cid >= clt_path->s.irq_con_num)
>                 err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
>                                         cq_vector, cq_num, max_send_wr,
> -                                       max_recv_wr, IB_POLL_DIRECT);
> +                                       max_recv_wr, RTRS_CLT_MAX_INLINE_DATA,
> +                                       IB_POLL_DIRECT);
>         else
>                 err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
>                                         cq_vector, cq_num, max_send_wr,
> -                                       max_recv_wr, IB_POLL_SOFTIRQ);
> +                                       max_recv_wr, RTRS_CLT_MAX_INLINE_DATA,
> +                                       IB_POLL_SOFTIRQ);
>         /*
>          * In case of error we do not bother to clean previous allocations,
>          * since destroy_con_cq_qp() must be called.
> diff --git a/drivers/infiniband/ulp/rtrs/rtrs-pri.h b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
> index 59529d593869..bcba80d3c214 100644
> --- a/drivers/infiniband/ulp/rtrs/rtrs-pri.h
> +++ b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
> @@ -97,6 +97,7 @@ struct rtrs_con {
>         int                     nr_cqe;
>         atomic_t                wr_cnt;
>         atomic_t                sq_wr_avail;
> +       u32                     max_inline_data;
>  };
>
>  struct rtrs_path {
> @@ -286,6 +287,13 @@ struct rtrs_msg_rdma_read {
>         struct rtrs_sg_desc    desc[];
>  };
>
> +/*
> + * Inline data the client asks for on its QPs: enough for a read request
> + * with one memory descriptor and a user message of up to 64 bytes.
> + */
> +#define RTRS_CLT_MAX_INLINE_DATA (sizeof(struct rtrs_msg_rdma_read) + \
> +                                 sizeof(struct rtrs_sg_desc) + 64)
> +
>  /**
>   * struct rtrs_msg_rdma_write - Message transferred to server with RDMA-Write
>   * @type:              always @RTRS_MSG_WRITE
> @@ -325,7 +333,7 @@ int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe);
>  int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
>                       u32 max_send_sge, int cq_vector, int nr_cqe,
>                       u32 max_send_wr, u32 max_recv_wr,
> -                     enum ib_poll_context poll_ctx);
> +                     u32 max_inline_data, enum ib_poll_context poll_ctx);
>  void rtrs_cq_qp_destroy(struct rtrs_con *con);
>
>  void rtrs_init_hb(struct rtrs_path *path, struct ib_cqe *cqe,
> diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
> index 5314599d29fe..30fa4c7f64dc 100644
> --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
> +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
> @@ -1759,7 +1759,7 @@ static int create_con(struct rtrs_srv_path *srv_path,
>
>         /* TODO: SOFTIRQ can be faster, but be careful with softirq context */
>         err = rtrs_cq_qp_create(&srv_path->s, &con->c, 1, cq_vector, cq_num,
> -                                max_send_wr, max_recv_wr,
> +                                max_send_wr, max_recv_wr, 0,
>                                  IB_POLL_WORKQUEUE);
>         if (err) {
>                 rtrs_err(s, "rtrs_cq_qp_create(), err: %pe\n", ERR_PTR(err));
> diff --git a/drivers/infiniband/ulp/rtrs/rtrs.c b/drivers/infiniband/ulp/rtrs/rtrs.c
> index a642626e0ecb..08b8b1abb1d3 100644
> --- a/drivers/infiniband/ulp/rtrs/rtrs.c
> +++ b/drivers/infiniband/ulp/rtrs/rtrs.c
> @@ -253,9 +253,10 @@ static int create_cq(struct rtrs_con *con, int cq_vector, int nr_cqe,
>  }
>
>  static int create_qp(struct rtrs_con *con, struct ib_pd *pd,
> -                    u32 max_send_wr, u32 max_recv_wr, u32 max_sge)
> +                    u32 max_send_wr, u32 max_recv_wr, u32 max_sge,
> +                    u32 max_inline_data)
>  {
> -       struct ib_qp_init_attr init_attr = {};
> +       struct ib_qp_init_attr init_attr = {}, attr;
>         struct rdma_cm_id *cm_id = con->cm_id;
>         int ret;
>
> @@ -271,13 +272,24 @@ static int create_qp(struct rtrs_con *con, struct ib_pd *pd,
>         init_attr.recv_cq = con->cq;
>         init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
>
> -       ret = rdma_create_qp(cm_id, pd, &init_attr);
> +       /*
> +        * If the device cannot provide the requested inline data, create the
> +        * QP without it.
> +        */
> +       attr = init_attr;
> +       attr.cap.max_inline_data = max_inline_data;
> +       ret = rdma_create_qp(cm_id, pd, &attr);
> +       if (ret && max_inline_data) {
> +               attr = init_attr;
> +               ret = rdma_create_qp(cm_id, pd, &attr);
> +       }
>         if (ret) {
>                 rtrs_err(con->path, "Creating QP failed, err: %pe\n",
>                          ERR_PTR(ret));
>                 return ret;
>         }
>         con->qp = cm_id->qp;
> +       con->max_inline_data = attr.cap.max_inline_data;
>
>         return ret;
>  }
> @@ -296,7 +308,7 @@ static void destroy_cq(struct rtrs_con *con)
>  int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
>                        u32 max_send_sge, int cq_vector, int nr_cqe,
>                        u32 max_send_wr, u32 max_recv_wr,
> -                      enum ib_poll_context poll_ctx)
> +                      u32 max_inline_data, enum ib_poll_context poll_ctx)
>  {
>         int err;
>
> @@ -305,7 +317,7 @@ int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con,
>                 return err;
>
>         err = create_qp(con, path->dev->ib_pd, max_send_wr, max_recv_wr,
> -                       max_send_sge);
> +                       max_send_sge, max_inline_data);
>         if (err) {
>                 destroy_cq(con);
>                 return err;
> --
> 2.56.0
>
>