[PATCH v2 24/26] net/rds: Inline ib_dma_map_*() functions

[Bart Van Assche]

Signed-off-by: Bart Van Assche <bart.vanassche@xxxxxxxxxxx>
Reviewed-by: Christoph Hellwig <hch@xxxxxx>
Reviewed-by: Sagi Grimberg <sagi@xxxxxxxxxxx>
Cc: Santosh Shilimkar <santosh.shilimkar@xxxxxxxxxx>
---
 net/rds/ib.h      | 39 ---------------------------------------
 net/rds/ib_cm.c   | 42 ++++++++++++++++++++++--------------------
 net/rds/ib_fmr.c  | 10 +++++-----
 net/rds/ib_frmr.c | 12 ++++++------
 net/rds/ib_rdma.c | 13 ++++++-------
 net/rds/ib_recv.c | 19 +++++++++----------
 net/rds/ib_send.c | 50 +++++++++++++++++++++++++-------------------------
 7 files changed, 73 insertions(+), 112 deletions(-)

diff --git a/net/rds/ib.h b/net/rds/ib.h
index d21ca88ab628..02e5fe8d6af8 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -275,45 +275,6 @@ struct rds_ib_statistics {
 
 extern struct workqueue_struct *rds_ib_wq;
 
-/*
- * Fake ib_dma_sync_sg_for_{cpu,device} as long as ib_verbs.h
- * doesn't define it.
- */
-static inline void rds_ib_dma_sync_sg_for_cpu(struct ib_device *dev,
-					      struct scatterlist *sglist,
-					      unsigned int sg_dma_len,
-					      int direction)
-{
-	struct scatterlist *sg;
-	unsigned int i;
-
-	for_each_sg(sglist, sg, sg_dma_len, i) {
-		ib_dma_sync_single_for_cpu(dev,
-				ib_sg_dma_address(dev, sg),
-				ib_sg_dma_len(dev, sg),
-				direction);
-	}
-}
-#define ib_dma_sync_sg_for_cpu	rds_ib_dma_sync_sg_for_cpu
-
-static inline void rds_ib_dma_sync_sg_for_device(struct ib_device *dev,
-						 struct scatterlist *sglist,
-						 unsigned int sg_dma_len,
-						 int direction)
-{
-	struct scatterlist *sg;
-	unsigned int i;
-
-	for_each_sg(sglist, sg, sg_dma_len, i) {
-		ib_dma_sync_single_for_device(dev,
-				ib_sg_dma_address(dev, sg),
-				ib_sg_dma_len(dev, sg),
-				direction);
-	}
-}
-#define ib_dma_sync_sg_for_device	rds_ib_dma_sync_sg_for_device
-
-
 /* ib.c */
 extern struct rds_transport rds_ib_transport;
 struct rds_ib_device *rds_ib_get_client_data(struct ib_device *device);
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 5b2ab95afa07..a6d3726ea27d 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -456,31 +456,32 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
 		goto out;
 	}
 
-	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,
-					   ic->i_send_ring.w_nr *
+	ic->i_send_hdrs = dma_alloc_coherent(dev->dma_device,
+					     ic->i_send_ring.w_nr *
 						sizeof(struct rds_header),
-					   &ic->i_send_hdrs_dma, GFP_KERNEL);
+					     &ic->i_send_hdrs_dma, GFP_KERNEL);
 	if (!ic->i_send_hdrs) {
 		ret = -ENOMEM;
-		rdsdebug("ib_dma_alloc_coherent send failed\n");
+		rdsdebug("dma_alloc_coherent send failed\n");
 		goto out;
 	}
 
-	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,
-					   ic->i_recv_ring.w_nr *
+	ic->i_recv_hdrs = dma_alloc_coherent(dev->dma_device,
+					     ic->i_recv_ring.w_nr *
 						sizeof(struct rds_header),
-					   &ic->i_recv_hdrs_dma, GFP_KERNEL);
+					     &ic->i_recv_hdrs_dma, GFP_KERNEL);
 	if (!ic->i_recv_hdrs) {
 		ret = -ENOMEM;
-		rdsdebug("ib_dma_alloc_coherent recv failed\n");
+		rdsdebug("dma_alloc_coherent recv failed\n");
 		goto out;
 	}
 
-	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),
+	ic->i_ack = dma_alloc_coherent(dev->dma_device,
+				       sizeof(struct rds_header),
 				       &ic->i_ack_dma, GFP_KERNEL);
 	if (!ic->i_ack) {
 		ret = -ENOMEM;
-		rdsdebug("ib_dma_alloc_coherent ack failed\n");
+		rdsdebug("dma_alloc_coherent ack failed\n");
 		goto out;
 	}
 
@@ -781,22 +782,23 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
 
 		/* then free the resources that ib callbacks use */
 		if (ic->i_send_hdrs)
-			ib_dma_free_coherent(dev,
-					   ic->i_send_ring.w_nr *
+			dma_free_coherent(dev->dma_device,
+					  ic->i_send_ring.w_nr *
 						sizeof(struct rds_header),
-					   ic->i_send_hdrs,
-					   ic->i_send_hdrs_dma);
+					  ic->i_send_hdrs,
+					  ic->i_send_hdrs_dma);
 
 		if (ic->i_recv_hdrs)
-			ib_dma_free_coherent(dev,
-					   ic->i_recv_ring.w_nr *
+			dma_free_coherent(dev->dma_device,
+					  ic->i_recv_ring.w_nr *
 						sizeof(struct rds_header),
-					   ic->i_recv_hdrs,
-					   ic->i_recv_hdrs_dma);
+					  ic->i_recv_hdrs,
+					  ic->i_recv_hdrs_dma);
 
 		if (ic->i_ack)
-			ib_dma_free_coherent(dev, sizeof(struct rds_header),
-					     ic->i_ack, ic->i_ack_dma);
+			dma_free_coherent(dev->dma_device,
+					  sizeof(struct rds_header),
+					  ic->i_ack, ic->i_ack_dma);
 
 		if (ic->i_sends)
 			rds_ib_send_clear_ring(ic);
diff --git a/net/rds/ib_fmr.c b/net/rds/ib_fmr.c
index 4fe8f4fec4ee..150e8f756bd9 100644
--- a/net/rds/ib_fmr.c
+++ b/net/rds/ib_fmr.c
@@ -100,7 +100,7 @@ int rds_ib_map_fmr(struct rds_ib_device *rds_ibdev, struct rds_ib_mr *ibmr,
 	int i, j;
 	int ret;
 
-	sg_dma_len = ib_dma_map_sg(dev, sg, nents, DMA_BIDIRECTIONAL);
+	sg_dma_len = dma_map_sg(dev->dma_device, sg, nents, DMA_BIDIRECTIONAL);
 	if (unlikely(!sg_dma_len)) {
 		pr_warn("RDS/IB: %s failed!\n", __func__);
 		return -EBUSY;
@@ -110,8 +110,8 @@ int rds_ib_map_fmr(struct rds_ib_device *rds_ibdev, struct rds_ib_mr *ibmr,
 	page_cnt = 0;
 
 	for (i = 0; i < sg_dma_len; ++i) {
-		unsigned int dma_len = ib_sg_dma_len(dev, &scat[i]);
-		u64 dma_addr = ib_sg_dma_address(dev, &scat[i]);
+		unsigned int dma_len = sg_dma_len(&scat[i]);
+		u64 dma_addr = sg_dma_address(&scat[i]);
 
 		if (dma_addr & ~PAGE_MASK) {
 			if (i > 0)
@@ -140,8 +140,8 @@ int rds_ib_map_fmr(struct rds_ib_device *rds_ibdev, struct rds_ib_mr *ibmr,
 
 	page_cnt = 0;
 	for (i = 0; i < sg_dma_len; ++i) {
-		unsigned int dma_len = ib_sg_dma_len(dev, &scat[i]);
-		u64 dma_addr = ib_sg_dma_address(dev, &scat[i]);
+		unsigned int dma_len = sg_dma_len(&scat[i]);
+		u64 dma_addr = sg_dma_address(&scat[i]);
 
 		for (j = 0; j < dma_len; j += PAGE_SIZE)
 			dma_pages[page_cnt++] =
diff --git a/net/rds/ib_frmr.c b/net/rds/ib_frmr.c
index d921adc62765..2d44bf11d97f 100644
--- a/net/rds/ib_frmr.c
+++ b/net/rds/ib_frmr.c
@@ -169,8 +169,8 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev,
 	ibmr->sg_dma_len = 0;
 	frmr->sg_byte_len = 0;
 	WARN_ON(ibmr->sg_dma_len);
-	ibmr->sg_dma_len = ib_dma_map_sg(dev, ibmr->sg, ibmr->sg_len,
-					 DMA_BIDIRECTIONAL);
+	ibmr->sg_dma_len = dma_map_sg(dev->dma_device, ibmr->sg, ibmr->sg_len,
+				      DMA_BIDIRECTIONAL);
 	if (unlikely(!ibmr->sg_dma_len)) {
 		pr_warn("RDS/IB: %s failed!\n", __func__);
 		return -EBUSY;
@@ -182,8 +182,8 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev,
 
 	ret = -EINVAL;
 	for (i = 0; i < ibmr->sg_dma_len; ++i) {
-		unsigned int dma_len = ib_sg_dma_len(dev, &ibmr->sg[i]);
-		u64 dma_addr = ib_sg_dma_address(dev, &ibmr->sg[i]);
+		unsigned int dma_len = sg_dma_len(&ibmr->sg[i]);
+		u64 dma_addr = sg_dma_address(&ibmr->sg[i]);
 
 		frmr->sg_byte_len += dma_len;
 		if (dma_addr & ~PAGE_MASK) {
@@ -221,8 +221,8 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev,
 	return ret;
 
 out_unmap:
-	ib_dma_unmap_sg(rds_ibdev->dev, ibmr->sg, ibmr->sg_len,
-			DMA_BIDIRECTIONAL);
+	dma_unmap_sg(rds_ibdev->dev->dma_device, ibmr->sg, ibmr->sg_len,
+		     DMA_BIDIRECTIONAL);
 	ibmr->sg_dma_len = 0;
 	return ret;
 }
diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c
index 977f69886c00..9f5acba71a05 100644
--- a/net/rds/ib_rdma.c
+++ b/net/rds/ib_rdma.c
@@ -221,12 +221,12 @@ void rds_ib_sync_mr(void *trans_private, int direction)
 
 	switch (direction) {
 	case DMA_FROM_DEVICE:
-		ib_dma_sync_sg_for_cpu(rds_ibdev->dev, ibmr->sg,
-			ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
+		dma_sync_sg_for_cpu(rds_ibdev->dev->dma_device, ibmr->sg,
+				    ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
 		break;
 	case DMA_TO_DEVICE:
-		ib_dma_sync_sg_for_device(rds_ibdev->dev, ibmr->sg,
-			ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
+		dma_sync_sg_for_device(rds_ibdev->dev->dma_device, ibmr->sg,
+				       ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
 		break;
 	}
 }
@@ -236,9 +236,8 @@ void __rds_ib_teardown_mr(struct rds_ib_mr *ibmr)
 	struct rds_ib_device *rds_ibdev = ibmr->device;
 
 	if (ibmr->sg_dma_len) {
-		ib_dma_unmap_sg(rds_ibdev->dev,
-				ibmr->sg, ibmr->sg_len,
-				DMA_BIDIRECTIONAL);
+		dma_unmap_sg(rds_ibdev->dev->dma_device, ibmr->sg, ibmr->sg_len,
+			     DMA_BIDIRECTIONAL);
 		ibmr->sg_dma_len = 0;
 	}
 
diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c
index 606a11f681d2..32bc2df27e5a 100644
--- a/net/rds/ib_recv.c
+++ b/net/rds/ib_recv.c
@@ -225,7 +225,8 @@ static void rds_ib_recv_clear_one(struct rds_ib_connection *ic,
 		recv->r_ibinc = NULL;
 	}
 	if (recv->r_frag) {
-		ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE);
+		dma_unmap_sg(ic->i_cm_id->device->dma_device,
+			     &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE);
 		rds_ib_frag_free(ic, recv->r_frag);
 		recv->r_frag = NULL;
 	}
@@ -331,8 +332,8 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
 	if (!recv->r_frag)
 		goto out;
 
-	ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg,
-			    1, DMA_FROM_DEVICE);
+	ret = dma_map_sg(ic->i_cm_id->device->dma_device, &recv->r_frag->f_sg,
+			 1, DMA_FROM_DEVICE);
 	WARN_ON(ret != 1);
 
 	sge = &recv->r_sge[0];
@@ -340,8 +341,8 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
 	sge->length = sizeof(struct rds_header);
 
 	sge = &recv->r_sge[1];
-	sge->addr = ib_sg_dma_address(ic->i_cm_id->device, &recv->r_frag->f_sg);
-	sge->length = ib_sg_dma_len(ic->i_cm_id->device, &recv->r_frag->f_sg);
+	sge->addr = sg_dma_address(&recv->r_frag->f_sg);
+	sge->length = sg_dma_len(&recv->r_frag->f_sg);
 
 	ret = 0;
 out:
@@ -408,9 +409,7 @@ void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp)
 		ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, &failed_wr);
 		rdsdebug("recv %p ibinc %p page %p addr %lu ret %d\n", recv,
 			 recv->r_ibinc, sg_page(&recv->r_frag->f_sg),
-			 (long) ib_sg_dma_address(
-				ic->i_cm_id->device,
-				&recv->r_frag->f_sg),
+			 (long)sg_dma_address(&recv->r_frag->f_sg),
 			ret);
 		if (ret) {
 			rds_ib_conn_error(conn, "recv post on "
@@ -968,8 +967,8 @@ void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic,
 
 	rds_ib_stats_inc(s_ib_rx_cq_event);
 	recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)];
-	ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1,
-			DMA_FROM_DEVICE);
+	dma_unmap_sg(ic->i_cm_id->device->dma_device, &recv->r_frag->f_sg, 1,
+		     DMA_FROM_DEVICE);
 
 	/* Also process recvs in connecting state because it is possible
 	 * to get a recv completion _before_ the rdmacm ESTABLISHED
diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c
index 84d90c97332f..aa79b6aa488e 100644
--- a/net/rds/ib_send.c
+++ b/net/rds/ib_send.c
@@ -74,9 +74,8 @@ static void rds_ib_send_unmap_data(struct rds_ib_connection *ic,
 				   int wc_status)
 {
 	if (op->op_nents)
-		ib_dma_unmap_sg(ic->i_cm_id->device,
-				op->op_sg, op->op_nents,
-				DMA_TO_DEVICE);
+		dma_unmap_sg(ic->i_cm_id->device->dma_device, op->op_sg,
+			     op->op_nents, DMA_TO_DEVICE);
 }
 
 static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,
@@ -84,9 +83,9 @@ static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,
 				   int wc_status)
 {
 	if (op->op_mapped) {
-		ib_dma_unmap_sg(ic->i_cm_id->device,
-				op->op_sg, op->op_nents,
-				op->op_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
+		dma_unmap_sg(ic->i_cm_id->device->dma_device, op->op_sg,
+			     op->op_nents, op->op_write ? DMA_TO_DEVICE :
+			     DMA_FROM_DEVICE);
 		op->op_mapped = 0;
 	}
 
@@ -106,7 +105,7 @@ static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,
 	 * handling in the ACK processing code.
 	 *
 	 * Note: There's no need to explicitly sync any RDMA buffers using
-	 * ib_dma_sync_sg_for_cpu - the completion for the RDMA
+	 * dma_sync_sg_for_cpu - the completion for the RDMA
 	 * operation itself unmapped the RDMA buffers, which takes care
 	 * of synching.
 	 */
@@ -125,8 +124,8 @@ static void rds_ib_send_unmap_atomic(struct rds_ib_connection *ic,
 {
 	/* unmap atomic recvbuf */
 	if (op->op_mapped) {
-		ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,
-				DMA_FROM_DEVICE);
+		dma_unmap_sg(ic->i_cm_id->device->dma_device, op->op_sg, 1,
+			     DMA_FROM_DEVICE);
 		op->op_mapped = 0;
 	}
 
@@ -546,10 +545,10 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
 	/* map the message the first time we see it */
 	if (!ic->i_data_op) {
 		if (rm->data.op_nents) {
-			rm->data.op_count = ib_dma_map_sg(dev,
-							  rm->data.op_sg,
-							  rm->data.op_nents,
-							  DMA_TO_DEVICE);
+			rm->data.op_count = dma_map_sg(dev->dma_device,
+						       rm->data.op_sg,
+						       rm->data.op_nents,
+						       DMA_TO_DEVICE);
 			rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);
 			if (rm->data.op_count == 0) {
 				rds_ib_stats_inc(s_ib_tx_sg_mapping_failure);
@@ -640,16 +639,16 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
 		if (i < work_alloc
 		    && scat != &rm->data.op_sg[rm->data.op_count]) {
 			len = min(RDS_FRAG_SIZE,
-				ib_sg_dma_len(dev, scat) - rm->data.op_dmaoff);
+				sg_dma_len(scat) - rm->data.op_dmaoff);
 			send->s_wr.num_sge = 2;
 
-			send->s_sge[1].addr = ib_sg_dma_address(dev, scat);
+			send->s_sge[1].addr = sg_dma_address(scat);
 			send->s_sge[1].addr += rm->data.op_dmaoff;
 			send->s_sge[1].length = len;
 
 			bytes_sent += len;
 			rm->data.op_dmaoff += len;
-			if (rm->data.op_dmaoff == ib_sg_dma_len(dev, scat)) {
+			if (rm->data.op_dmaoff == sg_dma_len(scat)) {
 				scat++;
 				rm->data.op_dmasg++;
 				rm->data.op_dmaoff = 0;
@@ -797,7 +796,8 @@ int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)
 	rds_message_addref(container_of(send->s_op, struct rds_message, atomic));
 
 	/* map 8 byte retval buffer to the device */
-	ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);
+	ret = dma_map_sg(ic->i_cm_id->device->dma_device, op->op_sg, 1,
+			 DMA_FROM_DEVICE);
 	rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);
 	if (ret != 1) {
 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
@@ -807,8 +807,8 @@ int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op)
 	}
 
 	/* Convert our struct scatterlist to struct ib_sge */
-	send->s_sge[0].addr = ib_sg_dma_address(ic->i_cm_id->device, op->op_sg);
-	send->s_sge[0].length = ib_sg_dma_len(ic->i_cm_id->device, op->op_sg);
+	send->s_sge[0].addr = sg_dma_address(op->op_sg);
+	send->s_sge[0].length = sg_dma_len(op->op_sg);
 	send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;
 
 	rdsdebug("rva %Lx rpa %Lx len %u\n", op->op_remote_addr,
@@ -861,9 +861,10 @@ int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
 
 	/* map the op the first time we see it */
 	if (!op->op_mapped) {
-		op->op_count = ib_dma_map_sg(ic->i_cm_id->device,
-					     op->op_sg, op->op_nents, (op->op_write) ?
-					     DMA_TO_DEVICE : DMA_FROM_DEVICE);
+		op->op_count = dma_map_sg(ic->i_cm_id->device->dma_device,
+					  op->op_sg, op->op_nents,
+					  op->op_write ? DMA_TO_DEVICE :
+					  DMA_FROM_DEVICE);
 		rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count);
 		if (op->op_count == 0) {
 			rds_ib_stats_inc(s_ib_tx_sg_mapping_failure);
@@ -920,9 +921,8 @@ int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op)
 
 		for (j = 0; j < send->s_rdma_wr.wr.num_sge &&
 		     scat != &op->op_sg[op->op_count]; j++) {
-			len = ib_sg_dma_len(ic->i_cm_id->device, scat);
-			send->s_sge[j].addr =
-				 ib_sg_dma_address(ic->i_cm_id->device, scat);
+			len = sg_dma_len(scat);
+			send->s_sge[j].addr = sg_dma_address(scat);
 			send->s_sge[j].length = len;
 			send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;
 
-- 
2.11.0