Re: [PATCH v8 10/22] IB/hns: Add process flow to init RoCE engine

From: Leon Romanovsky
Date: Wed May 25 2016 - 14:21:25 EST

Next message: Arnd Bergmann: "Re: [PATCH v3] Axi-usb: Add support for 64-bit addressing."
Previous message: Sinan Kaya: "Re: [PATCH] PCI/ASPM: fix reverse ASPM L0s assignment of upstream and downstream"
In reply to: Lijun Ou: "[PATCH v8 10/22] IB/hns: Add process flow to init RoCE engine"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

On Wed, May 25, 2016 at 11:05:13PM +0800, Lijun Ou wrote:
> This patch mainly initialized the RoCE engine. It is absolutely
> necessary to run RoCE. It mainly includes that configure DMAE
> user, initialize doorbell and raq operations, enable port.
>
> Signed-off-by: Wei Hu <xavier.huwei@xxxxxxxxxx>
> Signed-off-by: Nenglong Zhao <zhaonenglong@xxxxxxxxxxxxx>
> Signed-off-by: Lijun Ou <oulijun@xxxxxxxxxx>
> ---
> drivers/infiniband/hw/hns/hns_roce_common.h | 107 +++++++
> drivers/infiniband/hw/hns/hns_roce_device.h | 15 +
> drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 477 ++++++++++++++++++++++++++++
> drivers/infiniband/hw/hns/hns_roce_hw_v1.h | 68 +++-
> drivers/infiniband/hw/hns/hns_roce_main.c | 20 ++
> 5 files changed, 686 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h
> index 5998778..73c6220 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_common.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_common.h
> @@ -53,6 +53,93 @@
> #define roce_set_bit(origin, shift, val) \
> roce_set_field((origin), (1ul << (shift)), (shift), (val))
>
> +#define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
> +#define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
> +
> +#define ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S 5
> +
> +#define ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S 6
> +
> +#define ROCEE_GLB_CFG_ROCEE_PORT_ST_S 10
> +#define ROCEE_GLB_CFG_ROCEE_PORT_ST_M \
> + (((1UL << 6) - 1) << ROCEE_GLB_CFG_ROCEE_PORT_ST_S)
> +
> +#define ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S 16
> +
> +#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S 0
> +#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M \
> + (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S)
> +
> +#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S 24
> +#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M \
> + (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S)
> +
> +#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S 0
> +#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M \
> + (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S)
> +
> +#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S 24
> +#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M \
> + (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S)
> +
> +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S 0
> +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M \
> + (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S)
> +
> +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S 16
> +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M \
> + (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S)
> +
> +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S 0
> +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M \
> + (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S)
> +
> +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S 16
> +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M \
> + (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S)
> +
> +#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_S 0
> +#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_M \
> + (((1UL << 8) - 1) << ROCEE_RAQ_WL_ROCEE_RAQ_WL_S)
> +
> +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S 0
> +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M \
> + (((1UL << 15) - 1) << \
> + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S)
> +
> +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S 16
> +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M \
> + (((1UL << 4) - 1) << \
> + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S)
> +
> +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S 20
> +
> +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE 21
> +
> +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S 0
> +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M \
> + (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S)
> +
> +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S 5
> +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M \
> + (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S)
> +
> +#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S 0
> +#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M \
> + (((1UL << 5) - 1) << ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S)
> +
> +#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S 5
> +#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M \
> + (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S)
> +
> +#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S 0
> +#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M \
> + (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S)
> +
> +#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S 8
> +#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M \
> + (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S)
> +
> #define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S 0
> #define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M \
> (((1UL << 19) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S)
> @@ -120,6 +207,26 @@
> #define ROCEE_ECC_CERR_ALM2_REG 0xB48
>
> #define ROCEE_ACK_DELAY_REG 0x14
> +#define ROCEE_GLB_CFG_REG 0x18
> +
> +#define ROCEE_DMAE_USER_CFG1_REG 0x40
> +#define ROCEE_DMAE_USER_CFG2_REG 0x44
> +
> +#define ROCEE_DB_SQ_WL_REG 0x154
> +#define ROCEE_DB_OTHERS_WL_REG 0x158
> +#define ROCEE_RAQ_WL_REG 0x15C
> +#define ROCEE_WRMS_POL_TIME_INTERVAL_REG 0x160
> +#define ROCEE_EXT_DB_SQ_REG 0x164
> +#define ROCEE_EXT_DB_SQ_H_REG 0x168
> +#define ROCEE_EXT_DB_OTH_REG 0x16C
> +
> +#define ROCEE_EXT_DB_OTH_H_REG 0x170
> +#define ROCEE_EXT_DB_SQ_WL_EMPTY_REG 0x174
> +#define ROCEE_EXT_DB_SQ_WL_REG 0x178
> +#define ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG 0x17C
> +#define ROCEE_EXT_DB_OTHERS_WL_REG 0x180
> +#define ROCEE_EXT_RAQ_REG 0x184
> +#define ROCEE_EXT_RAQ_H_REG 0x188
>
> #define ROCEE_CAEP_CE_INTERVAL_CFG_REG 0x190
> #define ROCEE_CAEP_CE_BURST_NUM_CFG_REG 0x194
> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
> index 46f5d9f..fa906c1 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_device.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
> @@ -64,6 +64,8 @@
> /* Address shift 44bit with the special hardware address operation of RoCEE */
> #define ADDR_SHIFT_44 44
>
> +#define PAGES_SHIFT_16 16
> +
> enum hns_roce_event {
> HNS_ROCE_EVENT_TYPE_PATH_MIG = 0x01,
> HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED = 0x02,
> @@ -120,6 +122,9 @@ enum {
> HNS_ROCE_CMD_SUCCESS = 1,
> };
>
> +#define HNS_ROCE_PORT_DOWN 0
> +#define HNS_ROCE_PORT_UP 1
> +
> struct hns_roce_uar {
> u64 pfn;
> unsigned long index;
> @@ -205,6 +210,13 @@ struct hns_roce_cq_table {
> struct hns_roce_icm_table table;
> };
>
> +struct hns_roce_raq_table {
> + void __iomem *e_raq_addr;
> + void __iomem *e_raq_wl_addr;
> + void __iomem *e_raq_shift_addr;
> + struct hns_roce_buf_list *e_raq_buf;
> +};
> +
> struct hns_roce_cmd_context {
> struct completion done;
> int result;
> @@ -326,6 +338,9 @@ struct hns_roce_caps {
> struct hns_roce_hw {
> int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
> void (*hw_profile)(struct hns_roce_dev *hr_dev);
> + int (*hw_init)(struct hns_roce_dev *hr_dev);
> + void (*hw_uninit)(struct hns_roce_dev *hr_dev);

uninit -> "exit or cleanup", it is relevant to whole patch.

> + void *priv;
> };
>
> struct hns_roce_dev {
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
> index dd92012..9cd6c1a 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
> @@ -39,6 +39,431 @@
> #include "hns_roce_device.h"
> #include "hns_roce_hw_v1.h"
>
> +void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev, int sdb_mode,
> + int odb_mode)
> +{
> + u32 val;
> +
> + val = roce_readl(hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
> + roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
> + roce_writel(val, hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> +}
> +
> +void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev,
> + u32 sdb_mode, u32 odb_mode)
> +{
> + u32 val;
> +
> + /* Configure SDB/ODB extend mode */
> + val = roce_readl(hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + roce_set_bit(val, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
> + roce_set_bit(val, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
> + roce_writel(val, hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> +}
> +
> +void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept, u32 sdb_alful)
> +{
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_db_table *db;
> + u32 val;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + db = &priv->db_table;
> +
> + db->sdb_almept = sdb_alept;
> + db->sdb_almful = sdb_alful;
> +
> + /* Configure SDB */
> + val = roce_readl(hr_dev->reg_base + ROCEE_DB_SQ_WL_REG);
> + roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
> + ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, db->sdb_almful);
> + roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
> + ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, db->sdb_almept);
> + roce_writel(val, hr_dev->reg_base + ROCEE_DB_SQ_WL_REG);
> +}
> +
> +void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept, u32 odb_alful)
> +{
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_db_table *db;
> + u32 val;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + db = &priv->db_table;
> +
> + db->odb_almept = odb_alept;
> + db->odb_almful = odb_alful;
> +
> + /* Configure ODB */
> + val = roce_readl(hr_dev->reg_base + ROCEE_DB_OTHERS_WL_REG);
> + roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
> + ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, db->odb_almful);
> + roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
> + ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S,
> + db->odb_almept);
> + roce_writel(val, hr_dev->reg_base + ROCEE_DB_OTHERS_WL_REG);
> +}
> +
> +void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
> + u32 ext_sdb_alful)
> +{
> + struct device *dev = &hr_dev->pdev->dev;
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_db_table *db;
> + dma_addr_t sdb_dma_addr;
> + u32 val;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + db = &priv->db_table;
> +
> + db->ext_db->esdb_almept = ext_sdb_alept;
> + db->ext_db->esdb_alful = ext_sdb_alful;
> +
> + /* Configure extend SDB threshold */
> + roce_writel(db->ext_db->esdb_almept,
> + hr_dev->reg_base + ROCEE_EXT_DB_SQ_WL_EMPTY_REG);
> + roce_writel(db->ext_db->esdb_alful,
> + hr_dev->reg_base + ROCEE_EXT_DB_SQ_WL_REG);
> +
> + /* Configure extend SDB base addr */
> + sdb_dma_addr = db->ext_db->sdb_buf_list->map;
> + roce_writel((u32)(sdb_dma_addr >> ADDR_SHIFT_12),
> + hr_dev->reg_base + ROCEE_EXT_DB_SQ_REG);
> +
> + /* Configure extend SDB depth */
> + val = roce_readl(hr_dev->reg_base + ROCEE_EXT_DB_SQ_H_REG);
> + roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
> + ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
> + db->ext_db->esdb_dep);
> + roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
> + ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S,
> + sdb_dma_addr >> ADDR_SHIFT_44);
> + roce_writel(val, hr_dev->reg_base + ROCEE_EXT_DB_SQ_H_REG);
> +
> + dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
> + dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
> + db->ext_db->esdb_almept, db->ext_db->esdb_alful);
> +}
> +
> +void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
> + u32 ext_odb_alful)
> +{
> + struct device *dev = &hr_dev->pdev->dev;
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_db_table *db;
> + dma_addr_t odb_dma_addr;
> + u32 val;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + db = &priv->db_table;
> +
> + db->ext_db->eodb_almept = ext_odb_alept;
> + db->ext_db->eodb_alful = ext_odb_alful;
> +
> + /* Configure extend ODB threshold */
> + roce_writel(db->ext_db->eodb_almept,
> + hr_dev->reg_base + ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG);
> + roce_writel(db->ext_db->eodb_alful,
> + hr_dev->reg_base + ROCEE_EXT_DB_OTHERS_WL_REG);
> +
> + /* Configure extend ODB base addr */
> + odb_dma_addr = db->ext_db->odb_buf_list->map;
> + roce_writel((u32)(odb_dma_addr >> ADDR_SHIFT_12),
> + hr_dev->reg_base + ROCEE_EXT_DB_OTH_REG);
> +
> + /* Configure extend ODB depth */
> + val = roce_readl(hr_dev->reg_base + ROCEE_EXT_DB_OTH_H_REG);
> + roce_set_field(val, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
> + ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
> + db->ext_db->eodb_dep);
> + roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
> + ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
> + db->ext_db->eodb_dep);
> + roce_writel(val, hr_dev->reg_base + ROCEE_EXT_DB_OTH_H_REG);
> +
> + dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
> + dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
> + db->ext_db->eodb_almept, db->ext_db->eodb_alful);
> +}
> +
> +int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
> + u32 odb_ext_mod)
> +{
> + struct device *dev = &hr_dev->pdev->dev;
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_db_table *db;
> + dma_addr_t sdb_dma_addr;
> + dma_addr_t odb_dma_addr;
> + int ret = 0;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + db = &priv->db_table;
> +
> + db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
> + if (!db->ext_db) {
> + ret = -ENOMEM;
> + dev_err(dev, "extend db buf alloc fail\n");

There is no need to print error/debug messages for memory allocation
errors. Memory allocator will print it for you.

> + return ret;
> + }
> +
> + if (sdb_ext_mod) {
> + db->ext_db->sdb_buf_list = kmalloc(
> + sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
> + if (!db->ext_db->sdb_buf_list) {
> + ret = -ENOMEM;
> + dev_err(dev, "sdb buf alloc failed\n");

The same comment as above.
It is relevant to all patches.

> + goto ext_sdb_buf_fail_out;
> + }
> +
> + db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
> + HNS_ROCE_V1_EXT_SDB_SIZE,
> + &sdb_dma_addr, GFP_KERNEL);
> + if (!db->ext_db->sdb_buf_list->buf) {
> + ret = -ENOMEM;
> + dev_err(dev, "Send queue db buf alloc fail\n");
> + goto alloc_sq_db_buf_fail;
> + }
> + db->ext_db->sdb_buf_list->map = sdb_dma_addr;
> +
> + db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
> + hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
> + HNS_ROCE_V1_EXT_SDB_ALFUL);
> + } else
> + hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
> + HNS_ROCE_V1_SDB_ALFUL);
> +
> + if (odb_ext_mod) {
> + db->ext_db->odb_buf_list = kmalloc(
> + sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
> + if (!db->ext_db->odb_buf_list) {
> + ret = -ENOMEM;
> + dev_err(dev, "odb buf alloc failed\n");
> + goto ext_odb_buf_fail_out;
> + }
> +
> + db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
> + HNS_ROCE_V1_EXT_ODB_SIZE,
> + &odb_dma_addr, GFP_KERNEL);
> + if (!db->ext_db->odb_buf_list->buf) {
> + ret = -ENOMEM;
> + dev_err(dev, "Other queue db buf alloc fail\n");
> + goto alloc_otr_db_buf_fail;
> + }
> + db->ext_db->odb_buf_list->map = odb_dma_addr;
> +
> + db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
> + hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
> + HNS_ROCE_V1_EXT_ODB_ALFUL);
> + } else
> + hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
> + HNS_ROCE_V1_ODB_ALFUL);
> +
> + hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
> +
> + return 0;
> +
> +alloc_otr_db_buf_fail:
> + kfree(db->ext_db->odb_buf_list);
> +
> +ext_odb_buf_fail_out:
> + if (sdb_ext_mod) {
> + dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
> + db->ext_db->sdb_buf_list->buf,
> + db->ext_db->sdb_buf_list->map);
> + }
> +
> +alloc_sq_db_buf_fail:
> + if (sdb_ext_mod)
> + kfree(db->ext_db->sdb_buf_list);
> +
> +ext_sdb_buf_fail_out:
> + kfree(db->ext_db);
> + return ret;
> +}
> +
> +int hns_roce_db_init(struct hns_roce_dev *hr_dev)
> +{
> + struct device *dev = &hr_dev->pdev->dev;
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_db_table *db;
> + u32 sdb_ext_mod;
> + u32 odb_ext_mod;
> + u32 sdb_evt_mod;
> + u32 odb_evt_mod;
> + int ret = 0;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + db = &priv->db_table;
> +
> + memset(db, 0, sizeof(*db));
> +
> + /* Default DB mode */
> + sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
> + odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
> + sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
> + odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
> +
> + db->sdb_ext_mod = sdb_ext_mod;
> + db->odb_ext_mod = odb_ext_mod;
> +
> + /* Init extend DB */
> + ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
> + if (ret) {
> + dev_err(dev, "Failed in extend DB configuration.\n");
> + return ret;
> + }
> +
> + hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
> +
> + return 0;
> +}
> +
> +void hns_roce_db_free(struct hns_roce_dev *hr_dev)
> +{
> + struct device *dev = &hr_dev->pdev->dev;
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_db_table *db;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + db = &priv->db_table;
> +
> + if (db->sdb_ext_mod) {
> + dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
> + db->ext_db->sdb_buf_list->buf,
> + db->ext_db->sdb_buf_list->map);
> + kfree(db->ext_db->sdb_buf_list);
> + }
> +
> + if (db->odb_ext_mod) {
> + dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
> + db->ext_db->odb_buf_list->buf,
> + db->ext_db->odb_buf_list->map);
> + kfree(db->ext_db->odb_buf_list);
> + }
> +
> + kfree(db->ext_db);
> +}
> +
> +int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
> +{
> + int ret;
> + int raq_shift = 0;
> + dma_addr_t addr;
> + u32 val;
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_raq_table *raq;
> + struct device *dev = &hr_dev->pdev->dev;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + raq = &priv->raq_table;
> +
> + raq->e_raq_addr = hr_dev->reg_base + ROCEE_EXT_RAQ_REG;
> + raq->e_raq_wl_addr = hr_dev->reg_base + ROCEE_RAQ_WL_REG;
> + raq->e_raq_shift_addr = hr_dev->reg_base + ROCEE_EXT_RAQ_H_REG;
> +
> + raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
> + if (!raq->e_raq_buf) {
> + ret = -ENOMEM;
> + dev_err(dev, "Failed to alloc raq buf, Aborting.\n");
> + return ret;
> + }
> +
> + raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
> + &addr, GFP_KERNEL);
> + if (!raq->e_raq_buf->buf) {
> + ret = -ENOMEM;
> + dev_err(dev, "Failed to dma_alloc ext raq buf.\n");
> + goto _err_dma_alloc_raq;
> + }
> + raq->e_raq_buf->map = addr;
> +
> + /* Configure raq extended address. 48bit 4K align*/
> + roce_writel(raq->e_raq_buf->map >> ADDR_SHIFT_12, raq->e_raq_addr);
> +
> + /* Configure raq_shift */
> + raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
> + val = roce_readl(raq->e_raq_shift_addr);
> + roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
> + ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S,
> + raq_shift);
> + roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
> + ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
> + raq->e_raq_buf->map >> ADDR_SHIFT_44);
> + roce_writel(val, raq->e_raq_shift_addr);
> + dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
> +
> + /* Configure raq threshold */
> + val = roce_readl(raq->e_raq_wl_addr);
> + roce_set_field(val, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
> + ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
> + HNS_ROCE_V1_EXT_RAQ_WF);
> + roce_writel(val, raq->e_raq_wl_addr);
> + dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
> +
> + /* Enable extend raq */
> + val = roce_readl(hr_dev->reg_base + ROCEE_WRMS_POL_TIME_INTERVAL_REG);
> + roce_set_field(val,
> + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
> + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
> + POL_TIME_INTERVAL_VAL);
> + roce_set_bit(val, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
> + roce_set_field(val,
> + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
> + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
> + 2);
> + roce_set_bit(val,
> + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
> + roce_writel(val, hr_dev->reg_base + ROCEE_WRMS_POL_TIME_INTERVAL_REG);
> + dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
> +
> + /* Enable raq drop */
> + val = roce_readl(hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + roce_set_bit(val, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
> + roce_writel(val, hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
> +
> + return 0;
> +
> +_err_dma_alloc_raq:
> + kfree(raq->e_raq_buf);
> + return ret;
> +}
> +
> +void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
> +{
> + struct device *dev = &hr_dev->pdev->dev;
> + struct hns_roce_v1_priv *priv;
> + struct hns_roce_raq_table *raq;
> +
> + priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
> + raq = &priv->raq_table;
> +
> + dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
> + raq->e_raq_buf->map);
> + kfree(raq->e_raq_buf);
> +}
> +
> +void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
> +{
> + u32 val;
> +
> + if (enable_flag) {
> + val = roce_readl(hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + /* Open all ports */
> + roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
> + ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
> + ALL_PORT_VAL_OPEN);
> + roce_writel(val, hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + } else {
> + val = roce_readl(hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + /* Close all ports */
> + roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
> + ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
> + roce_writel(val, hr_dev->reg_base + ROCEE_GLB_CFG_REG);
> + }
> +}
> +
> /**
> * hns_roce_v1_reset - reset roce
> * @hr_dev: roce device struct pointer
> @@ -144,7 +569,59 @@ void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
> caps->max_mtu = IB_MTU_2048;
> }
>
> +int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
> +{
> + int ret;
> + u32 val;
> + struct device *dev = &hr_dev->pdev->dev;
> +
> + /* DMAE user config */
> + val = roce_readl(hr_dev->reg_base + ROCEE_DMAE_USER_CFG1_REG);
> + roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
> + ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
> + roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
> + ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
> + 1 << PAGES_SHIFT_16);
> + roce_writel(val, hr_dev->reg_base + ROCEE_DMAE_USER_CFG1_REG);
> +
> + val = roce_readl(hr_dev->reg_base + ROCEE_DMAE_USER_CFG2_REG);
> + roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
> + ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
> + roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
> + ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
> + 1 << PAGES_SHIFT_16);
> +
> + ret = hns_roce_db_init(hr_dev);
> + if (ret) {
> + dev_err(dev, "doorbell init failed!\n");
> + return ret;
> + }
> +
> + ret = hns_roce_raq_init(hr_dev);
> + if (ret) {
> + dev_err(dev, "raq init failed!\n");
> + goto _error_failed_raq_init;

It is uncommon to set label with underscore.

> + }
> +
> + hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
> +
> + return 0;
> +
> +_error_failed_raq_init:
> + hns_roce_db_free(hr_dev);
> + return ret;
> +}
> +
> +void hns_roce_v1_uninit(struct hns_roce_dev *hr_dev)
> +{
> + hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
> + hns_roce_raq_free(hr_dev);
> + hns_roce_db_free(hr_dev);
> +}
> +
> struct hns_roce_hw hns_roce_hw_v1 = {
> .reset = hns_roce_v1_reset,
> .hw_profile = hns_roce_v1_profile,
> + .hw_init = hns_roce_v1_init,
> + .hw_uninit = hns_roce_v1_uninit,
> };
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
> index 8ddcf58..f538734 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
> @@ -69,7 +69,73 @@
> #define HNS_ROCE_V1_CQE_ENTRY_SIZE 32
> #define HNS_ROCE_V1_PAGE_SIZE_SUPPORT 0xFFFFF000
>

Extra white spaces,
Please clean your patches with ./scripts/cleanfile or
./scripts/cleanpatch

And indentation below seems completely out of sync.

> -#define SLEEP_TIME_INTERVAL 20
> +#define HNS_ROCE_V1_EXT_RAQ_WF 8
> +#define HNS_ROCE_V1_RAQ_ENTRY 64
> +#define HNS_ROCE_V1_RAQ_DEPTH 32768
> +#define HNS_ROCE_V1_RAQ_SIZE (HNS_ROCE_V1_RAQ_ENTRY * HNS_ROCE_V1_RAQ_DEPTH)
> +
> +#define HNS_ROCE_V1_SDB_DEPTH 0x400
> +#define HNS_ROCE_V1_ODB_DEPTH 0x400
> +
> +#define HNS_ROCE_V1_DB_RSVD 0x80
> +
> +#define HNS_ROCE_V1_SDB_ALEPT HNS_ROCE_V1_DB_RSVD
> +#define HNS_ROCE_V1_SDB_ALFUL (HNS_ROCE_V1_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)
> +#define HNS_ROCE_V1_ODB_ALEPT HNS_ROCE_V1_DB_RSVD
> +#define HNS_ROCE_V1_ODB_ALFUL (HNS_ROCE_V1_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
> +
> +#define HNS_ROCE_V1_EXT_SDB_DEPTH 0x4000
> +#define HNS_ROCE_V1_EXT_ODB_DEPTH 0x4000
> +#define HNS_ROCE_V1_EXT_SDB_ENTRY 16
> +#define HNS_ROCE_V1_EXT_ODB_ENTRY 16
> +#define HNS_ROCE_V1_EXT_SDB_SIZE \
> + (HNS_ROCE_V1_EXT_SDB_DEPTH * HNS_ROCE_V1_EXT_SDB_ENTRY)
> +#define HNS_ROCE_V1_EXT_ODB_SIZE \
> + (HNS_ROCE_V1_EXT_ODB_DEPTH * HNS_ROCE_V1_EXT_ODB_ENTRY)
> +
> +#define HNS_ROCE_V1_EXT_SDB_ALEPT HNS_ROCE_V1_DB_RSVD
> +#define HNS_ROCE_V1_EXT_SDB_ALFUL \
> + (HNS_ROCE_V1_EXT_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)
> +#define HNS_ROCE_V1_EXT_ODB_ALEPT HNS_ROCE_V1_DB_RSVD
> +#define HNS_ROCE_V1_EXT_ODB_ALFUL \
> + (HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
> +
> +#define HNS_ROCE_ODB_POLL_MODE 0
> +
> +#define HNS_ROCE_SDB_NORMAL_MODE 0
> +#define HNS_ROCE_SDB_EXTEND_MODE 1
> +
> +#define HNS_ROCE_ODB_EXTEND_MODE 1
> +
> +#define ALL_PORT_VAL_OPEN 0x3f
> +#define POL_TIME_INTERVAL_VAL 0x80
> +#define SLEEP_TIME_INTERVAL 20
> +
> +struct hns_roce_ext_db {
> + int esdb_almept;
> + int esdb_alful;
> + int eodb_almept;
> + int eodb_alful;
> + int esdb_dep;
> + int eodb_dep;
> + struct hns_roce_buf_list *sdb_buf_list;
> + struct hns_roce_buf_list *odb_buf_list;
> +};
> +
> +struct hns_roce_db_table {
> + int sdb_ext_mod;
> + int odb_ext_mod;
> + int sdb_almept;
> + int sdb_almful;
> + int odb_almept;
> + int odb_almful;
> + struct hns_roce_ext_db *ext_db;
> +};
> +
> +struct hns_roce_v1_priv {
> + struct hns_roce_db_table db_table;
> + struct hns_roce_raq_table raq_table;
> +};
>
> extern int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool enable);
>
> diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
> index fe487d6..589a9f7 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_main.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_main.c
> @@ -202,6 +202,16 @@ err_unmap_mtt:
> return ret;
> }
>
> +int hns_roce_engine_init(struct hns_roce_dev *hr_dev)
> +{
> + return hr_dev->hw->hw_init(hr_dev);
> +}
> +
> +void hns_roce_engine_uninit(struct hns_roce_dev *hr_dev)
> +{
> + hr_dev->hw->hw_uninit(hr_dev);
> +}
> +
> /**
> * hns_roce_setup_hca - setup host channel adapter
> * @hr_dev: pointer to hns roce device
> @@ -351,6 +361,15 @@ static int hns_roce_probe(struct platform_device *pdev)
> goto error_failed_setup_hca;
> }
>
> + ret = hns_roce_engine_init(hr_dev);
> + if (ret) {
> + dev_err(dev, "hw_init failed!\n");
> + goto error_failed_engine_init;
> + }
> +
> +error_failed_engine_init:
> + hns_roce_cleanup_bitmap(hr_dev);
> +
> error_failed_setup_hca:
> hns_roce_cleanup_icm(hr_dev);
>
> @@ -383,6 +402,7 @@ static int hns_roce_remove(struct platform_device *pdev)
> {
> struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev);
>
> + hns_roce_engine_uninit(hr_dev);
> hns_roce_cleanup_bitmap(hr_dev);
> hns_roce_cleanup_icm(hr_dev);
>
> --
> 1.9.1
>

Attachment: signature.asc
Description: Digital signature

Next message: Arnd Bergmann: "Re: [PATCH v3] Axi-usb: Add support for 64-bit addressing."
Previous message: Sinan Kaya: "Re: [PATCH] PCI/ASPM: fix reverse ASPM L0s assignment of upstream and downstream"
In reply to: Lijun Ou: "[PATCH v8 10/22] IB/hns: Add process flow to init RoCE engine"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]