Re: [PATCH] PCI: qcom-ep: Add ICC bandwidth voting support

[Krishna Chaitanya Chundru]

On 6/6/2023 4:18 PM, Georgi Djakov wrote:
> Hi Krishna,
>
> Thanks for the patch!
>
> On 6.06.23 8:49, Krishna chaitanya chundru wrote:
> > Add support to vote for ICC bandwidth based up on the link
> > speed and width.
> >
> > Signed-off-by: Krishna chaitanya chundru <quic_krichai@xxxxxxxxxxx>
> > ---
> >   drivers/pci/controller/dwc/pcie-qcom-ep.c | 73 
> > +++++++++++++++++++++++++++++++
> >   1 file changed, 73 insertions(+)
> >
> > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c 
> > b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> > index 19b3283..79e7559 100644
> > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
> > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
> > @@ -17,6 +17,7 @@
> >   #include <linux/phy/pcie.h>
> >   #include <linux/phy/phy.h>
> >   #include <linux/platform_device.h>
> > +#include <linux/interconnect.h>
> >   #include <linux/pm_domain.h>
> >   #include <linux/regmap.h>
> >   #include <linux/reset.h>
> > @@ -28,6 +29,7 @@
> >   #define PARF_SYS_CTRL                0x00
> >   #define PARF_DB_CTRL                0x10
> >   #define PARF_PM_CTRL                0x20
> > +#define PARF_PM_STTS                0x24
> >   #define PARF_MHI_CLOCK_RESET_CTRL        0x174
> >   #define PARF_MHI_BASE_ADDR_LOWER        0x178
> >   #define PARF_MHI_BASE_ADDR_UPPER        0x17c
> > @@ -128,6 +130,9 @@
> >   /* DBI register fields */
> >   #define DBI_CON_STATUS_POWER_STATE_MASK        GENMASK(1, 0)
> >   +#define DBI_LINKCTRLSTATUS            0x80
> > +#define DBI_LINKCTRKSTATUS_SHIFT    16
> > +
> >   #define XMLH_LINK_UP                0x400
> >   #define CORE_RESET_TIME_US_MIN            1000
> >   #define CORE_RESET_TIME_US_MAX            1005
> > @@ -187,6 +192,8 @@ struct qcom_pcie_ep {
> >       enum qcom_pcie_ep_link_status link_status;
> >       int global_irq;
> >       int perst_irq;
> > +
> > +    struct icc_path *icc;
> >   };
> >     static int qcom_pcie_ep_core_reset(struct qcom_pcie_ep *pcie_ep)
> > @@ -253,9 +260,56 @@ static void qcom_pcie_dw_stop_link(struct 
> > dw_pcie *pci)
> >       disable_irq(pcie_ep->perst_irq);
> >   }
> >   +static void qcom_pcie_icc_update(struct qcom_pcie_ep *pcie_ep)
> > +{
> > +    struct dw_pcie *pci = &pcie_ep->pci;
> > +    u32 val, bw;
> > +    int speed, width;
> > +    int ret;
> > +
> > +    if (!pcie_ep->icc)
> > +        return;
> > +
> > +    val = dw_pcie_readl_dbi(pci, DBI_LINKCTRLSTATUS);
> > +    val = val >> DBI_LINKCTRKSTATUS_SHIFT;
> > +
> > +    speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);
> > +    width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
> > +
> > +    /*
> > +     * ICC needs avg bw in KBps.
> > +     *
> > +     * For example for 2Gbps the avg BW = 2x1000x1000x1000/8*1000 = 
> > 250000
> > +     */
> > +    switch (speed) {
> > +    case 1:
> > +        bw = 250000;    /* avg bw for GEN1 per lane: 2Gbps, peak bw: 
> > no vote */
> > +        break;
> > +    case 2:
> > +        bw = 500000;    /* avg bw for GEN2 per lane: 4Gbps, peak bw 
> > no vote */
> > +        break;
> > +    case 3:
> > +        bw = 1000000;    /* avg bw for GEN3 per lane: 8Gbps, peak bw 
> > no vote */
> > +        break;
> > +    default:
> > +        WARN_ON_ONCE(1);
> > +        fallthrough;
> > +    case 4:
> > +        bw = 2000000;    /* avg bw for GEN4 per lane: 16Gbps, peak 
> > bw no vote */
> > +        break;
> > +    }
> > +
> > +    ret = icc_set_bw(pcie_ep->icc, width * bw, 0);
>
> Here you should use a non-zero value for peak bandwidth. You can use 
> the average value also as peak. There are some existing macros like 
> GBps_to_icc(). Please use them.
updated the logic in the new patch.
> > +    if (ret) {
> > +        dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
> > +            ret);
> > +    }
> > +}
> > +
> >   static int qcom_pcie_enable_resources(struct qcom_pcie_ep *pcie_ep)
> >   {
> >       int ret;
> > +    struct dw_pcie *pci = &pcie_ep->pci;
> >         ret = clk_bulk_prepare_enable(pcie_ep->num_clks, pcie_ep->clks);
> >       if (ret)
> > @@ -277,6 +331,20 @@ static int qcom_pcie_enable_resources(struct 
> > qcom_pcie_ep *pcie_ep)
> >       if (ret)
> >           goto err_phy_exit;
> >   +    /*
> > +     * Some Qualcomm platforms require interconnect bandwidth 
> > constraints
> > +     * to be set before enabling interconnect clocks.
> > +     *
> > +     * Set an initial average bandwidth corresponding to single-lane 
> > Gen 1
> > +     * for the pcie to mem path.
> > +     */
> > +    ret = icc_set_bw(pcie_ep->icc, 250000, 0); /* avg bw: 2Gbps, 
> > peak bw: no vote */
>
> Ditto.
done
> > +    if (ret) {
> > +        dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
> > +            ret);
> > +        goto err_phy_exit;
> > +    }
> > +
> >       return 0;
> >     err_phy_exit:
> > @@ -550,6 +618,10 @@ static int qcom_pcie_ep_get_resources(struct 
> > platform_device *pdev,
> >       if (IS_ERR(pcie_ep->phy))
> >           ret = PTR_ERR(pcie_ep->phy);
> >   +    pcie_ep->icc = devm_of_icc_get(dev, "pci");
>
> Is this "pci" path documented in the bindings?
>
> Thanks,
> Georgi
Added the bindings patch in new patch

Thanks,

Krishna Chaitanya.

> > +    if (IS_ERR(pcie_ep->icc))
> > +        ret = PTR_ERR(pcie_ep->icc);
> > +
> >       return ret;
> >   }
> >   @@ -572,6 +644,7 @@ static irqreturn_t 
> > qcom_pcie_ep_global_irq_thread(int irq, void *data)
> >       } else if (FIELD_GET(PARF_INT_ALL_BME, status)) {
> >           dev_dbg(dev, "Received BME event. Link is enabled!\n");
> >           pcie_ep->link_status = QCOM_PCIE_EP_LINK_ENABLED;
> > +        qcom_pcie_icc_update(pcie_ep);
> >       } else if (FIELD_GET(PARF_INT_ALL_PM_TURNOFF, status)) {
> >           dev_dbg(dev, "Received PM Turn-off event! Entering L23\n");
> >           val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL);