Re: [PATCH v6 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Zong Li]

On Wed, Aug 26, 2026 at 4:28 PM Yicong Yang <yang.yicong@xxxxxxxxxxxxx> wrote:
>
> On 8/24/26 10:20 AM, Zong Li wrote:
> > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > device driver created by the parent RISC-V IOMMU driver.
> >
> > The performance monitor provides counters with filtering support to
> > collect events for specific device ID/process ID, or GSCID/PSCID.
> >
> > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > The cycle counter is not associated with iohpmevt0, so a software-defined
> > cycle event is required for the perf subsystem.
> >
> > The number and width of the counters are hardware-implemented and must
> > be detected at runtime.
> >
> > Leave out all the dead cleanup code (i.e. .remove() operation) if the
> > PMU driver is tied to the IOMMU driver and can never realistically be
> > removed.
> >
> > PMU-related definitions are moved into the perf driver, where they are
> > used exclusively.
> >
> > According to RISC-V IOMMU specification Chapter 6:
> > Whether an 8 byte access to an IOMMU register is single-copy atomic is
> > UNSPECIFIED. Use two separate 4 byte accesses for hardware
> > compatibility.
> >
> > Tested-by: Chen Pei <cp0613@xxxxxxxxxxxxxxxxx>
> > Tested-by: Fangyu Yu <fangyu.yu@xxxxxxxxxxxxxxxxx>
> > Reviewed-by: Guo Ren (Alibaba DAMO Academy) <guoren@xxxxxxxxxx>
> > Suggested-by: David Laight <david.laight.linux@xxxxxxxxx>
> > Suggested-by: Guo Ren <guoren@xxxxxxxxxx>
> > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > Signed-off-by: Zong Li <zong.li@xxxxxxxxxx>
> > ---
> >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> >  drivers/perf/Kconfig             |  12 +
> >  drivers/perf/Makefile            |   1 +
> >  drivers/perf/riscv_iommu_pmu.c   | 813 +++++++++++++++++++++++++++++++
> >  4 files changed, 826 insertions(+), 61 deletions(-)
> >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> >
>
> [...]
>
> > +
> > +/* cpumask */
> > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > +                                     struct device_attribute *attr,
> > +                                     char *buf)
> > +{
> > +     struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > +
> > +     return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
>
> needs to use sysfs_emit(), cpumap_print_to_pagebuf is removed in 7.3-rc
>
> https://lore.kernel.org/lkml/20260702154725.185376-1-ynorov@xxxxxxxxxx/
>

I will fix it in the next version. Thanks

> > +}
> > +
>
> [...]
>
> > +
> > +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
> > +                              const struct auxiliary_device_id *id)
> > +{
> > +     struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
> > +     struct riscv_iommu_pmu *iommu_pmu;
> > +     void __iomem *addr;
> > +     char *name;
> > +     int ret;
> > +
> > +     iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
> > +     if (!iommu_pmu)
> > +             return -ENOMEM;
> > +
> > +     iommu_pmu->reg = iommu_dev->reg;
> > +
> > +     /*
> > +      * Counter number and width are hardware-implemented.
> > +      * Detect them by write 1s. Counter should be consecutive.
> > +      */
> > +     addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > +     writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
> > +     iommu_pmu->num_counters = hweight32(readl(addr));
> > +
> > +     addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
> > +     riscv_iommu_pmu_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> > +     iommu_pmu->cycle_cntr_mask = riscv_iommu_pmu_readq(addr);
> > +
> > +     /* Assume the width of all event counters are the same */
> > +     addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
> > +     riscv_iommu_pmu_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> > +     iommu_pmu->event_cntr_mask = riscv_iommu_pmu_readq(addr);
> > +
> > +     iommu_pmu->pmu = (struct pmu) {
> > +             .module         = THIS_MODULE,
> > +             .parent         = &auxdev->dev,
> > +             .task_ctx_nr    = perf_invalid_context,
> > +             .event_init     = riscv_iommu_pmu_event_init,
> > +             .add            = riscv_iommu_pmu_add,
> > +             .del            = riscv_iommu_pmu_del,
> > +             .start          = riscv_iommu_pmu_start,
> > +             .stop           = riscv_iommu_pmu_stop,
> > +             .read           = riscv_iommu_pmu_read,
> > +             .attr_groups    = riscv_iommu_pmu_attr_grps,
> > +             .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
> > +     };
> > +
> > +     auxiliary_set_drvdata(auxdev, iommu_pmu);
> > +
> > +     name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
> > +                           "riscv_iommu_pmu_%u", auxdev->id);
> > +     if (!name) {
> > +             dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu%u\n",
> > +                     auxdev->id);
> > +             return -ENOMEM;
> > +     }
> > +
> > +     iommu_pmu->numa_node = dev_to_node(iommu_dev->dev);
> > +     iommu_pmu->irq = riscv_iommu_pmu_get_irq_num(iommu_dev);
> > +
> > +     ret = riscv_iommu_pmu_request_irq(auxdev, iommu_dev, iommu_pmu);
> > +     if (ret) {
> > +             dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
> > +             return ret;
> > +     }
> > +
> > +     /*
> > +      * Bind all events to the same cpu context to avoid race enabling.
> > +      * Pick a local CPU from the numa node of this iommu for the locality.
> > +      *
> > +      * Hold the hotplug lock until this instance is registered, otherwise
> > +      * the CPU picked here could go offline in between. The teardown
> > +      * callback does not run for an instance which is not registered yet,
> > +      * so nothing would move the PMU off that CPU afterwards.
> > +      */
> > +     cpus_read_lock();
> > +
> > +     iommu_pmu->on_cpu = cpumask_local_spread(0, iommu_pmu->numa_node);
> > +     ret = irq_set_affinity(iommu_pmu->irq, cpumask_of(iommu_pmu->on_cpu));
> > +     if (ret)
> > +             dev_dbg(&auxdev->dev, "failed to set irq %u affinity to cpu %u: %d\n",
> > +                     iommu_pmu->irq, iommu_pmu->on_cpu, ret);
> > +
> > +     ret = cpuhp_state_add_instance_nocalls_cpuslocked(cpuhp_state,
> > +                                                      &iommu_pmu->node);
> > +
> > +     cpus_read_unlock();
>
> it could be simplified to init the ->on_cpu in the hotplug handler without
> holding the cpu hotplug lock like below:
>
> iommu_pmu->on_cpu = -1;
> ret = cpuhp_state_add_instance() /* init the ->on_cpu in the hotplug handler */

It seems we might lose the benefits of cumask_local_spread this way.

>
> otherwise looks good to me.
>
> Reviewed-by: Yicong Yang <yang.yicong@xxxxxxxxxxxxx>
>
> > +
> > +     if (ret) {
> > +             dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
> > +             return ret;
> > +     }
> > +
> > +     ret = devm_add_action_or_reset(&auxdev->dev,
> > +                                    riscv_iommu_pmu_remove_cpuhp_instance,
> > +                                    iommu_pmu);
> > +     if (ret)
> > +             return ret;
> > +
> > +     ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
> > +     if (ret) {
> > +             dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
> > +             return ret;
> > +     }
> > +
> > +     ret = devm_add_action_or_reset(&auxdev->dev,
> > +                                    riscv_iommu_pmu_do_unregister,
> > +                                    iommu_pmu);
> > +     if (ret)
> > +             return ret;
> > +
> > +     /*
> > +      * The PMU name only carries the aux dev id, not the iommu dev name, so
> > +      * find the iommu dev name here to map this PMU back to its iommu dev.
> > +      */
> > +     dev_info(&auxdev->dev, "%s: Registered with %d counters (iommu %s)\n",
> > +              name, iommu_pmu->num_counters, dev_name(iommu_dev->dev));
> > +
> > +     return 0;
> > +}
> > +
> > +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
> > +     { .name = "riscv-iommu.pmu" },
> > +     {}
> > +};
> > +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
> > +
> > +static struct auxiliary_driver iommu_pmu_driver = {
> > +     .driver = {
> > +             .suppress_bind_attrs = true,
> > +     },
> > +     .probe          = riscv_iommu_pmu_probe,
> > +     .id_table       = riscv_iommu_pmu_id_table,
> > +};
> > +
> > +static int __init riscv_iommu_pmu_init(void)
> > +{
> > +     int ret;
> > +
> > +     cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
> > +                                           "perf/riscv/iommu:online",
> > +                                           riscv_iommu_pmu_online_cpu,
> > +                                           riscv_iommu_pmu_offline_cpu);
> > +     if (cpuhp_state < 0)
> > +             return cpuhp_state;
> > +
> > +     ret = auxiliary_driver_register(&iommu_pmu_driver);
> > +     if (ret)
> > +             cpuhp_remove_multi_state(cpuhp_state);
> > +
> > +     return ret;
> > +}
> > +module_init(riscv_iommu_pmu_init);
> > +
> > +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
> > +MODULE_LICENSE("GPL");