Re: [PATCH v2 2/4] hwtracing: hisi_ptt: Add support for dynamically updating the filter list

[Yicong Yang]

On 2023/3/31 15:03, Yicong Yang wrote:
> From: Yicong Yang <yangyicong@xxxxxxxxxxxxx>
> 
> The PCIe devices supported by the PTT trace can be removed/rescanned by
> hotplug or through sysfs.  Add support for dynamically updating the
> available filter list by registering a PCI bus notifier block. Then user
> can always get latest information about available tracing filters and
> driver can block the invalid filters of which related devices no longer
> exist in the system.
> 
> Signed-off-by: Yicong Yang <yangyicong@xxxxxxxxxxxxx>
> ---
>  Documentation/trace/hisi-ptt.rst |   6 +-
>  drivers/hwtracing/ptt/hisi_ptt.c | 146 ++++++++++++++++++++++++++++++-
>  drivers/hwtracing/ptt/hisi_ptt.h |  36 ++++++++
>  3 files changed, 184 insertions(+), 4 deletions(-)
> 
> diff --git a/Documentation/trace/hisi-ptt.rst b/Documentation/trace/hisi-ptt.rst
> index 4f87d8e21065..3641aca4287a 100644
> --- a/Documentation/trace/hisi-ptt.rst
> +++ b/Documentation/trace/hisi-ptt.rst
> @@ -153,9 +153,9 @@ Endpoint function can be specified in one trace. Specifying both Root Port
>  and function at the same time is not supported. Driver maintains a list of
>  available filters and will check the invalid inputs.
>  
> -Currently the available filters are detected in driver's probe. If the supported
> -devices are removed/added after probe, you may need to reload the driver to update
> -the filters.
> +The available filters will be dynamically updates, which means you will always
> +get correct filter information when hotplug events happen, or when you manually
> +remove/rescan the devices.
>  
>  2. Type
>  -------
> diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c
> index bbbee8c2d7fd..b16bc18421cd 100644
> --- a/drivers/hwtracing/ptt/hisi_ptt.c
> +++ b/drivers/hwtracing/ptt/hisi_ptt.c
> @@ -357,6 +357,9 @@ static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt)
>  static void hisi_ptt_del_free_filters(struct hisi_ptt *hisi_ptt,
>  				      struct hisi_ptt_filter_desc *filter)
>  {
> +	if (filter->is_port)
> +		hisi_ptt->port_mask &= ~hisi_ptt_get_filter_val(filter->devid, true);
> +
>  	list_del(&filter->list);
>  	kfree(filter);
>  }
> @@ -387,6 +390,108 @@ hisi_ptt_alloc_add_filters(struct hisi_ptt *hisi_ptt, struct pci_dev *pdev)
>  	return filter;
>  }
>  
> +static void hisi_ptt_update_filters(struct work_struct *work)
> +{
> +	struct delayed_work *delayed_work = to_delayed_work(work);
> +	struct hisi_ptt_filter_update_info info;
> +	struct hisi_ptt_filter_desc *filter;
> +	struct hisi_ptt *hisi_ptt;
> +
> +	hisi_ptt = container_of(delayed_work, struct hisi_ptt, work);
> +
> +	if (!mutex_trylock(&hisi_ptt->filter_lock)) {
> +		schedule_delayed_work(&hisi_ptt->work, HISI_PTT_WORK_DELAY_MS);
> +		return;
> +	}
> +
> +	while (kfifo_get(&hisi_ptt->filter_update_kfifo, &info)) {
> +		if (info.is_add) {
> +			/*
> +			 * Notify the users if failed to add this filter, others
> +			 * still work and available. See the comments in
> +			 * hisi_ptt_init_filters().
> +			 */
> +			filter = hisi_ptt_alloc_add_filters(hisi_ptt, info.pdev);
> +			if (!filter)
> +				continue;
> +		} else {
> +			bool is_port = pci_pcie_type(info.pdev) == PCI_EXP_TYPE_ROOT_PORT;
> +			u16 devid = PCI_DEVID(info.pdev->bus->number, info.pdev->devfn);
> +			struct hisi_ptt_filter_desc *tmp;
> +			struct list_head *target_list;
> +
> +			target_list = is_port ? &hisi_ptt->port_filters :
> +				      &hisi_ptt->req_filters;
> +
> +			list_for_each_entry_safe(filter, tmp, target_list, list)
> +				if (filter->devid == devid) {
> +					hisi_ptt_del_free_filters(hisi_ptt, filter);
> +					break;
> +				}
> +		}
> +	}
> +
> +	mutex_unlock(&hisi_ptt->filter_lock);
> +}
> +
> +static void hisi_ptt_update_fifo_in(struct hisi_ptt *hisi_ptt,
> +				    struct hisi_ptt_filter_update_info *info)
> +{
> +	struct pci_dev *root_port = pcie_find_root_port(info->pdev);
> +	u32 port_devid;
> +
> +	if (!root_port)
> +		return;
> +
> +	port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn);
> +	if (port_devid < hisi_ptt->lower_bdf ||
> +	    port_devid > hisi_ptt->upper_bdf)
> +		return;
> +
> +	/*
> +	 * The FIFO size is 16 which is sufficient for almost all the cases,
> +	 * since each PCIe core will have most 8 Root Ports (typically only
> +	 * 1~4 Root Ports). On failure log the failed filter and let user
> +	 * handle it.
> +	 */
> +	if (kfifo_in_spinlocked(&hisi_ptt->filter_update_kfifo, info, 1,
> +				&hisi_ptt->filter_update_lock))
> +		schedule_delayed_work(&hisi_ptt->work, 0);
> +	else
> +		pci_warn(hisi_ptt->pdev,
> +			 "filter update fifo overflow for target %s\n",
> +			 pci_name(info->pdev));
> +}
> +
> +/*
> + * A PCI bus notifier is used here for dynamically updating the filter
> + * list.
> + */
> +static int hisi_ptt_notifier_call(struct notifier_block *nb, unsigned long action,
> +				  void *data)
> +{
> +	struct hisi_ptt *hisi_ptt = container_of(nb, struct hisi_ptt, hisi_ptt_nb);
> +	struct hisi_ptt_filter_update_info info;
> +	struct device *dev = data;
> +
> +	info.pdev = to_pci_dev(dev);

It's problematic here to directly store the *pdev, since we'll use this and dereference
this pointer in a work thread. On device removal, it's possible that the pointer has been
invalid before we use it later which will cause a NULL pointer dereference in the work
thread.

Will fix this problem by not using this pointer directly but store devid of the test device
to update the filter list instead.

> +
> +	switch (action) {
> +	case BUS_NOTIFY_ADD_DEVICE:
> +		info.is_add = true;
> +		break;
> +	case BUS_NOTIFY_DEL_DEVICE:
> +		info.is_add = false;
> +		break;
> +	default:
> +		return 0;
> +	}
> +
> +	hisi_ptt_update_fifo_in(hisi_ptt, &info);
> +
> +	return 0;
> +}
> +
>  static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data)
>  {
>  	struct pci_dev *root_port = pcie_find_root_port(pdev);
> @@ -466,8 +571,13 @@ static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt)
>  	int ret;
>  	u32 reg;
>  
> +	INIT_DELAYED_WORK(&hisi_ptt->work, hisi_ptt_update_filters);
> +	INIT_KFIFO(hisi_ptt->filter_update_kfifo);
> +	spin_lock_init(&hisi_ptt->filter_update_lock);
> +
>  	INIT_LIST_HEAD(&hisi_ptt->port_filters);
>  	INIT_LIST_HEAD(&hisi_ptt->req_filters);
> +	mutex_init(&hisi_ptt->filter_lock);
>  
>  	ret = hisi_ptt_config_trace_buf(hisi_ptt);
>  	if (ret)
> @@ -633,14 +743,19 @@ static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config)
>  	 * For Requester ID filters, walk the available filter list to see
>  	 * whether we have one matched.
>  	 */
> +	mutex_lock(&hisi_ptt->filter_lock);
>  	if (!hisi_ptt->trace_ctrl.is_port) {
>  		list_for_each_entry(filter, &hisi_ptt->req_filters, list) {
> -			if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port))
> +			if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port)) {
> +				mutex_unlock(&hisi_ptt->filter_lock);
>  				return 0;
> +			}
>  		}
>  	} else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) {
> +		mutex_unlock(&hisi_ptt->filter_lock);
>  		return 0;
>  	}
> +	mutex_unlock(&hisi_ptt->filter_lock);
>  
>  	return -EINVAL;
>  }
> @@ -916,6 +1031,31 @@ static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt)
>  					&hisi_ptt->hisi_ptt_pmu);
>  }
>  
> +static void hisi_ptt_unregister_filter_update_notifier(void *data)
> +{
> +	struct hisi_ptt *hisi_ptt = data;
> +
> +	bus_unregister_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);
> +
> +	/* Cancel any work that has been queued */
> +	cancel_delayed_work_sync(&hisi_ptt->work);
> +}
> +
> +/* Register the bus notifier for dynamically updating the filter list */
> +static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt *hisi_ptt)
> +{
> +	int ret;
> +
> +	hisi_ptt->hisi_ptt_nb.notifier_call = hisi_ptt_notifier_call;
> +	ret = bus_register_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);
> +	if (ret)
> +		return ret;
> +
> +	return devm_add_action_or_reset(&hisi_ptt->pdev->dev,
> +					hisi_ptt_unregister_filter_update_notifier,
> +					hisi_ptt);
> +}
> +
>  /*
>   * The DMA of PTT trace can only use direct mappings due to some
>   * hardware restriction. Check whether there is no IOMMU or the
> @@ -987,6 +1127,10 @@ static int hisi_ptt_probe(struct pci_dev *pdev,
>  		return ret;
>  	}
>  
> +	ret = hisi_ptt_register_filter_update_notifier(hisi_ptt);
> +	if (ret)
> +		pci_warn(pdev, "failed to register filter update notifier, ret = %d", ret);
> +
>  	ret = hisi_ptt_register_pmu(hisi_ptt);
>  	if (ret) {
>  		pci_err(pdev, "failed to register PMU device, ret = %d", ret);
> diff --git a/drivers/hwtracing/ptt/hisi_ptt.h b/drivers/hwtracing/ptt/hisi_ptt.h
> index 5beb1648c93a..a389b943944c 100644
> --- a/drivers/hwtracing/ptt/hisi_ptt.h
> +++ b/drivers/hwtracing/ptt/hisi_ptt.h
> @@ -11,12 +11,15 @@
>  
>  #include <linux/bits.h>
>  #include <linux/cpumask.h>
> +#include <linux/kfifo.h>
>  #include <linux/list.h>
>  #include <linux/mutex.h>
> +#include <linux/notifier.h>
>  #include <linux/pci.h>
>  #include <linux/perf_event.h>
>  #include <linux/spinlock.h>
>  #include <linux/types.h>
> +#include <linux/workqueue.h>
>  
>  #define DRV_NAME "hisi_ptt"
>  
> @@ -71,6 +74,11 @@
>  #define HISI_PTT_WAIT_TRACE_TIMEOUT_US	100UL
>  #define HISI_PTT_WAIT_POLL_INTERVAL_US	10UL
>  
> +/* FIFO size for dynamically updating the PTT trace filter list. */
> +#define HISI_PTT_FILTER_UPDATE_FIFO_SIZE	16
> +/* Delay time for filter updating work */
> +#define HISI_PTT_WORK_DELAY_MS			100UL
> +
>  #define HISI_PCIE_CORE_PORT_ID(devfn)	((PCI_SLOT(devfn) & 0x7) << 1)
>  
>  /* Definition of the PMU configs */
> @@ -143,6 +151,16 @@ struct hisi_ptt_filter_desc {
>  	u16 devid;
>  };
>  
> +/**
> + * struct hisi_ptt_filter_update_info - Information for PTT filter updating
> + * @pdev:       the PCI device to update in the filter list
> + * @is_add:     adding to the filter or not
> + */
> +struct hisi_ptt_filter_update_info {
> +	struct pci_dev *pdev;
> +	bool is_add;
> +};
> +
>  /**
>   * struct hisi_ptt_pmu_buf - Descriptor of the AUX buffer of PTT trace
>   * @length:   size of the AUX buffer
> @@ -161,6 +179,7 @@ struct hisi_ptt_pmu_buf {
>   * struct hisi_ptt - Per PTT device data
>   * @trace_ctrl:   the control information of PTT trace
>   * @hotplug_node: node for register cpu hotplug event
> + * @hisi_ptt_nb:  dynamic filter update notifier
>   * @hisi_ptt_pmu: the pum device of trace
>   * @iobase:       base IO address of the device
>   * @pdev:         pci_dev of this PTT device
> @@ -170,10 +189,15 @@ struct hisi_ptt_pmu_buf {
>   * @lower_bdf:    the lower BDF range of the PCI devices managed by this PTT device
>   * @port_filters: the filter list of root ports
>   * @req_filters:  the filter list of requester ID
> + * @filter_lock:  lock to protect the filters
>   * @port_mask:    port mask of the managed root ports
> + * @work:         delayed work for filter updating
> + * @filter_update_lock: spinlock to protect the filter update fifo
> + * @filter_update_fifo: fifo of the filters waiting to update the filter list
>   */
>  struct hisi_ptt {
>  	struct hisi_ptt_trace_ctrl trace_ctrl;
> +	struct notifier_block hisi_ptt_nb;
>  	struct hlist_node hotplug_node;
>  	struct pmu hisi_ptt_pmu;
>  	void __iomem *iobase;
> @@ -192,7 +216,19 @@ struct hisi_ptt {
>  	 */
>  	struct list_head port_filters;
>  	struct list_head req_filters;
> +	struct mutex filter_lock;
>  	u16 port_mask;
> +
> +	/*
> +	 * We use a delayed work here to avoid indefinitely waiting for
> +	 * the hisi_ptt->mutex which protecting the filter list. The
> +	 * work will be delayed only if the mutex can not be held,
> +	 * otherwise no delay will be applied.
> +	 */
> +	struct delayed_work work;
> +	spinlock_t filter_update_lock;
> +	DECLARE_KFIFO(filter_update_kfifo, struct hisi_ptt_filter_update_info,
> +		      HISI_PTT_FILTER_UPDATE_FIFO_SIZE);
>  };
>  
>  #define to_hisi_ptt(pmu) container_of(pmu, struct hisi_ptt, hisi_ptt_pmu)
>