Re: [PATCH v17 11/19] PCI/CXL: Cache endpoint HDM state during PCI enumeration

From: Li Ming

Date: Tue Nov 10 2026 - 22:55:14 EST



在 2026/11/10 22:12, Li Ming 写道:

在 2026/11/6 02:59, Srirangan Madhavan 写道:
Cache HDM state in pci_dev->hdm after BAR resource setup and before driver
binding, so reset can restore it without a bound CXL driver.

Limit caching to devices that advertise CXL.cache, CXL.mem, CXL Reset,
and Reset Memory Clear. Save their HDM register location, decoder
settings, and Device DVSEC state.

Validate capability reads and decoder counts, and restore PCI_COMMAND
after temporary MMIO access, reporting restoration failures. Publish the
cache under the PCI device lock and protect later updates with the DPA
lock.

Signed-off-by: Srirangan Madhavan <smadhavan@xxxxxxxxxx>
Assisted-by: LLM

---
Changes in v17:
- Publish the cache under the PCI device lock without taking
   cxl_rwsem.dpa, and drop the cxl_hdm_cache_present() and
   cxl_hdm_cache_publish() helpers (Li Ming)
https://lore.kernel.org/linux-cxl/d581f2a0-dd9e-4d0a-bd19-639c1bd9b832@xxxxxxxxxxxx/
- Reject an all-ones DVSEC Control read in cxl_set_mem_enable() before
   it is written back or cached (Sashiko)
https://sashiko.dev/#/patchset/20261029145703.2185231-1-smadhavan@xxxxxxxxxx?part=11
- Explain why cxl_set_mem_enable() refreshes the cached DVSEC Control
   when MEM_ENABLE already matches (Richard)
https://lore.kernel.org/linux-cxl/auRQN6a3QVtqjd5a@MWDK4CY14F/

Reference branch: https://git.kernel.org/pub/scm/linux/kernel/git/djiang/linux.git/commit/?id=b12baa1105f33cc1a10f9c9c0c6376194397d9f3
---
  MAINTAINERS                   |   1 +
  drivers/cxl/core/hdm_regs.c   |  96 +++++++++++++++
  drivers/cxl/core/hdm_state.c  |  37 +++++-
  drivers/cxl/core/pci.c        |  29 +++--
  drivers/pci/Makefile          |   1 +
  drivers/pci/bus.c             |   1 +
  drivers/pci/cxl.c             | 225 ++++++++++++++++++++++++++++++++++
  drivers/pci/pci.h             |   8 ++
  drivers/pci/probe.c           |   1 +
  include/cxl/hdm.h             |   9 ++
  include/uapi/linux/pci_regs.h |   2 +
  11 files changed, 399 insertions(+), 11 deletions(-)
  create mode 100644 drivers/pci/cxl.c

diff --git a/MAINTAINERS b/MAINTAINERS
index c2414447892c..b874340b0265 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6572,6 +6572,7 @@ S:    Maintained
  F:    Documentation/driver-api/cxl
  F:    Documentation/userspace-api/fwctl/fwctl-cxl.rst
  F:    drivers/cxl/
+F:    drivers/pci/cxl.c
  F:    include/cxl/
  F:    include/uapi/linux/cxl_mem.h
  F:    tools/testing/cxl/
diff --git a/drivers/cxl/core/hdm_regs.c b/drivers/cxl/core/hdm_regs.c
index cc5d77c4abf5..c92dbb017162 100644
--- a/drivers/cxl/core/hdm_regs.c
+++ b/drivers/cxl/core/hdm_regs.c
@@ -2,6 +2,7 @@
  /* Copyright (c) 2026 NVIDIA Corporation & Affiliates */
  #include <linux/delay.h>
  #include <linux/bug.h>
+#include <linux/device.h>
  #include <linux/errno.h>
  #include <linux/export.h>
  #include <linux/kernel.h>
@@ -190,3 +191,98 @@ int cxl_hdm_read_decoder(struct cxl_decoder_config *config, void __iomem *hdm,
      return cxl_hdm_unpack_decoder(config, id, ctrl, base, size);
  }
  EXPORT_SYMBOL_FOR_MODULES(cxl_hdm_read_decoder, "cxl_core");
+
+/**
+ * cxl_hdm_decoder_count_mapped() - read the decoder count of a mapped block
+ * @dev: device for error reporting
+ * @hdm: mapped HDM decoder capability
+ * @size: size of the mapping
+ *
+ * Return: the decoder count, or a negative errno if the count is reserved or
+ * the mapping cannot hold every decoder.
+ */
+int cxl_hdm_decoder_count_mapped(struct device *dev, void __iomem *hdm,
+                 resource_size_t size)
+{
+    int count;
+
+    count = cxl_hdm_decoder_count(readl(hdm + CXL_HDM_DECODER_CAP_OFFSET));
+    if (count < 0)
+        return count;
+
+    /* The last decoder's target-list high register must fit in the mapping. */
+    if (CXL_HDM_DECODER0_TL_HIGH(count - 1) + sizeof(u32) > size) {
+        dev_err(dev, "CXL HDM decoder count exceeds mapped register block\n");
+        return -ENXIO;
+    }
+
+    return count;
+}
+
+/* Read one endpoint decoder; a disabled decoder keeps only its id. */
+static int cxl_hdm_read_decoder_settings(struct device *dev,
+                     struct cxl_endpoint_decoder_settings *settings,
+                     void __iomem *hdm, int id)
+{
+    u64 lo, hi;
+    u32 ctrl;
+    int rc;
+
+    ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
+    rc = cxl_hdm_read_decoder(&settings->config, hdm, id, ctrl);
+    if (rc) {
+        dev_err(dev,
+            "CXL HDM decoder %d has invalid configuration: %d\n",
+            id, rc);
+        return rc;
+    }
+
+    if (!(settings->config.flags & CXL_DECODER_F_ENABLE)) {
+        *settings = (struct cxl_endpoint_decoder_settings) {
+            .config.id = id,
+        };
+    } else {
+        lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(id));
+        hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(id));
+        settings->skip = (hi << 32) | lo;
+    }
+
+    return 0;
+}
+
+/**
+ * cxl_hdm_read_state() - read endpoint HDM decoder state
+ * @dev: device for error messages
+ * @hdm: mapped HDM decoder capability
+ * @info: state to fill; @info->decoder_count must be set
+ *
+ * Return: 0 on success, or a negative errno if a decoder holds an invalid
+ * configuration or the decoder count changed while reading.
+ */
+int cxl_hdm_read_state(struct device *dev, void __iomem *hdm,
+               struct cxl_hdm_info *info)
+{
+    int count;
+    int rc;
+
+    info->global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
+    for (int i = 0; i < info->decoder_count; i++) {
+        rc = cxl_hdm_read_decoder_settings(dev, &info->settings[i],
+                           hdm, i);
+        if (rc)
+            return rc;
+    }
+
+    /*
+     * A changed decoder count means the snapshot no longer describes the
+     * complete hardware state. Reject it rather than publish a partial cache.
+     */
+    count = cxl_hdm_decoder_count(readl(hdm + CXL_HDM_DECODER_CAP_OFFSET));
+    if (count != info->decoder_count) {
+        dev_err(dev, "CXL HDM decoder count changed from %d to %d\n",
+            info->decoder_count, count);
+        return -ENXIO;
+    }
+
+    return 0;
+}
diff --git a/drivers/cxl/core/hdm_state.c b/drivers/cxl/core/hdm_state.c
index d4c2be1ef833..9def4a7439c9 100644
--- a/drivers/cxl/core/hdm_state.c
+++ b/drivers/cxl/core/hdm_state.c
@@ -1,21 +1,37 @@
  // SPDX-License-Identifier: GPL-2.0-only
  /* Copyright (c) 2026 NVIDIA Corporation & Affiliates */
+#include <linux/cleanup.h>
  #include <linux/export.h>
  #include <linux/io.h>
  #include <linux/pci.h>
  #include <linux/rwsem.h>
+#include <linux/slab.h>
    #include <cxl/hdm.h>
-
  #include "cxl.h"
  #include "core.h"
  +/*
+ * CXL state that must stay available while no CXL driver is bound: the CXL
+ * locks and the per-device HDM cache filled during PCI enumeration.
+ */
  struct cxl_rwsem cxl_rwsem = {
      .region = __RWSEM_INITIALIZER(cxl_rwsem.region),
      .dpa = __RWSEM_INITIALIZER(cxl_rwsem.dpa),
  };
  EXPORT_SYMBOL_FOR_MODULES(cxl_rwsem, "cxl_core");
  +void cxl_hdm_cache_set_dvsec_ctrl(struct pci_dev *pdev, u16 ctrl)
+{
+    guard(rwsem_write)(&cxl_rwsem.dpa);
Is this lock needed? “pdev->hdm->dvsec_ctrl” could be modified concurrently? Seems like it is called during the endpoint port probing.

I realize that we may need this lock while I was reviewing patch #13. The CXL reset operation could be triggered during the endpoint port probing.

Would you mind moving the code of holding cxl_rwsem.dpa to patch #13? I think this could make it easier to review.


[snip]