Re: [PATCH v15 05/16] cxl: Introduce endpoint HDM decoder settings

[Dave Jiang]

On 10/10/26 7:14 PM, Srirangan Madhavan wrote:
> Keep common decoder configuration in cxl_decoder_config and represent
> endpoint DPA skip in cxl_endpoint_decoder_settings. Share range
> programming, but keep endpoint skip and switch target-list writes in
> their respective commit paths.
> 
> Separate commit initiation from completion waiting so reset restoration
> can reuse register programming without changing normal DPA-lock policy.
> 
> Link: https://git.kernel.org/pub/scm/linux/kernel/git/djiang/linux.git/commit/?id=20f9084b5b763238e2f18f629916bbbb46ec9f7d
> Suggested-by: Dave Jiang <dave.jiang@xxxxxxxxx>
> Signed-off-by: Srirangan Madhavan <smadhavan@xxxxxxxxxx>
> Assisted-by: LLM

Reviewed-by: Dave Jiang <dave.jiang@xxxxxxxxx>

> ---
>  drivers/cxl/core/core.h |  8 ++++
>  drivers/cxl/core/hdm.c  | 96 ++++++++++++++++++++++++++++-------------
>  include/cxl/hdm.h       | 19 ++++++++
>  3 files changed, 93 insertions(+), 30 deletions(-)
>  create mode 100644 include/cxl/hdm.h
> 
> diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
> index 983d7690c3a5..fb14b222f3ca 100644
> --- a/drivers/cxl/core/core.h
> +++ b/drivers/cxl/core/core.h
> @@ -7,6 +7,8 @@
>  #include <cxl/mailbox.h>
>  #include <linux/rwsem.h>
>  
> +struct cxl_endpoint_decoder_settings;
> +
>  extern const struct device_type cxl_nvdimm_bridge_type;
>  extern const struct device_type cxl_nvdimm_type;
>  extern const struct device_type cxl_pmu_type;
> @@ -139,6 +141,12 @@ struct cxl_rwsem {
>  
>  extern struct cxl_rwsem cxl_rwsem;
>  
> +void cxl_commit_start_endpoint(const struct cxl_endpoint_decoder_settings *settings,
> +			       void __iomem *hdm);
> +void cxl_commit_start_switch(const struct cxl_decoder_config *config,
> +			     u64 targets, void __iomem *hdm);
> +int cxld_await_commit(void __iomem *hdm, int id);
> +
>  int cxl_memdev_init(void);
>  void cxl_memdev_exit(void);
>  void cxl_mbox_init(void);
> diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
> index 9e05032a5426..7d3254a712e4 100644
> --- a/drivers/cxl/core/hdm.c
> +++ b/drivers/cxl/core/hdm.c
> @@ -3,6 +3,7 @@
>  #include <linux/seq_file.h>
>  #include <linux/device.h>
>  #include <linux/delay.h>
> +#include <cxl/hdm.h>
>  
>  #include "cxlmem.h"
>  #include "core.h"
> @@ -707,7 +708,7 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size)
>  	return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
>  }
>  
> -static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
> +static void cxld_set_interleave(const struct cxl_decoder_config *config, u32 *ctrl)
>  {
>  	u16 eig;
>  	u8 eiw;
> @@ -716,12 +717,12 @@ static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
>  	 * Input validation ensures these warns never fire, but otherwise
>  	 * suppress unititalized variable usage warnings.
>  	 */
> -	if (WARN_ONCE(ways_to_eiw(cxld->config.interleave_ways, &eiw),
> -		      "invalid interleave_ways: %d\n", cxld->config.interleave_ways))
> +	if (WARN_ONCE(ways_to_eiw(config->interleave_ways, &eiw),
> +		      "invalid interleave_ways: %d\n", config->interleave_ways))
>  		return;
> -	if (WARN_ONCE(granularity_to_eig(cxld->config.interleave_granularity, &eig),
> +	if (WARN_ONCE(granularity_to_eig(config->interleave_granularity, &eig),
>  		      "invalid interleave_granularity: %d\n",
> -		      cxld->config.interleave_granularity))
> +		      config->interleave_granularity))
>  		return;
>  
>  	u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
> @@ -729,10 +730,10 @@ static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
>  	*ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
>  }
>  
> -static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
> +static void cxld_set_type(const struct cxl_decoder_config *config, u32 *ctrl)
>  {
>  	u32p_replace_bits(ctrl,
> -			  !!(cxld->config.target_type == CXL_DECODER_HOSTONLYMEM),
> +			  !!(config->target_type == CXL_DECODER_HOSTONLYMEM),
>  			  CXL_HDM_DECODER0_CTRL_HOSTONLY);
>  }
>  
> @@ -764,7 +765,7 @@ static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
>   * clock skew and other marginal behavior
>   */
>  #define COMMIT_TIMEOUT_MS 20
> -static int cxld_await_commit(void __iomem *hdm, int id)
> +int cxld_await_commit(void __iomem *hdm, int id)
>  {
>  	u32 ctrl;
>  	int i;
> @@ -784,45 +785,80 @@ static int cxld_await_commit(void __iomem *hdm, int id)
>  	return -ETIMEDOUT;
>  }
>  
> -static void setup_hw_decoder(struct cxl_decoder *cxld, void __iomem *hdm)
> +static u32 cxl_hdm_write_range(void __iomem *hdm,
> +			       const struct cxl_decoder_config *config)
>  {
> -	int id = cxld->config.id;
> -	u64 base, size;
> +	u64 size = range_len(&config->hpa_range);
> +	u64 base = config->hpa_range.start;
> +	int id = config->id;
>  	u32 ctrl;
>  
> -	/* common decoder settings */
> -	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->config.id));
> -	cxld_set_interleave(cxld, &ctrl);
> -	cxld_set_type(cxld, &ctrl);
> -	base = cxld->config.hpa_range.start;
> -	size = range_len(&cxld->config.hpa_range);
> +	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
> +	cxld_set_interleave(config, &ctrl);
> +	cxld_set_type(config, &ctrl);
>  
>  	writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
>  	writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
>  	writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
>  	writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
> +	return ctrl;
> +}
> +
> +void cxl_commit_start_endpoint(const struct cxl_endpoint_decoder_settings *settings,
> +			       void __iomem *hdm)
> +{
> +	int id = settings->config.id;
> +	u32 ctrl;
> +
> +	lockdep_assert_held(&cxl_rwsem.dpa);
> +	ctrl = cxl_hdm_write_range(hdm, &settings->config);
> +	writel(upper_32_bits(settings->skip), hdm + CXL_HDM_DECODER0_SKIP_HIGH(id));
> +	writel(lower_32_bits(settings->skip), hdm + CXL_HDM_DECODER0_SKIP_LOW(id));
> +	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
> +}
> +
> +void cxl_commit_start_switch(const struct cxl_decoder_config *config,
> +			     u64 targets, void __iomem *hdm)
> +{
> +	int id = config->id;
> +	u32 ctrl;
> +
> +	lockdep_assert_held(&cxl_rwsem.dpa);
> +	ctrl = cxl_hdm_write_range(hdm, config);
> +	writel(upper_32_bits(targets), hdm + CXL_HDM_DECODER0_TL_HIGH(id));
> +	writel(lower_32_bits(targets), hdm + CXL_HDM_DECODER0_TL_LOW(id));
> +	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
> +}
> +
> +static void
> +cxl_endpoint_decoder_snapshot(struct cxl_endpoint_decoder *cxled,
> +			      struct cxl_endpoint_decoder_settings *settings)
> +{
> +	lockdep_assert_held(&cxl_rwsem.dpa);
> +
> +	*settings = (struct cxl_endpoint_decoder_settings) {
> +		.config = cxled->cxld.config,
> +		.skip = cxled->skip,
> +	};
> +}
> +
> +static void setup_hw_decoder(struct cxl_decoder *cxld, void __iomem *hdm)
> +{
> +	struct cxl_endpoint_decoder_settings settings;
>  
>  	if (is_switch_decoder(&cxld->dev)) {
>  		struct cxl_switch_decoder *cxlsd =
>  			to_cxl_switch_decoder(&cxld->dev);
> -		void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
> -		void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
>  		u64 targets;
>  
>  		cxlsd_set_targets(cxlsd, &targets);
> -		writel(upper_32_bits(targets), tl_hi);
> -		writel(lower_32_bits(targets), tl_lo);
> -	} else {
> -		struct cxl_endpoint_decoder *cxled =
> -			to_cxl_endpoint_decoder(&cxld->dev);
> -		void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
> -		void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
> -
> -		writel(upper_32_bits(cxled->skip), sk_hi);
> -		writel(lower_32_bits(cxled->skip), sk_lo);
> +		cxl_commit_start_switch(&cxld->config, targets, hdm);
> +		return;
>  	}
>  
> -	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
> +	cxl_endpoint_decoder_snapshot(to_cxl_endpoint_decoder(&cxld->dev),
> +				      &settings);
> +	cxl_commit_start_endpoint(&settings, hdm);
>  }
>  
>  static int cxl_decoder_commit(struct cxl_decoder *cxld)
> diff --git a/include/cxl/hdm.h b/include/cxl/hdm.h
> new file mode 100644
> index 000000000000..219b9bb67440
> --- /dev/null
> +++ b/include/cxl/hdm.h
> @@ -0,0 +1,19 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/* Copyright (c) 2026 NVIDIA Corporation & Affiliates */
> +#ifndef __CXL_HDM_H__
> +#define __CXL_HDM_H__
> +
> +#include <linux/types.h>
> +#include <cxl/cxl.h>
> +
> +/**
> + * struct cxl_endpoint_decoder_settings - endpoint HDM decoder programming
> + * @config: common decoder configuration
> + * @skip: length of DPA space skipped before this decoder's DPA mapping
> + */
> +struct cxl_endpoint_decoder_settings {
> +	struct cxl_decoder_config config;
> +	u64 skip;
> +};
> +
> +#endif /* __CXL_HDM_H__ */