[PATCH RFC 07/10] PM / devfreq: Introduce the QCOM SCMI Memlat devfreq driver

[Pragnesh Papaniya]

From: Sibi Sankar <sibi.sankar@xxxxxxxxxxxxxxxx>

On Qualcomm Glymur, Mahua, X1E/X1P (Hamoa/Purwa) and Kaanapali SoCs, the
memory-latency (memlat) governor runs entirely on the CPU Control
Processor (CPUCP). Every sampling window the CPUCP reads the per-CPU PMU
counters, derives its statistics (instructions-per-miss, back-end stall),
maps them to a target operating point for the DDR, LLCC and DDR_QOS buses
and votes it directly to the interconnect. The kernel is never in that
runtime decision loop; the CPUCP algorithm and its controls are reached
through the QCOM SCMI Generic Extension Protocol, addressed via the
"MEMLAT" algorithm string.

Introduce a devfreq SCMI client driver whose role is split in two. At probe
it configures the CPUCP algorithm once (event maps, per-monitor tuneables,
the cpufreq-to-memfreq maps and min/max clamps) and starts it; this half is
required, as without it the CPUCP has no per-SoC data to run the algorithm.
Thereafter the driver only reads back the current operating point.

Each bus is modelled as a devfreq device and uses the remote devfreq
governor precisely for this "observe a frequency someone else drives" case:
it exposes the read-back through trans_stat and lets userspace retune the
remote governor's parameters.

Co-developed-by: Amir Vajid <amir.vajid@xxxxxxxxxxxxxxxx>
Signed-off-by: Amir Vajid <amir.vajid@xxxxxxxxxxxxxxxx>
Co-developed-by: Ramakrishna Gottimukkula <ramakrishna.gottimukkula@xxxxxxxxxxxxxxxx>
Signed-off-by: Ramakrishna Gottimukkula <ramakrishna.gottimukkula@xxxxxxxxxxxxxxxx>
Signed-off-by: Sibi Sankar <sibi.sankar@xxxxxxxxxxxxxxxx>
Co-developed-by: Pragnesh Papaniya <pragnesh.papaniya@xxxxxxxxxxxxxxxx>
Signed-off-by: Pragnesh Papaniya <pragnesh.papaniya@xxxxxxxxxxxxxxxx>
---
 MAINTAINERS                                |    8 +
 drivers/devfreq/Kconfig                    |   15 +
 drivers/devfreq/Makefile                   |    1 +
 drivers/devfreq/scmi-qcom-memlat-devfreq.c | 1331 ++++++++++++++++++++++++++++
 4 files changed, 1355 insertions(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 265eb99bdc7a..a58e2219d5df 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22495,6 +22495,14 @@ F:	Documentation/networking/device_drivers/cellular/qualcomm/rmnet.rst
 F:	drivers/net/ethernet/qualcomm/rmnet/
 F:	include/linux/if_rmnet.h
 
+QUALCOMM SCMI MEMLAT DEVFREQ DRIVER
+M:	Pragnesh Papaniya <pragnesh.papaniya@xxxxxxxxxxxxxxxx>
+M:	Sibi Sankar <sibi.sankar@xxxxxxxxxxxxxxxx>
+L:	linux-arm-msm@xxxxxxxxxxxxxxx
+L:	linux-pm@xxxxxxxxxxxxxxx
+S:	Maintained
+F:	drivers/devfreq/scmi-qcom-memlat-devfreq.c
+
 QUALCOMM TEE (QCOMTEE) DRIVER
 M:	Amirreza Zarrabi <amirreza.zarrabi@xxxxxxxxxxxxxxxx>
 L:	linux-arm-msm@xxxxxxxxxxxxxxx
diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig
index 2caa87554914..fdf6484fc72c 100644
--- a/drivers/devfreq/Kconfig
+++ b/drivers/devfreq/Kconfig
@@ -169,6 +169,21 @@ config ARM_SUN8I_A33_MBUS_DEVFREQ
 	  This adds the DEVFREQ driver for the MBUS controller in some
 	  Allwinner sun8i (A33 through H3) and sun50i (A64 and H5) SoCs.
 
+config SCMI_QCOM_MEMLAT_DEVFREQ
+	tristate "Qualcomm SCMI memory latency (memlat) devfreq driver"
+	depends on QCOM_SCMI_GENERIC_EXT || COMPILE_TEST
+	select DEVFREQ_GOV_REMOTE
+	help
+	  Enable memory-bus (DDR/LLCC/DDR_QOS) scaling on Qualcomm Glymur,
+	  X1E/X1P-class and Kaanapali SoCs. The scaling algorithm itself runs
+	  autonomously on the CPU Control Processor (CPUCP); this driver only
+	  configures and starts it at boot and then exposes each bus as a
+	  devfreq device so its operating point is visible via trans_stat and
+	  tunable from userspace.
+
+	  Nothing else needs to be enabled for the scaling to take effect. Say Y
+	  or M here on the affected Qualcomm SoCs; otherwise say N.
+
 source "drivers/devfreq/event/Kconfig"
 
 endif # PM_DEVFREQ
diff --git a/drivers/devfreq/Makefile b/drivers/devfreq/Makefile
index cde57c8cda76..b11f94e2f485 100644
--- a/drivers/devfreq/Makefile
+++ b/drivers/devfreq/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_ARM_MEDIATEK_CCI_DEVFREQ)	+= mtk-cci-devfreq.o
 obj-$(CONFIG_ARM_RK3399_DMC_DEVFREQ)	+= rk3399_dmc.o
 obj-$(CONFIG_ARM_SUN8I_A33_MBUS_DEVFREQ)	+= sun8i-a33-mbus.o
 obj-$(CONFIG_ARM_TEGRA_DEVFREQ)		+= tegra30-devfreq.o
+obj-$(CONFIG_SCMI_QCOM_MEMLAT_DEVFREQ)	+= scmi-qcom-memlat-devfreq.o
 
 # DEVFREQ Event Drivers
 obj-$(CONFIG_PM_DEVFREQ_EVENT)		+= event/
diff --git a/drivers/devfreq/scmi-qcom-memlat-devfreq.c b/drivers/devfreq/scmi-qcom-memlat-devfreq.c
new file mode 100644
index 000000000000..9ca85b2f4c1a
--- /dev/null
+++ b/drivers/devfreq/scmi-qcom-memlat-devfreq.c
@@ -0,0 +1,1331 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/cpu.h>
+#include <linux/devfreq.h>
+#include <linux/device/faux.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pm_opp.h>
+#include <linux/scmi_protocol.h>
+#include <linux/scmi_qcom_protocol.h>
+#include <linux/units.h>
+
+/* CPUCP AMU event indices programmed into the common/group event maps. */
+#define MEMLAT_EV_CPU_CYCLES		0
+#define MEMLAT_EV_CNT_CYCLES		1
+#define MEMLAT_EV_INST_RETIRED		2
+#define MEMLAT_EV_STALL_BACKEND_MEM	3
+#define MEMLAT_EV_L2_D_RFILL		5
+
+/* Sentinel for an event slot that is not wired to any AMU counter. */
+#define MEMLAT_INVALID_IDX		0xff
+
+/* hw_type value that targets every group (common events are not group-scoped). */
+#define MEMLAT_HW_TYPE_ALL		0xff
+
+#define MEMLAT_ALGO_STR			0x4d454d4c4154ULL /* "MEMLAT" */
+
+/**
+ * struct scmi_qcom_map_table - one row of a monitor's cpufreq->memfreq map
+ * @cpu_freq_mhz: CPU frequency threshold, in MHz. When the monitor's busiest
+ *                CPU is at or above this frequency, the firmware votes the
+ *                paired memory frequency.
+ * @mem_freq_mhz: memory frequency to vote, in MHz (a 0/1 level for DDR_QOS).
+ */
+struct scmi_qcom_map_table {
+	unsigned int cpu_freq_mhz;
+	unsigned int mem_freq_mhz;
+};
+
+struct scmi_qcom_opp_data {
+	u64 freq;
+};
+
+struct scmi_qcom_memory_range {
+	unsigned int min_freq_khz;
+	unsigned int max_freq_khz;
+};
+
+enum common_ev_idx {
+	INST_IDX,
+	CYC_IDX,
+	CONST_CYC_IDX,
+	FE_STALL_IDX,
+	BE_STALL_IDX,
+	NUM_COMMON_EVS
+};
+
+enum grp_ev_idx {
+	MISS_IDX,
+	WB_IDX,
+	ACC_IDX,
+	NUM_GRP_EVS
+};
+
+/*
+ * hw_type identifiers for the memory buses. These are a firmware-defined
+ * encoding carried in the node/scalar/map/ev messages; the driver only
+ * mirrors them here.
+ */
+enum scmi_qcom_memlat_hw_type {
+	MEMLAT_HW_DDR			= 0,
+	MEMLAT_HW_LLCC			= 1,
+	MEMLAT_HW_DDR_QOS_COMPUTE	= 2,
+	MEMLAT_HW_DDR_QOS_MOBILE	= 3,
+};
+
+/**
+ * struct scmi_qcom_monitor_cfg - per-monitor firmware tuneables
+ * @table: cpufreq->memfreq voting map for this monitor.
+ * @name: monitor name, forwarded to the firmware and used in its log lines.
+ * @be_stall_floor: back-end-stall gating threshold, in percent. Each sample
+ *                  window the firmware computes a per-CPU back-end-stall
+ *                  percentage (stall cycles / total cycles); a CPU only
+ *                  contributes its frequency vote to this monitor when that
+ *                  percentage is at or above @be_stall_floor. A value of 1
+ *                  therefore means "any stall counts", i.e. the CPU is
+ *                  effectively always eligible.
+ * @cpu_mask: bitmask of CPUs whose counters this monitor aggregates.
+ * @ipm_ceil: instructions-per-miss ceiling. A CPU is treated as
+ *            memory-latency-bound (and eligible to vote) only when its IPM is
+ *            at or below this ceiling; a monitor with @ipm_ceil == 0 is a
+ *            "compute" monitor and scales passively with cpufreq.
+ * @table_len: number of valid rows in @table.
+ */
+struct scmi_qcom_monitor_cfg {
+	const struct scmi_qcom_map_table *table;
+	const char *name;
+	u32 be_stall_floor;
+	u32 cpu_mask;
+	u32 ipm_ceil;
+	u32 table_len;
+};
+
+struct scmi_qcom_memory_cfg {
+	const struct scmi_qcom_monitor_cfg *monitor_cfg;
+	const struct scmi_qcom_opp_data *mem_table;
+	struct scmi_qcom_memory_range memory_range;
+	const u32 *grp_ev;
+	const char *name;
+	enum scmi_qcom_memlat_hw_type memory_type;
+	u32 monitor_cnt;
+	u32 num_opps;
+};
+
+struct scmi_qcom_memlat_cfg_data {
+	const struct scmi_qcom_memory_cfg *memory_cfg;
+	const u32 *common_ev;
+	bool cpucp_legacy_freq_method;
+	u32 cpucp_sample_ms;
+	u32 memory_cnt;
+};
+
+#define MEMLAT_MAX_NAME_LEN			20
+#define MEMLAT_MAX_MAP_ENTRIES		10
+
+/*
+ * param_ids of the "MEMLAT" algo_str hosted by the Qualcomm Generic Vendor
+ * Protocol. MEMLAT detects memory-latency-bound workloads and scales the
+ * memory buses accordingly; these are the fixed wire-protocol tokens that
+ * the firmware matches.
+ *
+ * MEMLAT_SET_MEM_GROUP: initializes the frequency/level scaling functions for the memory bus.
+ * MEMLAT_SET_MONITOR: configures the monitor to work on a specific memory bus.
+ * MEMLAT_SET_COMMON_EV_MAP: set up common counters used to monitor the cpu frequency.
+ * MEMLAT_SET_GRP_EV_MAP: set up any specific counters used to monitor the memory bus.
+ * MEMLAT_ADAPTIVE_LOW_FREQ: set the adaptive low frequency floor of the memory bus.
+ * MEMLAT_ADAPTIVE_HIGH_FREQ: set the adaptive high frequency floor of the memory bus.
+ * MEMLAT_GET_ADAPTIVE_CUR_FREQ: query the current adaptive frequency of the memory bus.
+ * MEMLAT_IPM_CEIL: set the IPM (Instruction Per Misses) ceiling per monitor.
+ * MEMLAT_FE_STALL_FLOOR: set the front-end stall floor per monitor.
+ * MEMLAT_BE_STALL_FLOOR: set the back-end stall floor per monitor.
+ * MEMLAT_WB_PCT: set the write-back percentage threshold per monitor.
+ * MEMLAT_IPM_FILTER: set the IPM filter used to reject non-latency samples per monitor.
+ * MEMLAT_FREQ_SCALE_PCT: set the frequency scaling percentage per monitor.
+ * MEMLAT_FREQ_SCALE_CEIL_MHZ: set the cpu frequency ceiling for scaling per monitor.
+ * MEMLAT_FREQ_SCALE_FLOOR_MHZ: set the cpu frequency floor for scaling per monitor.
+ * MEMLAT_SAMPLE_MS: set the sampling period for all the monitors.
+ * MEMLAT_MON_FREQ_MAP: setup the cpufreq to memfreq map.
+ * MEMLAT_SET_MIN_FREQ: set the min frequency of the memory bus.
+ * MEMLAT_SET_MAX_FREQ: set the max frequency of the memory bus.
+ * MEMLAT_GET_CUR_FREQ: query the current frequency/level of the memory bus.
+ * MEMLAT_START_TIMER: start all the monitors with the requested sampling period.
+ * MEMLAT_STOP_TIMER: stop all the running monitors.
+ * MEMLAT_GET_TIMESTAMP: query the firmware timestamp.
+ * MEMLAT_SET_EFFECTIVE_FREQ_METHOD: set the method used to determine cpu frequency.
+ */
+#define MEMLAT_SET_MEM_GROUP			16
+#define MEMLAT_SET_MONITOR				17
+#define MEMLAT_SET_COMMON_EV_MAP		18
+#define MEMLAT_SET_GRP_EV_MAP			19
+#define MEMLAT_ADAPTIVE_LOW_FREQ		20
+#define MEMLAT_ADAPTIVE_HIGH_FREQ		21
+#define MEMLAT_GET_ADAPTIVE_CUR_FREQ	22
+#define MEMLAT_IPM_CEIL					23
+#define MEMLAT_FE_STALL_FLOOR			24
+#define MEMLAT_BE_STALL_FLOOR			25
+#define MEMLAT_WB_PCT					26
+#define MEMLAT_IPM_FILTER				27
+#define MEMLAT_FREQ_SCALE_PCT			28
+#define MEMLAT_FREQ_SCALE_CEIL_MHZ		29
+#define MEMLAT_FREQ_SCALE_FLOOR_MHZ		30
+#define MEMLAT_SAMPLE_MS				31
+#define MEMLAT_MON_FREQ_MAP				32
+#define MEMLAT_SET_MIN_FREQ				33
+#define MEMLAT_SET_MAX_FREQ				34
+#define MEMLAT_GET_CUR_FREQ				35
+#define MEMLAT_START_TIMER				36
+#define MEMLAT_STOP_TIMER				37
+#define MEMLAT_GET_TIMESTAMP			38
+#define MEMLAT_SET_EFFECTIVE_FREQ_METHOD 39
+
+struct cpucp_map_table {
+	__le16 cpu_freq_mhz;
+	__le16 mem_freq_mhz;
+};
+
+struct map_param_msg {
+	__le32 hw_type;
+	__le32 mon_idx;
+	__le32 nr_rows;
+	struct cpucp_map_table tbl[MEMLAT_MAX_MAP_ENTRIES];
+};
+
+struct node_msg {
+	__le32 cpumask;
+	__le32 hw_type;
+	__le32 mon_type;
+	__le32 mon_idx;
+	char mon_name[MEMLAT_MAX_NAME_LEN];
+};
+
+struct scalar_param_msg {
+	__le32 hw_type;
+	__le32 mon_idx;
+	__le32 val;
+};
+
+struct ev_map_msg {
+	__le32 num_evs;
+	__le32 hw_type;
+	__le32 cid[NUM_COMMON_EVS];
+};
+
+/*
+ * MEMLAT_GET_CUR_FREQ reuses the request buffer for its response, so express
+ * the in/out aliasing as a union rather than reinterpreting the bytes.
+ */
+union cur_freq_msg {
+	struct scalar_param_msg req;
+	__le32 resp;
+};
+
+struct scmi_qcom_memory_info {
+	const struct scmi_qcom_memory_cfg *cfg;
+	struct devfreq_dev_profile profile;
+	struct devfreq *devfreq;
+	struct faux_device *fdev;
+	struct scmi_protocol_handle *ph;
+	const struct qcom_generic_ext_ops *ops;
+};
+
+struct scmi_qcom_memlat_info {
+	struct scmi_protocol_handle *ph;
+	const struct qcom_generic_ext_ops *ops;
+	const struct scmi_qcom_memlat_cfg_data *cfg_data;
+	struct scmi_qcom_memory_info **memory;
+	u32 memory_cnt;
+};
+
+/*
+ * ==========================================================================
+ * Per-SoC configuration
+ * ==========================================================================
+ */
+
+static const u32 glymur_common_ev[NUM_COMMON_EVS] = {
+	[INST_IDX]      = MEMLAT_EV_INST_RETIRED,
+	[CYC_IDX]       = MEMLAT_EV_CPU_CYCLES,
+	[CONST_CYC_IDX] = MEMLAT_EV_CNT_CYCLES,
+	[FE_STALL_IDX]  = MEMLAT_INVALID_IDX,
+	[BE_STALL_IDX]  = MEMLAT_EV_STALL_BACKEND_MEM,
+};
+
+static const u32 glymur_ddr_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 glymur_llcc_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 glymur_ddr_qos_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 hamoa_common_ev[NUM_COMMON_EVS] = {
+	[INST_IDX]      = MEMLAT_EV_INST_RETIRED,
+	[CYC_IDX]       = MEMLAT_EV_CPU_CYCLES,
+	[CONST_CYC_IDX] = MEMLAT_EV_CNT_CYCLES,
+	[FE_STALL_IDX]  = MEMLAT_INVALID_IDX,
+	[BE_STALL_IDX]  = MEMLAT_EV_STALL_BACKEND_MEM,
+};
+
+static const u32 hamoa_ddr_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 hamoa_llcc_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 hamoa_ddr_qos_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 kaanapali_common_ev[NUM_COMMON_EVS] = {
+	[INST_IDX]      = MEMLAT_EV_INST_RETIRED,
+	[CYC_IDX]       = MEMLAT_EV_CPU_CYCLES,
+	[CONST_CYC_IDX] = MEMLAT_INVALID_IDX,
+	[FE_STALL_IDX]  = MEMLAT_INVALID_IDX,
+	[BE_STALL_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 kaanapali_ddr_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 kaanapali_llcc_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const u32 kaanapali_ddr_qos_grp_ev[NUM_GRP_EVS] = {
+	[MISS_IDX] = MEMLAT_EV_L2_D_RFILL,
+	[WB_IDX]   = MEMLAT_INVALID_IDX,
+	[ACC_IDX]  = MEMLAT_INVALID_IDX,
+};
+
+static const struct scmi_qcom_opp_data glymur_llcc_table[] = {
+	{ .freq = 315000000 },
+	{ .freq = 479000000 },
+	{ .freq = 545000000 },
+	{ .freq = 725000000 },
+	{ .freq = 840000000 },
+	{ .freq = 959000000 },
+	{ .freq = 1090000000 },
+	{ .freq = 1211000000 },
+};
+
+static const struct scmi_qcom_opp_data hamoa_llcc_table[] = {
+	{ .freq = 300000000 },
+	{ .freq = 466000000 },
+	{ .freq = 600000000 },
+	{ .freq = 806000000 },
+	{ .freq = 933000000 },
+	{ .freq = 1066000000 },
+};
+
+static const struct scmi_qcom_opp_data kaanapali_llcc_table[] = {
+	{ .freq = 282000000 },
+	{ .freq = 350000000 },
+	{ .freq = 533000000 },
+	{ .freq = 605600000 },
+	{ .freq = 806000000 },
+	{ .freq = 933000000 },
+	{ .freq = 1066000000 },
+	{ .freq = 1211000000 },
+	{ .freq = 1350000000 },
+};
+
+static const struct scmi_qcom_opp_data glymur_ddr_table[] = {
+	{ .freq = 200000000 },
+	{ .freq = 547000000 },
+	{ .freq = 1353000000 },
+	{ .freq = 1555000000 },
+	{ .freq = 1708000000 },
+	{ .freq = 2092000000 },
+	{ .freq = 2736000000 },
+	{ .freq = 3187000000 },
+	{ .freq = 3686000000 },
+	{ .freq = 4224000000 },
+	{ .freq = 4761000000 },
+};
+
+static const struct scmi_qcom_opp_data hamoa_ddr_table[] = {
+	{ .freq = 200000000 },
+	{ .freq = 547000000 },
+	{ .freq = 768000000 },
+	{ .freq = 1555000000 },
+	{ .freq = 1708000000 },
+	{ .freq = 2092000000 },
+	{ .freq = 2736000000 },
+	{ .freq = 3187000000 },
+	{ .freq = 3686000000 },
+	{ .freq = 4224000000 },
+};
+
+static const struct scmi_qcom_opp_data kaanapali_ddr_table[] = {
+	{ .freq = 547000000 },
+	{ .freq = 1353000000 },
+	{ .freq = 1555000000 },
+	{ .freq = 1708000000 },
+	{ .freq = 2092000000 },
+	{ .freq = 2736000000 },
+	{ .freq = 3187000000 },
+	{ .freq = 3686000000 },
+	{ .freq = 4224000000 },
+	{ .freq = 4780000000 },
+	{ .freq = 5333000000 },
+};
+
+/*
+ * DDR_QOS is a boolean bus: the firmware reports level 0 (nominal) or 1
+ * (boost) rather than a frequency. dev_pm_opp_add_dynamic() rejects a zero
+ * frequency, so the two OPP entries below use 1 and 100 purely as distinct,
+ * non-zero devfreq keys for the two levels; the values are not real
+ * frequencies. scmi_qcom_devfreq_get_cur_freq() maps the reported level back
+ * to the matching OPP frequency.
+ */
+static const struct scmi_qcom_opp_data glymur_ddr_qos_table[] = {
+	{ .freq = 1 },
+	{ .freq = 100 },
+};
+
+static const struct scmi_qcom_memory_cfg glymur_memory_cfg[] = {
+	{
+		.memory_type = MEMLAT_HW_DDR,
+		.name = "ddr",
+		.mem_table = glymur_ddr_table,
+		.num_opps = ARRAY_SIZE(glymur_ddr_table),
+		.grp_ev = glymur_ddr_grp_ev,
+		.monitor_cnt = 4,
+		.memory_range = { .min_freq_khz = 547000, .max_freq_khz = 4761000 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0x3f,
+				.ipm_ceil = 60000000,
+				.be_stall_floor = 1,
+				.table_len = 8,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 960, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 1133, .mem_freq_mhz = 1353 },
+					{ .cpu_freq_mhz = 1594, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 1920, .mem_freq_mhz = 1708 },
+					{ .cpu_freq_mhz = 2228, .mem_freq_mhz = 2736 },
+					{ .cpu_freq_mhz = 2362, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 2650, .mem_freq_mhz = 3686 },
+					{ .cpu_freq_mhz = 2938, .mem_freq_mhz = 4761 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xfc0,
+				.ipm_ceil = 60000000,
+				.be_stall_floor = 1,
+				.table_len = 8,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 356, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 1018, .mem_freq_mhz = 1353 },
+					{ .cpu_freq_mhz = 1536, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 1748, .mem_freq_mhz = 1708 },
+					{ .cpu_freq_mhz = 2324, .mem_freq_mhz = 2736 },
+					{ .cpu_freq_mhz = 2496, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 2900, .mem_freq_mhz = 3686 },
+					{ .cpu_freq_mhz = 3514, .mem_freq_mhz = 4761 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0x3f000,
+				.ipm_ceil = 60000000,
+				.be_stall_floor = 1,
+				.table_len = 8,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 356, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 1018, .mem_freq_mhz = 1353 },
+					{ .cpu_freq_mhz = 1536, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 1748, .mem_freq_mhz = 1708 },
+					{ .cpu_freq_mhz = 2324, .mem_freq_mhz = 2736 },
+					{ .cpu_freq_mhz = 2496, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 2900, .mem_freq_mhz = 3686 },
+					{ .cpu_freq_mhz = 3514, .mem_freq_mhz = 4761 },
+				}
+			},
+			{
+				.name = "mon_3",
+				.cpu_mask = 0x3ffff,
+				.table_len = 4,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2823, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 3034, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 3226, .mem_freq_mhz = 1708 },
+					{ .cpu_freq_mhz = 5012, .mem_freq_mhz = 2092 },
+				}
+			},
+		},
+	},
+	{
+		.memory_type = MEMLAT_HW_LLCC,
+		.name = "llcc",
+		.mem_table = glymur_llcc_table,
+		.num_opps = ARRAY_SIZE(glymur_llcc_table),
+		.grp_ev = glymur_llcc_grp_ev,
+		.monitor_cnt = 3,
+		.memory_range = { .min_freq_khz = 315000, .max_freq_khz = 1211000 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0x3f,
+				.ipm_ceil = 60000000,
+				.be_stall_floor = 1,
+				.table_len = 7,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 960, .mem_freq_mhz = 315 },
+					{ .cpu_freq_mhz = 1113, .mem_freq_mhz = 479 },
+					{ .cpu_freq_mhz = 1594, .mem_freq_mhz = 545 },
+					{ .cpu_freq_mhz = 1920, .mem_freq_mhz = 725 },
+					{ .cpu_freq_mhz = 2362, .mem_freq_mhz = 840 },
+					{ .cpu_freq_mhz = 2650, .mem_freq_mhz = 959 },
+					{ .cpu_freq_mhz = 2938, .mem_freq_mhz = 1211 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xfc0,
+				.ipm_ceil = 60000000,
+				.be_stall_floor = 1,
+				.table_len = 7,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 356, .mem_freq_mhz = 315 },
+					{ .cpu_freq_mhz = 1018, .mem_freq_mhz = 479 },
+					{ .cpu_freq_mhz = 1536, .mem_freq_mhz = 545 },
+					{ .cpu_freq_mhz = 1748, .mem_freq_mhz = 725 },
+					{ .cpu_freq_mhz = 2496, .mem_freq_mhz = 840 },
+					{ .cpu_freq_mhz = 2900, .mem_freq_mhz = 959 },
+					{ .cpu_freq_mhz = 3514, .mem_freq_mhz = 1211 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0x3f000,
+				.ipm_ceil = 60000000,
+				.be_stall_floor = 1,
+				.table_len = 7,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 356, .mem_freq_mhz = 315 },
+					{ .cpu_freq_mhz = 1018, .mem_freq_mhz = 479 },
+					{ .cpu_freq_mhz = 1536, .mem_freq_mhz = 545 },
+					{ .cpu_freq_mhz = 1748, .mem_freq_mhz = 725 },
+					{ .cpu_freq_mhz = 2496, .mem_freq_mhz = 840 },
+					{ .cpu_freq_mhz = 2900, .mem_freq_mhz = 959 },
+					{ .cpu_freq_mhz = 3514, .mem_freq_mhz = 1211 },
+				}
+			},
+		},
+	},
+	{
+		.memory_type = MEMLAT_HW_DDR_QOS_COMPUTE,
+		.name = "ddr-qos",
+		.monitor_cnt = 3,
+		.mem_table = glymur_ddr_qos_table,
+		.num_opps = ARRAY_SIZE(glymur_ddr_qos_table),
+		.grp_ev = glymur_ddr_qos_grp_ev,
+		.memory_range = { .min_freq_khz = 0, .max_freq_khz = 1 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0x3f,
+				.ipm_ceil = 80000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2362, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 2938, .mem_freq_mhz = 1 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xfc0,
+				.ipm_ceil = 80000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2496, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 3514, .mem_freq_mhz = 1 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0x3f000,
+				.ipm_ceil = 80000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2496, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 3514, .mem_freq_mhz = 1 },
+				}
+			},
+		},
+	},
+};
+
+static const struct scmi_qcom_memory_cfg hamoa_memory_cfg[] = {
+	{
+		.memory_type = MEMLAT_HW_DDR,
+		.name = "ddr",
+		.mem_table = hamoa_ddr_table,
+		.num_opps = ARRAY_SIZE(hamoa_ddr_table),
+		.grp_ev = hamoa_ddr_grp_ev,
+		.monitor_cnt = 4,
+		.memory_range = { .min_freq_khz = 200000, .max_freq_khz = 4224000 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0xf,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 6,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 999, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 1440, .mem_freq_mhz = 768 },
+					{ .cpu_freq_mhz = 1671, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 2092 },
+					{ .cpu_freq_mhz = 2516, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 4224 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xf0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 6,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 999, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 1440, .mem_freq_mhz = 768 },
+					{ .cpu_freq_mhz = 1671, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 2092 },
+					{ .cpu_freq_mhz = 2516, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 4224 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0xf00,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 6,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 999, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 1440, .mem_freq_mhz = 768 },
+					{ .cpu_freq_mhz = 1671, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 2092 },
+					{ .cpu_freq_mhz = 2516, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 4224 },
+				}
+			},
+			{
+				.name = "mon_3",
+				.cpu_mask = 0xfff,
+				.table_len = 4,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 1440, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 768 },
+					{ .cpu_freq_mhz = 2516, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 2092 },
+				}
+			},
+		},
+	},
+	{
+		.memory_type = MEMLAT_HW_LLCC,
+		.name = "llcc",
+		.mem_table = hamoa_llcc_table,
+		.num_opps = ARRAY_SIZE(hamoa_llcc_table),
+		.grp_ev = hamoa_llcc_grp_ev,
+		.monitor_cnt = 3,
+		.memory_range = { .min_freq_khz = 300000, .max_freq_khz = 1066000 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0xf,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 6,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 999, .mem_freq_mhz = 300 },
+					{ .cpu_freq_mhz = 1440, .mem_freq_mhz = 466 },
+					{ .cpu_freq_mhz = 1671, .mem_freq_mhz = 600 },
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 806 },
+					{ .cpu_freq_mhz = 2516, .mem_freq_mhz = 933 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 1066 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xf0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 6,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 999, .mem_freq_mhz = 300 },
+					{ .cpu_freq_mhz = 1440, .mem_freq_mhz = 466 },
+					{ .cpu_freq_mhz = 1671, .mem_freq_mhz = 600 },
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 806 },
+					{ .cpu_freq_mhz = 2516, .mem_freq_mhz = 933 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 1066 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0xf00,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 6,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 999, .mem_freq_mhz = 300 },
+					{ .cpu_freq_mhz = 1440, .mem_freq_mhz = 466 },
+					{ .cpu_freq_mhz = 1671, .mem_freq_mhz = 600 },
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 806 },
+					{ .cpu_freq_mhz = 2516, .mem_freq_mhz = 933 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 1066 },
+				}
+			},
+		},
+	},
+	{
+		.memory_type = MEMLAT_HW_DDR_QOS_COMPUTE,
+		.name = "ddr-qos",
+		.monitor_cnt = 3,
+		.mem_table = glymur_ddr_qos_table,
+		.num_opps = ARRAY_SIZE(glymur_ddr_qos_table),
+		.grp_ev = hamoa_ddr_qos_grp_ev,
+		.memory_range = { .min_freq_khz = 0, .max_freq_khz = 1 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0xf,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 1 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xf0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 1 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0xf00,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2189, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 3860, .mem_freq_mhz = 1 },
+				}
+			},
+		},
+	},
+};
+
+static const struct scmi_qcom_memory_cfg kaanapali_memory_cfg[] = {
+	{
+		.memory_type = MEMLAT_HW_DDR,
+		.name = "ddr",
+		.mem_table = kaanapali_ddr_table,
+		.num_opps = ARRAY_SIZE(kaanapali_ddr_table),
+		.grp_ev = kaanapali_ddr_grp_ev,
+		.monitor_cnt = 4,
+		.memory_range = { .min_freq_khz = 547000, .max_freq_khz = 5333000 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0x3f,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 6,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 787, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 960, .mem_freq_mhz = 1353 },
+					{ .cpu_freq_mhz = 1200, .mem_freq_mhz = 1555 },
+					{ .cpu_freq_mhz = 1536, .mem_freq_mhz = 2092 },
+					{ .cpu_freq_mhz = 2534, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 3552, .mem_freq_mhz = 3686 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xc0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 8,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 614, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 864, .mem_freq_mhz = 1353 },
+					{ .cpu_freq_mhz = 1382, .mem_freq_mhz = 2092 },
+					{ .cpu_freq_mhz = 2131, .mem_freq_mhz = 3187 },
+					{ .cpu_freq_mhz = 2726, .mem_freq_mhz = 3686 },
+					{ .cpu_freq_mhz = 3513, .mem_freq_mhz = 4224 },
+					{ .cpu_freq_mhz = 3916, .mem_freq_mhz = 4780 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 5333 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0xff,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2035, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 2092 },
+				}
+			},
+			{
+				.name = "mon_3",
+				.cpu_mask = 0xc0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2726, .mem_freq_mhz = 547 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 4224 },
+				}
+			},
+		},
+	},
+	{
+		.memory_type = MEMLAT_HW_LLCC,
+		.name = "llcc",
+		.mem_table = kaanapali_llcc_table,
+		.num_opps = ARRAY_SIZE(kaanapali_llcc_table),
+		.grp_ev = kaanapali_llcc_grp_ev,
+		.monitor_cnt = 3,
+		.memory_range = { .min_freq_khz = 282000, .max_freq_khz = 1350000 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0x3f,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 5,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 787, .mem_freq_mhz = 282 },
+					{ .cpu_freq_mhz = 960, .mem_freq_mhz = 350 },
+					{ .cpu_freq_mhz = 1536, .mem_freq_mhz = 533 },
+					{ .cpu_freq_mhz = 2534, .mem_freq_mhz = 806 },
+					{ .cpu_freq_mhz = 3552, .mem_freq_mhz = 933 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xc0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 8,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 614, .mem_freq_mhz = 282 },
+					{ .cpu_freq_mhz = 864, .mem_freq_mhz = 350 },
+					{ .cpu_freq_mhz = 1382, .mem_freq_mhz = 533 },
+					{ .cpu_freq_mhz = 2131, .mem_freq_mhz = 806 },
+					{ .cpu_freq_mhz = 2726, .mem_freq_mhz = 933 },
+					{ .cpu_freq_mhz = 3513, .mem_freq_mhz = 1066 },
+					{ .cpu_freq_mhz = 3916, .mem_freq_mhz = 1211 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 1350 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0xff,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2035, .mem_freq_mhz = 282 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 533 },
+				}
+			},
+		},
+	},
+	{
+		.memory_type = MEMLAT_HW_DDR_QOS_MOBILE,
+		.name = "ddr-qos",
+		.monitor_cnt = 4,
+		.mem_table = glymur_ddr_qos_table,
+		.num_opps = ARRAY_SIZE(glymur_ddr_qos_table),
+		.grp_ev = kaanapali_ddr_qos_grp_ev,
+		.memory_range = { .min_freq_khz = 0, .max_freq_khz = 1 },
+		.monitor_cfg = (const struct scmi_qcom_monitor_cfg[]) {
+			{
+				.name = "mon_0",
+				.cpu_mask = 0xff,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2035, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 1 },
+				}
+			},
+			{
+				.name = "mon_1",
+				.cpu_mask = 0xc0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 1574, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 1 },
+				}
+			},
+			{
+				.name = "mon_2",
+				.cpu_mask = 0xc0,
+				.ipm_ceil = 20000000,
+				.be_stall_floor = 1,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 2131, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 1 },
+				}
+			},
+			{
+				.name = "mon_3",
+				.cpu_mask = 0xc0,
+				.table_len = 2,
+				.table = (const struct scmi_qcom_map_table[]) {
+					{ .cpu_freq_mhz = 3513, .mem_freq_mhz = 0 },
+					{ .cpu_freq_mhz = 4185, .mem_freq_mhz = 1 },
+				}
+			},
+		},
+	},
+};
+
+static const struct scmi_qcom_memlat_cfg_data glymur_memlat_data = {
+	.memory_cfg = glymur_memory_cfg,
+	.common_ev = glymur_common_ev,
+	.cpucp_legacy_freq_method = true,
+	.cpucp_sample_ms = 4,
+	.memory_cnt = ARRAY_SIZE(glymur_memory_cfg),
+};
+
+static const struct scmi_qcom_memlat_cfg_data hamoa_memlat_data = {
+	.memory_cfg = hamoa_memory_cfg,
+	.common_ev = hamoa_common_ev,
+	.cpucp_legacy_freq_method = true,
+	.cpucp_sample_ms = 4,
+	.memory_cnt = ARRAY_SIZE(hamoa_memory_cfg),
+};
+
+static const struct scmi_qcom_memlat_cfg_data kaanapali_memlat_data = {
+	.memory_cfg = kaanapali_memory_cfg,
+	.common_ev = kaanapali_common_ev,
+	.cpucp_legacy_freq_method = true,
+	.cpucp_sample_ms = 4,
+	.memory_cnt = ARRAY_SIZE(kaanapali_memory_cfg),
+};
+
+static const struct of_device_id scmi_qcom_memlat_configs[] = {
+	{ .compatible = "qcom,glymur", .data = &glymur_memlat_data },
+	{ .compatible = "qcom,mahua", .data = &glymur_memlat_data },
+	{ .compatible = "qcom,x1e80100", .data = &hamoa_memlat_data },
+	{ .compatible = "qcom,x1p42100", .data = &hamoa_memlat_data },
+	{ .compatible = "qcom,kaanapali", .data = &kaanapali_memlat_data },
+	{ }
+};
+
+static int memlat_set_param(struct scmi_qcom_memlat_info *info, void *msg,
+			    size_t size, u32 param_id)
+{
+	return info->ops->set_param(info->ph, msg, size, MEMLAT_ALGO_STR, param_id);
+}
+
+static int configure_cpucp_common_events(struct scmi_qcom_memlat_info *info)
+{
+	struct ev_map_msg ev = {};
+	int i;
+
+	ev.num_evs = cpu_to_le32(NUM_COMMON_EVS);
+	ev.hw_type = cpu_to_le32(MEMLAT_HW_TYPE_ALL);
+	for (i = 0; i < NUM_COMMON_EVS; i++)
+		ev.cid[i] = cpu_to_le32(info->cfg_data->common_ev[i]);
+
+	return memlat_set_param(info, &ev, sizeof(ev), MEMLAT_SET_COMMON_EV_MAP);
+}
+
+static int configure_cpucp_grp(struct device *dev, struct scmi_qcom_memlat_info *info,
+			       int memory_index)
+{
+	const struct scmi_qcom_memory_cfg *cfg = &info->cfg_data->memory_cfg[memory_index];
+	struct ev_map_msg ev = {};
+	struct node_msg node = {};
+	int ret, i;
+
+	node.cpumask = cpu_to_le32(*cpumask_bits(cpu_possible_mask));
+	node.hw_type = cpu_to_le32(cfg->memory_type);
+
+	/* mon_type/mon_idx are don't-care for SET_MEM_GROUP; leave them zero. */
+	ret = memlat_set_param(info, &node, sizeof(node), MEMLAT_SET_MEM_GROUP);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to configure mem type %d\n",
+				     cfg->memory_type);
+
+	ev.num_evs = cpu_to_le32(NUM_GRP_EVS);
+	ev.hw_type = cpu_to_le32(cfg->memory_type);
+	for (i = 0; i < NUM_GRP_EVS; i++)
+		ev.cid[i] = cpu_to_le32(cfg->grp_ev[i]);
+
+	ret = memlat_set_param(info, &ev, sizeof(ev), MEMLAT_SET_GRP_EV_MAP);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to configure event map for mem type %d\n",
+				     cfg->memory_type);
+
+	return 0;
+}
+
+static int configure_cpucp_mon(struct device *dev, struct scmi_qcom_memlat_info *info,
+			       int memory_index, int monitor_index)
+{
+	const struct scmi_qcom_memory_cfg *cfg = &info->cfg_data->memory_cfg[memory_index];
+	const struct scmi_qcom_monitor_cfg *mon = &cfg->monitor_cfg[monitor_index];
+	struct scalar_param_msg scalar = {};
+	struct map_param_msg map = {};
+	struct node_msg node = {};
+	int ret, i;
+
+	if (mon->table_len > MEMLAT_MAX_MAP_ENTRIES)
+		return -EINVAL;
+
+	node.cpumask = cpu_to_le32(mon->cpu_mask);
+	node.hw_type = cpu_to_le32(cfg->memory_type);
+
+	/* The compute monitor passively scales with cpufreq irrespective of IPM ratio. */
+	node.mon_type = cpu_to_le32(!mon->ipm_ceil);
+	node.mon_idx = cpu_to_le32(monitor_index);
+	strscpy(node.mon_name, mon->name, sizeof(node.mon_name));
+	ret = memlat_set_param(info, &node, sizeof(node), MEMLAT_SET_MONITOR);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to configure monitor %s\n", mon->name);
+
+	scalar.hw_type = cpu_to_le32(cfg->memory_type);
+	scalar.mon_idx = cpu_to_le32(monitor_index);
+	scalar.val = cpu_to_le32(mon->ipm_ceil);
+	ret = memlat_set_param(info, &scalar, sizeof(scalar), MEMLAT_IPM_CEIL);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to set ipm ceil for %s\n", mon->name);
+
+	scalar.val = cpu_to_le32(mon->be_stall_floor);
+	ret = memlat_set_param(info, &scalar, sizeof(scalar), MEMLAT_BE_STALL_FLOOR);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to set be_stall_floor for %s\n", mon->name);
+
+	map.hw_type = cpu_to_le32(cfg->memory_type);
+	map.mon_idx = cpu_to_le32(monitor_index);
+	map.nr_rows = cpu_to_le32(mon->table_len);
+	for (i = 0; i < mon->table_len; i++) {
+		map.tbl[i].cpu_freq_mhz = cpu_to_le16(mon->table[i].cpu_freq_mhz);
+		map.tbl[i].mem_freq_mhz = cpu_to_le16(mon->table[i].mem_freq_mhz);
+	}
+	ret = memlat_set_param(info, &map, sizeof(map), MEMLAT_MON_FREQ_MAP);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to configure freq_map for %s\n", mon->name);
+
+	scalar.val = cpu_to_le32(cfg->memory_range.min_freq_khz);
+	ret = memlat_set_param(info, &scalar, sizeof(scalar), MEMLAT_SET_MIN_FREQ);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to set min_freq for %s\n", mon->name);
+
+	scalar.val = cpu_to_le32(cfg->memory_range.max_freq_khz);
+	ret = memlat_set_param(info, &scalar, sizeof(scalar), MEMLAT_SET_MAX_FREQ);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to set max_freq for %s\n", mon->name);
+
+	return 0;
+}
+
+static int scmi_qcom_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
+{
+	struct scmi_qcom_memory_info *memory = dev_get_drvdata(dev);
+	const struct qcom_generic_ext_ops *ops = memory->ops;
+	const struct scmi_qcom_memory_cfg *cfg = memory->cfg;
+	u32 max_freq_khz = 0;
+	int ret, i;
+
+	/* The bus's voted frequency is the max reported across all its monitors. */
+	for (i = 0; i < cfg->monitor_cnt; i++) {
+		union cur_freq_msg msg = {};
+
+		msg.req.hw_type = cpu_to_le32(cfg->memory_type);
+		msg.req.mon_idx = cpu_to_le32(i);
+
+		ret = ops->get_param(memory->ph, &msg, sizeof(msg.req), MEMLAT_ALGO_STR,
+				     MEMLAT_GET_CUR_FREQ, sizeof(msg.resp));
+		if (ret < 0) {
+			/* One monitor failing shouldn't abort the whole read-back. */
+			dev_warn_once(dev, "failed to get current frequency for %s\n",
+				      cfg->monitor_cfg[i].name);
+			continue;
+		}
+
+		max_freq_khz = max(max_freq_khz, le32_to_cpu(msg.resp));
+	}
+
+	/*
+	 * DDR/LLCC report a frequency in kHz (converted to Hz for the OPP
+	 * table); DDR_QOS reports a 0/1 level mapped to its synthetic OPP keys.
+	 */
+	if (cfg->memory_range.max_freq_khz > 1)
+		*freq = (unsigned long)((u64)max_freq_khz * HZ_PER_KHZ);
+	else
+		*freq = max_freq_khz ? 100 : 1;
+
+	return 0;
+}
+
+static void scmi_qcom_memlat_teardown(struct scmi_qcom_memlat_info *info)
+{
+	for (int i = 0; i < info->memory_cnt; i++) {
+		struct scmi_qcom_memory_info *memory = info->memory[i];
+
+		if (!memory || !memory->fdev)
+			continue;
+
+		if (memory->devfreq)
+			devfreq_remove_device(memory->devfreq);
+		dev_pm_opp_remove_all_dynamic(&memory->fdev->dev);
+		faux_device_destroy(memory->fdev);
+	}
+}
+
+static int scmi_qcom_memlat_configure_events(struct scmi_device *sdev,
+					     struct scmi_qcom_memlat_info *info)
+{
+	const struct qcom_generic_ext_ops *ops = info->ops;
+	struct scmi_protocol_handle *ph = info->ph;
+	__le32 sample_ms, freq_method;
+	int i, j, ret;
+
+	ret = configure_cpucp_common_events(info);
+	if (ret < 0)
+		return dev_err_probe(&sdev->dev, ret, "failed to configure common events\n");
+
+	for (i = 0; i < info->memory_cnt; i++) {
+		ret = configure_cpucp_grp(&sdev->dev, info, i);
+		if (ret < 0)
+			return ret;
+
+		for (j = 0; j < info->memory[i]->cfg->monitor_cnt; j++) {
+			ret = configure_cpucp_mon(&sdev->dev, info, i, j);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	sample_ms = cpu_to_le32(info->cfg_data->cpucp_sample_ms);
+	ret = memlat_set_param(info, &sample_ms, sizeof(sample_ms), MEMLAT_SAMPLE_MS);
+	if (ret < 0)
+		return dev_err_probe(&sdev->dev, ret, "failed to set sample_ms\n");
+
+	freq_method = cpu_to_le32(info->cfg_data->cpucp_legacy_freq_method);
+	ret = memlat_set_param(info, &freq_method, sizeof(freq_method),
+			       MEMLAT_SET_EFFECTIVE_FREQ_METHOD);
+	if (ret < 0)
+		return dev_err_probe(&sdev->dev, ret,
+				     "failed to set effective frequency calc method\n");
+
+	/* Start sampling and voting timer */
+	ret = ops->start_activity(ph, NULL, 0, MEMLAT_ALGO_STR, MEMLAT_START_TIMER);
+	if (ret < 0)
+		return dev_err_probe(&sdev->dev, ret, "failed to start memory group timer\n");
+
+	for (i = 0; i < info->memory_cnt; i++) {
+		struct scmi_qcom_memory_info *memory = info->memory[i];
+		const struct scmi_qcom_memory_range *range = &memory->cfg->memory_range;
+		struct devfreq_dev_profile *profile = &memory->profile;
+
+		/*
+		 * CPUCP re-evaluates and re-votes at most once per cpucp_sample_ms,
+		 * so a new operating point can appear only that often. Polling at half
+		 * that period guarantees each distinct vote is observed in trans_stat
+		 * before it can change again.
+		 */
+		profile->polling_ms = max(1U, info->cfg_data->cpucp_sample_ms / 2);
+		profile->get_cur_freq = scmi_qcom_devfreq_get_cur_freq;
+		profile->initial_freq = range->max_freq_khz > 1 ?
+					(unsigned long)((u64)range->min_freq_khz * HZ_PER_KHZ) :
+					range->min_freq_khz;
+
+		faux_device_set_drvdata(memory->fdev, memory);
+
+		memory->devfreq = devfreq_add_device(&memory->fdev->dev, profile,
+						     DEVFREQ_GOV_REMOTE, NULL);
+		if (IS_ERR(memory->devfreq)) {
+			ret = PTR_ERR(memory->devfreq);
+			memory->devfreq = NULL;
+			ops->stop_activity(ph, NULL, 0, MEMLAT_ALGO_STR, MEMLAT_STOP_TIMER);
+			return dev_err_probe(&sdev->dev, ret, "failed to add devfreq device\n");
+		}
+	}
+
+	return 0;
+}
+
+static int scmi_qcom_memlat_setup(struct scmi_device *sdev, struct scmi_qcom_memlat_info *info)
+{
+	const struct scmi_qcom_memlat_cfg_data *cfg_data;
+	int ret, i, j;
+
+	cfg_data = of_machine_get_match_data(scmi_qcom_memlat_configs);
+	if (!cfg_data) {
+		dev_dbg(&sdev->dev, "no memlat config data for this platform\n");
+		return -ENODEV;
+	}
+
+	info->cfg_data = cfg_data;
+	info->memory = devm_kcalloc(&sdev->dev, cfg_data->memory_cnt,
+				    sizeof(*info->memory), GFP_KERNEL);
+	if (!info->memory)
+		return -ENOMEM;
+
+	for (i = 0; i < cfg_data->memory_cnt; i++) {
+		const struct scmi_qcom_memory_cfg *memory_cfg = &cfg_data->memory_cfg[i];
+		struct scmi_qcom_memory_info *memory;
+
+		memory = devm_kzalloc(&sdev->dev, sizeof(*memory), GFP_KERNEL);
+		if (!memory) {
+			ret = -ENOMEM;
+			goto err_teardown;
+		}
+
+		memory->cfg = memory_cfg;
+		memory->ops = info->ops;
+		memory->ph = info->ph;
+		info->memory[i] = memory;
+
+		memory->fdev = faux_device_create(memory_cfg->name, &sdev->dev, NULL);
+		if (!memory->fdev) {
+			ret = dev_err_probe(&sdev->dev, -ENODEV,
+					    "failed to create faux device\n");
+			goto err_teardown;
+		}
+		info->memory_cnt = i + 1;
+
+		for (j = 0; j < memory_cfg->num_opps; j++) {
+			struct dev_pm_opp_data data = {
+				.freq = memory_cfg->mem_table[j].freq,
+			};
+
+			ret = dev_pm_opp_add_dynamic(&memory->fdev->dev, &data);
+			if (ret) {
+				dev_err_probe(&sdev->dev, ret, "failed to add OPP\n");
+				goto err_teardown;
+			}
+		}
+	}
+
+	return 0;
+
+err_teardown:
+	scmi_qcom_memlat_teardown(info);
+	return ret;
+}
+
+static int scmi_qcom_devfreq_memlat_probe(struct scmi_device *sdev)
+{
+	const struct scmi_handle *handle = sdev->handle;
+	const struct qcom_generic_ext_ops *ops;
+	struct scmi_qcom_memlat_info *info;
+	struct scmi_protocol_handle *ph;
+	int ret;
+
+	if (!handle)
+		return -ENODEV;
+
+	info = devm_kzalloc(&sdev->dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	ops = handle->devm_protocol_get(sdev, SCMI_PROTOCOL_QCOM_GENERIC, &ph);
+	if (IS_ERR(ops))
+		return PTR_ERR(ops);
+
+	info->ops = ops;
+	info->ph = ph;
+
+	ret = scmi_qcom_memlat_setup(sdev, info);
+	if (ret)
+		return ret;
+
+	ret = scmi_qcom_memlat_configure_events(sdev, info);
+	if (ret) {
+		scmi_qcom_memlat_teardown(info);
+		return ret;
+	}
+
+	dev_set_drvdata(&sdev->dev, info);
+
+	return 0;
+}
+
+static void scmi_qcom_devfreq_memlat_remove(struct scmi_device *sdev)
+{
+	struct scmi_qcom_memlat_info *info = dev_get_drvdata(&sdev->dev);
+	int ret;
+
+	ret = info->ops->stop_activity(info->ph, NULL, 0, MEMLAT_ALGO_STR, MEMLAT_STOP_TIMER);
+	if (ret < 0)
+		dev_err(&sdev->dev, "failed to stop memory group timer\n");
+
+	scmi_qcom_memlat_teardown(info);
+}
+
+static const struct scmi_device_id scmi_id_table[] = {
+	{ SCMI_PROTOCOL_QCOM_GENERIC, "qcom-generic-ext" },
+	{ },
+};
+MODULE_DEVICE_TABLE(scmi, scmi_id_table);
+
+static struct scmi_driver scmi_qcom_devfreq_memlat_driver = {
+	.name		= "scmi-qcom-devfreq-memlat",
+	.probe		= scmi_qcom_devfreq_memlat_probe,
+	.remove		= scmi_qcom_devfreq_memlat_remove,
+	.id_table	= scmi_id_table,
+};
+module_scmi_driver(scmi_qcom_devfreq_memlat_driver);
+
+MODULE_AUTHOR("Pragnesh Papaniya <pragnesh.papaniya@xxxxxxxxxxxxxxxx>");
+MODULE_AUTHOR("Sibi Sankar <sibi.sankar@xxxxxxxxxxxxxxxx>");
+MODULE_DESCRIPTION("Qualcomm SCMI memlat devfreq driver");
+MODULE_LICENSE("GPL");

-- 
2.34.1