[PATCH v5 04/10] x86/resctrl: Attach ACPI ERDT information to L3 mon domain on CPU online

[Chen Yu]

After the rdt_hw_l3_mon_domain has been created during CPU online,
attach the pre-parsed ACPI ERDT table information to the
rdt_hw_l3_mon_domain to facilitate monitor data read via the
ERDT and its sub-tables information.

During attachment, a sanity check is triggered to verify whether the
CPU mask reported by silicon (CPUID leaf 4) matches the information
exposed by firmware (CACD table). If inconsistent, this CPU will
be cleared from the legitimate domain mask.

Suggested-by: Reinette Chatre <reinette.chatre@xxxxxxxxx>
Tested-by: Hongyu Ning <hongyu.ning@xxxxxxxxxxxxxxx>
Signed-off-by: Chen Yu <yu.c.chen@xxxxxxxxx>
---
v4->v5:
   new patch.
---
 arch/x86/kernel/cpu/resctrl/core.c     | 17 +++++++++
 arch/x86/kernel/cpu/resctrl/erdt.c     | 49 ++++++++++++++++++++++++++
 arch/x86/kernel/cpu/resctrl/internal.h |  4 +++
 3 files changed, 70 insertions(+)

diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 23925bcd71d7..2e95586ebe45 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -549,6 +549,19 @@ static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct
 	d->ci_id = ci->id;
 	cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
 
+	/*
+	 * Verify whether the CPU domain information matches the ACPI data.
+	 * Skip adding the newly created domain to the list if there is a mismatch.
+	 * ACPI information should be assigned to the domain prior to its insertion
+	 * into the list, in case others might iterate the list in parallel.
+	 */
+	if (erdt_l3_mon_domain_setup(cpu, &d->hdr)) {
+		pr_warn("CPU%d has inconsistent domain information, do not add this new domain\n", cpu);
+		cpumask_clear_cpu(cpu, &d->hdr.cpu_mask);
+		l3_mon_domain_free(hw_dom);
+		return;
+	}
+
 	arch_mon_domain_online(r, d);
 
 	if (l3_mon_domain_mbm_alloc(r->mon.num_rmid, hw_dom)) {
@@ -591,6 +604,10 @@ static void domain_add_cpu_mon(int cpu, struct rdt_resource *r)
 			resctrl_arch_mbm_cntr_assign_set_one(r);
 		if (!hdr)
 			l3_mon_domain_setup(cpu, id, r, add_pos);
+		else if (erdt_l3_mon_domain_setup(cpu, hdr)) {
+			pr_warn("CPU%d has inconsistent domain information, remove it from the domain\n", cpu);
+			cpumask_clear_cpu(cpu, &hdr->cpu_mask);
+		}
 		break;
 	case RDT_RESOURCE_PERF_PKG:
 		if (!hdr)
diff --git a/arch/x86/kernel/cpu/resctrl/erdt.c b/arch/x86/kernel/cpu/resctrl/erdt.c
index 6405df9be817..6c1df7e43eab 100644
--- a/arch/x86/kernel/cpu/resctrl/erdt.c
+++ b/arch/x86/kernel/cpu/resctrl/erdt.c
@@ -212,6 +212,55 @@ static __init bool parse_rmdd_entry(struct acpi_subtbl_hdr_16 *rmdd_hdr)
 	return false;
 }
 
+/*
+ * Associate ERDT table information with this domain.
+ */
+int erdt_l3_mon_domain_setup(int cpu, struct rdt_domain_hdr *hdr)
+{
+	struct rdt_hw_l3_mon_domain *hw_dom;
+	struct erdt_domain_info *d;
+	struct list_head *pos;
+
+	if (!__erdt_enabled)
+		return 0;
+
+	/*
+	 * Find the erdt_domain_info that contains this CPU,
+	 * compare erdt_domain_info's cpumask with the cpumask
+	 * exposed by hw_dom (derived from CPUID leaf 4).
+	 * If yes, assign the erdt_domain_info in the hw_dom,
+	 * otherwise this CPU should be isolated from resctrl.
+	 * For example, the hw_dom reports CPU{0,1} are in
+	 * l3 domain0, CPU{2,3} belongs to domain1. Meanwhile
+	 * erdt_domain_info reports that CPU{0,2} are in domain0,
+	 * CPU{1,3} are in domain1. So when it comes to CPU1,
+	 * a mismatch is detected, we should remove CPU1 from
+	 * resctrl.
+	 */
+	list_for_each(pos, &domain_info_list) {
+		d = container_of(pos, struct erdt_domain_info, list);
+
+		if (cpumask_test_cpu(cpu, d->cpu_mask)) {
+			if (!cpumask_subset(&hdr->cpu_mask, d->cpu_mask)) {
+				pr_warn(FW_BUG "Mismatch detected, CPU%d in L3 domain(%*pbl) and CACD domain(%*pbl)\n",
+					cpu, cpumask_pr_args(&hdr->cpu_mask), cpumask_pr_args(d->cpu_mask));
+
+				return -EIO;
+			}
+
+			hw_dom = resctrl_to_arch_mon_dom(container_of(hdr, struct rdt_l3_mon_domain, hdr));
+			/* No mismatch, assign the ERDT information to hw_dom */
+			if (!hw_dom->d_info)
+				hw_dom->d_info = d;
+
+			return 0;
+		}
+	}
+
+	pr_warn(FW_BUG "Cannot find CACD domain for CPU%d\n", cpu);
+	return -ENOENT;
+}
+
 void erdt_exit(void)
 {
 	struct erdt_domain_info *d;
diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h
index 299d7222f693..7d9100b7648f 100644
--- a/arch/x86/kernel/cpu/resctrl/internal.h
+++ b/arch/x86/kernel/cpu/resctrl/internal.h
@@ -92,14 +92,18 @@ struct rdt_hw_ctrl_domain {
  * @arch_mbm_states:	Per-event pointer to the MBM event's saved state.
  *			An MBM event's state is an array of struct arch_mbm_state
  *			indexed by RMID on x86.
+ * @d_info:		ERDT table information of this domain(read-only)
  *
  * Members of this structure are accessed via helpers that provide abstraction.
  */
 struct rdt_hw_l3_mon_domain {
 	struct rdt_l3_mon_domain	d_resctrl;
 	struct arch_mbm_state		*arch_mbm_states[QOS_NUM_L3_MBM_EVENTS];
+	struct erdt_domain_info		*d_info;
 };
 
+int erdt_l3_mon_domain_setup(int cpu, struct rdt_domain_hdr *hdr);
+
 static inline struct rdt_hw_ctrl_domain *resctrl_to_arch_ctrl_dom(struct rdt_ctrl_domain *r)
 {
 	return container_of(r, struct rdt_hw_ctrl_domain, d_resctrl);
-- 
2.45.2