[PATCH 16/21] EDAC, ghes: Create one memory controller device per node

[Robert Richter]

Typically for most systems, there is one edac memory controller device
per node. This patch implements the same for the ghes driver. Now,
create multiple mc devices and map the dimms based on the node id.

We need at least one node that is used as fallback if no node
information is available in the error report.

Here a complete and consistent error report from a ThunderX2 system
(zero counter values dropped):

 # find /sys/devices/system/edac/mc/ -name \*count | sort -V | xargs grep . | sed -e '/:0/d'
 /sys/devices/system/edac/mc/mc0/ce_count:11
 /sys/devices/system/edac/mc/mc0/ce_noinfo_count:1
 /sys/devices/system/edac/mc/mc0/csrow2/ce_count:5
 /sys/devices/system/edac/mc/mc0/csrow2/ch0_ce_count:5
 /sys/devices/system/edac/mc/mc0/csrow3/ce_count:3
 /sys/devices/system/edac/mc/mc0/csrow3/ch0_ce_count:3
 /sys/devices/system/edac/mc/mc0/csrow4/ce_count:2
 /sys/devices/system/edac/mc/mc0/csrow4/ch0_ce_count:2
 /sys/devices/system/edac/mc/mc0/dimm2/dimm_ce_count:5
 /sys/devices/system/edac/mc/mc0/dimm3/dimm_ce_count:3
 /sys/devices/system/edac/mc/mc0/dimm4/dimm_ce_count:2
 /sys/devices/system/edac/mc/mc1/ce_count:7
 /sys/devices/system/edac/mc/mc1/csrow2/ce_count:4
 /sys/devices/system/edac/mc/mc1/csrow2/ch0_ce_count:4
 /sys/devices/system/edac/mc/mc1/csrow3/ce_count:1
 /sys/devices/system/edac/mc/mc1/csrow3/ch0_ce_count:1
 /sys/devices/system/edac/mc/mc1/csrow6/ce_count:2
 /sys/devices/system/edac/mc/mc1/csrow6/ch0_ce_count:2
 /sys/devices/system/edac/mc/mc1/dimm2/dimm_ce_count:4
 /sys/devices/system/edac/mc/mc1/dimm3/dimm_ce_count:1
 /sys/devices/system/edac/mc/mc1/dimm6/dimm_ce_count:2

Signed-off-by: Robert Richter <rrichter@xxxxxxxxxxx>
---
 drivers/edac/ghes_edac.c | 126 ++++++++++++++++++++++++++++++++-------
 1 file changed, 104 insertions(+), 22 deletions(-)

diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c
index c39cdfdfb8db..e5fa977bcfd9 100644
--- a/drivers/edac/ghes_edac.c
+++ b/drivers/edac/ghes_edac.c
@@ -18,6 +18,7 @@
 #include <ras/ras_event.h>
 
 struct ghes_edac_pvt {
+	struct device dev;
 	struct list_head list;
 	struct ghes *ghes;
 	struct mem_ctl_info *mci;
@@ -28,7 +29,7 @@ struct ghes_edac_pvt {
 };
 
 static atomic_t ghes_init = ATOMIC_INIT(0);
-static struct ghes_edac_pvt *ghes_pvt;
+struct mem_ctl_info *fallback;
 
 /*
  * Sync with other, potentially concurrent callers of
@@ -161,15 +162,15 @@ static void ghes_edac_set_nid(const struct dmi_header *dh, void *arg)
 	}
 }
 
-static int get_dimm_smbios_index(u16 handle)
+static int get_dimm_smbios_index(struct mem_ctl_info *mci, u16 handle)
 {
-	struct mem_ctl_info *mci = ghes_pvt->mci;
 	struct dimm_info *dimm;
 
 	mci_for_each_dimm(mci, dimm) {
 		if (dimm->smbios_handle == handle)
 			return dimm->idx;
 	}
+
 	return -1;
 }
 
@@ -370,6 +371,9 @@ static void mci_add_dimm_info(struct mem_ctl_info *mci)
 	int index = 0;
 
 	for_each_dimm(dimm) {
+		if (mci->mc_idx != dimm->numa_node)
+			continue;
+
 		dmi_dimm = &dimm->dimm_info;
 		mci_dimm = edac_get_dimm_by_index(mci, index);
 
@@ -390,17 +394,35 @@ static void mci_add_dimm_info(struct mem_ctl_info *mci)
 			index, mci->tot_dimms);
 }
 
+static struct mem_ctl_info *get_mc_by_node(int nid)
+{
+	struct mem_ctl_info *mci = edac_mc_find(nid);
+
+	if (mci)
+		return mci;
+
+	if (num_possible_nodes() > 1) {
+		edac_mc_printk(fallback, KERN_WARNING,
+			"Invalid or no node information, falling back to first node: %s",
+			fallback->dev_name);
+	}
+
+	return fallback;
+}
+
 void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 {
 	struct dimm_info *dimm_info;
 	enum hw_event_mc_err_type type;
 	struct edac_raw_error_desc *e;
 	struct mem_ctl_info *mci;
-	struct ghes_edac_pvt *pvt = ghes_pvt;
+	struct ghes_edac_pvt *pvt;
 	unsigned long flags;
 	char *p;
+	int nid = NUMA_NO_NODE;
 
-	if (!pvt)
+	/* We need at least one mc */
+	if (WARN_ON_ONCE(!fallback))
 		return;
 
 	/*
@@ -413,7 +435,11 @@ void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 
 	spin_lock_irqsave(&ghes_lock, flags);
 
-	mci = pvt->mci;
+	/* select the node's mc device */
+	if (mem_err->validation_bits & CPER_MEM_VALID_NODE)
+		nid = mem_err->node;
+	mci = get_mc_by_node(nid);
+	pvt = mci->pvt_info;
 	e = &mci->error_desc;
 
 	/* Cleans the error report buffer */
@@ -546,7 +572,7 @@ void ghes_edac_report_mem_error(int sev, struct cper_sec_mem_err *mem_err)
 			p += sprintf(p, "DIMM DMI handle: 0x%.4x ",
 				     mem_err->mem_dev_handle);
 
-		index = get_dimm_smbios_index(mem_err->mem_dev_handle);
+		index = get_dimm_smbios_index(mci, mem_err->mem_dev_handle);
 		if (index >= 0)
 			e->top_layer = index;
 	}
@@ -645,15 +671,29 @@ static struct acpi_platform_list plat_list[] = {
 	{ } /* End */
 };
 
+void ghes_edac_release(struct device *dev)
+{
+	struct ghes_edac_pvt *ghes_pvt;
+	struct mem_ctl_info *mci;
+
+	ghes_pvt = container_of(dev, struct ghes_edac_pvt, dev);
+
+	mci = ghes_pvt->mci;
+	edac_mc_del_mc(mci->pdev);
+	edac_mc_free(mci);
+}
+
 static int
 ghes_edac_register_one(int nid, struct ghes *ghes, struct device *parent)
 {
+	struct device *dev;
+	struct ghes_edac_pvt *ghes_pvt;
 	int rc;
 	struct mem_ctl_info *mci;
 	struct edac_mc_layer layers[1];
 
 	layers[0].type = EDAC_MC_LAYER_ALL_MEM;
-	layers[0].size = mem_info.num_dimm;
+	layers[0].size = mem_info.num_per_node[nid];
 	layers[0].is_virt_csrow = true;
 
 	mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers,
@@ -667,43 +707,69 @@ ghes_edac_register_one(int nid, struct ghes *ghes, struct device *parent)
 	ghes_pvt->ghes	= ghes;
 	ghes_pvt->mci	= mci;
 
-	mci->pdev = parent;
+	dev		= &ghes_pvt->dev;
+	dev->parent	= parent;
+	dev->release	= ghes_edac_release;
+	dev_set_name(dev, "ghes_mc%d", nid);
+
+	rc = device_register(dev);
+	if (rc) {
+		pr_err("Can't create EDAC device (%d)\n", rc);
+		goto fail;
+	}
+
+	mci->pdev = dev;
 	mci->mtype_cap = MEM_FLAG_EMPTY;
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "ghes_edac.c";
-	mci->ctl_name = "ghes_edac";
-	mci->dev_name = "ghes";
+	mci->ctl_name = "ghes_mc";
+	mci->dev_name = dev_name(dev);
 
 	mci_add_dimm_info(mci);
 
 	rc = edac_mc_add_mc(mci);
 	if (rc < 0) {
-		pr_err("Can't register at EDAC core\n");
-		edac_mc_free(mci);
-		return -ENODEV;
+		pr_err("Can't register at EDAC core (%d)\n", rc);
+		goto fail;
 	}
+
 	return 0;
+fail:
+	put_device(dev);
+	return rc;
+}
+
+static void ghes_edac_unregister_one(struct mem_ctl_info *mci)
+{
+	struct ghes_edac_pvt *pvt = mci->pvt_info;
+
+	put_device(&pvt->dev);
 }
 
 void ghes_edac_unregister(struct ghes *ghes)
 {
 	struct mem_ctl_info *mci;
+	int nid;
 
-	if (!ghes_pvt)
-		return;
-
-	mci = ghes_pvt->mci;
-	edac_mc_del_mc(mci->pdev);
-	edac_mc_free(mci);
+	for_each_node(nid) {
+		mci = edac_mc_find(nid);
+		/* stop fallback at last */
+		if (mci && mci != fallback)
+			ghes_edac_unregister_one(mci);
+	}
 
+	ghes_edac_unregister_one(fallback);
+	fallback = NULL;
 	kfree(mem_info.dimms);
+	atomic_dec(&ghes_init);
 }
 
 int ghes_edac_register(struct ghes *ghes, struct device *dev)
 {
 	bool fake = false;
 	int rc;
+	int nid;
 	int idx = -1;
 
 	if (IS_ENABLED(CONFIG_X86)) {
@@ -743,7 +809,23 @@ int ghes_edac_register(struct ghes *ghes, struct device *dev)
 		pr_info("This system has %d DIMM sockets.\n", mem_info.num_dimm);
 	}
 
-	rc = ghes_edac_register_one(0, ghes, dev);
+	for_each_node(nid) {
+		if (!mem_info.num_per_node[nid])
+			continue;
 
-	return rc;
+		rc = ghes_edac_register_one(nid, ghes, dev);
+		if (rc) {
+			ghes_edac_unregister(ghes);
+			return rc;
+		}
+
+		/*
+		 * use the first node's mc as fallback in case we can
+		 * not detect the node from the error information
+		 */
+		if (!fallback)
+			fallback = edac_mc_find(nid);
+	}
+
+	return 0;
 }
-- 
2.20.1