Re: [PATCH] pseries/drmem: update LMBs after LPM

[Laurent Dufour]

Le 27/04/2021 à 19:01, Tyrel Datwyler a écrit :
> On 4/27/21 8:01 AM, Laurent Dufour wrote:
> > After a LPM, the device tree node ibm,dynamic-reconfiguration-memory may be
> > updated by the hypervisor in the case the NUMA topology of the LPAR's
> > memory is updated.
> >
> > This is caught by the kernel, but the memory's node is updated because
> > there is no way to move a memory block between nodes.
> >
> > If later a memory block is added or removed, drmem_update_dt() is called
> > and it is overwriting the DT node to match the added or removed LMB. But
> > the LMB's associativity node has not been updated after the DT node update
> > and thus the node is overwritten by the Linux's topology instead of the
> > hypervisor one.
> >
> > Introduce a hook called when the ibm,dynamic-reconfiguration-memory node is
> > updated to force an update of the LMB's associativity.
> >
> > Signed-off-by: Laurent Dufour <ldufour@xxxxxxxxxxxxx>
> > ---
> >   arch/powerpc/include/asm/drmem.h          |  1 +
> >   arch/powerpc/mm/drmem.c                   | 48 +++++++++++++++++++++++
> >   arch/powerpc/platforms/pseries/mobility.c |  9 +++++
> >   3 files changed, 58 insertions(+)
> >
> > diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h
> > index bf2402fed3e0..55c2c25085b0 100644
> > --- a/arch/powerpc/include/asm/drmem.h
> > +++ b/arch/powerpc/include/asm/drmem.h
> > @@ -111,6 +111,7 @@ int drmem_update_dt(void);
> >   int __init
> >   walk_drmem_lmbs_early(unsigned long node, void *data,
> >   		      int (*func)(struct drmem_lmb *, const __be32 **, void *));
> > +void drmem_update_lmbs(void);
> >   #endif
> >
> >   static inline void invalidate_lmb_associativity_index(struct drmem_lmb *lmb)
> > diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
> > index 9af3832c9d8d..46074bdfdb3c 100644
> > --- a/arch/powerpc/mm/drmem.c
> > +++ b/arch/powerpc/mm/drmem.c
> > @@ -307,6 +307,54 @@ int __init walk_drmem_lmbs_early(unsigned long node, void *data,
> >   	return ret;
> >   }
> >
> > +/*
> > + * Update the LMB associativity index.
> > + */
> > +static int update_lmb(struct drmem_lmb *updated_lmb,
> > +		      __maybe_unused const __be32 **usm,
> > +		      __maybe_unused void *data)
> > +{
> > +	struct drmem_lmb *lmb;
> > +
> > +	/*
> > +	 * Brut force there may be better way to fetch the LMB
> > +	 */
> > +	for_each_drmem_lmb(lmb) {
> > +		if (lmb->drc_index != updated_lmb->drc_index)
> > +			continue;
> > +
> > +		lmb->aa_index = updated_lmb->aa_index;
> > +		break;
> > +	}
> > +	return 0;
> > +}
> > +
> > +/*
> > + * Update the LMB associativity index.
> > + *
> > + * This needs to be called when the hypervisor is updating the
> > + * dynamic-reconfiguration-memory node property.
> > + */
> > +void drmem_update_lmbs(void)
> > +{
> > +	struct device_node *node;
> > +	const __be32 *prop;
> > +
> > +	node = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
> > +	if (!node)
> > +		return;
> > +
> > +	prop = of_get_property(node, "ibm,dynamic-memory", NULL);
> > +	if (prop) {
> > +		__walk_drmem_v1_lmbs(prop, NULL, NULL, update_lmb);
> > +	} else {
> > +		prop = of_get_property(node, "ibm,dynamic-memory-v2", NULL);
> > +		if (prop)
> > +			__walk_drmem_v2_lmbs(prop, NULL, NULL, update_lmb);
> > +	}
> > +
> > +	of_node_put(node);
> > +}
> >   #endif
> >
> >   static int init_drmem_lmb_size(struct device_node *dn)
> > diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
> > index ea4d6a660e0d..c68eccc6e8df 100644
> > --- a/arch/powerpc/platforms/pseries/mobility.c
> > +++ b/arch/powerpc/platforms/pseries/mobility.c
> > @@ -25,6 +25,7 @@
> >
> >   #include <asm/machdep.h>
> >   #include <asm/rtas.h>
> > +#include <asm/drmem.h>
> >   #include "pseries.h"
> >   #include "../../kernel/cacheinfo.h"
> >
> > @@ -237,6 +238,7 @@ int pseries_devicetree_update(s32 scope)
> >   	__be32 *data;
> >   	int update_nodes_token;
> >   	int rc;
> > +	bool drmem_updated = false;
> >
> >   	update_nodes_token = rtas_token("ibm,update-nodes");
> >   	if (update_nodes_token == RTAS_UNKNOWN_SERVICE)
> > @@ -271,6 +273,10 @@ int pseries_devicetree_update(s32 scope)
> >   					continue;
> >   				}
> >
> > +				if (!strcmp(np->full_name,
> > +					    "ibm,dynamic-reconfiguration-memory"))
> > +					drmem_updated = true;
>
> Is there a reason that we can't use the existing pseries_memory_notifier()
> callback in pseries/hotplug-memory.c to trigger the drmem_update_lmbs() when
> either the ibm,dynamic-memory or ibm,dynamic-memory-v2 properties are updated?

Thanks a lot Tyrel!

That's far more elegant, I'll send a v2 soon.

Laurent.

> Something like:
>
> static int pseries_memory_notifier(struct notifier_block *nb,
>                                     unsigned long action, void *data)
> {
>          struct of_reconfig_data *rd = data;
>          int err = 0;
>
>          switch (action) {
>          case OF_RECONFIG_ATTACH_NODE:
>                  err = pseries_add_mem_node(rd->dn);
>                  break;
>          case OF_RECONFIG_DETACH_NODE:
>                  err = pseries_remove_mem_node(rd->dn);
>                  break;
> 	case OF_RECONFIG_UPDATE_PROPERTY:
> 		if (!strcmp(rd->dn->full_name, "ibm,dynamic-reconfiguration-memory"));
> 			drmem_update_lmbs(rd->prop);
> 		break;
>          }
>          return notifier_from_errno(err);
> }
>
> Your drmem_update_lmbs() would need to be updated to take a property and to
> check the property name against ibm,dyanmic-memory[-v2].
>
> -Tyrel
>
> > +
> >   				switch (action) {
> >   				case DELETE_DT_NODE:
> >   					delete_dt_node(np);
> > @@ -293,6 +299,9 @@ int pseries_devicetree_update(s32 scope)
> >   	} while (rc == 1);
> >
> >   	kfree(rtas_buf);
> > +
> > +	if (drmem_updated)
> > +		drmem_update_lmbs();
> >   	return rc;
> >   }