Re: [PATCH v11 4/8] mm/demotion/dax/kmem: Set node's abstract distance to MEMTIER_ADISTANCE_PMEM

[Huang, Ying]

"Aneesh Kumar K.V" <aneesh.kumar@xxxxxxxxxxxxx> writes:

> By default, all nodes are assigned to the default memory tier which
> is the memory tier designated for nodes with DRAM
>
> Set dax kmem device node's tier to slower memory tier by assigning
> abstract distance to MEMTIER_ADISTANCE_PMEM. PMEM tier
> appears below the default memory tier in demotion order.
>
> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@xxxxxxxxxxxxx>
> ---
>  drivers/dax/kmem.c           |  9 +++++++++
>  include/linux/memory-tiers.h | 19 ++++++++++++++++++-
>  mm/memory-tiers.c            | 28 ++++++++++++++++------------
>  3 files changed, 43 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
> index a37622060fff..6b0d5de9a3e9 100644
> --- a/drivers/dax/kmem.c
> +++ b/drivers/dax/kmem.c
> @@ -11,6 +11,7 @@
>  #include <linux/fs.h>
>  #include <linux/mm.h>
>  #include <linux/mman.h>
> +#include <linux/memory-tiers.h>
>  #include "dax-private.h"
>  #include "bus.h"
>  
> @@ -41,6 +42,12 @@ struct dax_kmem_data {
>  	struct resource *res[];
>  };
>  
> +static struct memory_dev_type default_pmem_type  = {

Why is this named as default_pmem_type?  We will not change the memory
type of a node usually.

> +	.adistance = MEMTIER_ADISTANCE_PMEM,
> +	.tier_sibiling = LIST_HEAD_INIT(default_pmem_type.tier_sibiling),
> +	.nodes  = NODE_MASK_NONE,
> +};
> +
>  static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
>  {
>  	struct device *dev = &dev_dax->dev;
> @@ -62,6 +69,8 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
>  		return -EINVAL;
>  	}
>  
> +	init_node_memory_type(numa_node, &default_pmem_type);
> +

The memory hot-add below may fail.  So the error handling needs to be
added.

And, it appears that the memory type and memory tier of a node may be
fully initialized here before NUMA hot-adding started.  So I suggest to
set node_memory_types[] here only.  And set memory_dev_type->nodes in
node hot-add callback.  I think there is the proper place to complete
the initialization.

And, in theory dax/kmem.c can be unloaded.  So we need to clear
node_memory_types[] for nodes somewhere.

Best Regards,
Huang, Ying

>  	for (i = 0; i < dev_dax->nr_range; i++) {
>  		struct range range;
>  
> diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h
> index 976f43a5e3be..4f4baf0bf430 100644
> --- a/include/linux/memory-tiers.h
> +++ b/include/linux/memory-tiers.h
> @@ -2,6 +2,8 @@
>  #ifndef _LINUX_MEMORY_TIERS_H
>  #define _LINUX_MEMORY_TIERS_H
>  
> +#include <linux/types.h>
> +#include <linux/nodemask.h>
>  /*
>   * Each tier cover a abstrace distance chunk size of 128
>   */
> @@ -15,12 +17,27 @@
>  #define MEMTIER_ADISTANCE_PMEM	(1 << MEMTIER_CHUNK_BITS)
>  #define MEMTIER_HOTPLUG_PRIO	100
>  
> +struct memory_tier;
> +struct memory_dev_type {
> +	/* list of memory types that are are part of same tier as this type */
> +	struct list_head tier_sibiling;
> +	/* abstract distance for this specific memory type */
> +	int adistance;
> +	/* Nodes of same abstract distance */
> +	nodemask_t nodes;
> +	struct memory_tier *memtier;
> +};
> +
>  #ifdef CONFIG_NUMA
> -#include <linux/types.h>
>  extern bool numa_demotion_enabled;
> +struct memory_dev_type *init_node_memory_type(int node, struct memory_dev_type *default_type);
>  
>  #else
>  
>  #define numa_demotion_enabled	false
> +static inline struct memory_dev_type *init_node_memory_type(int node, struct memory_dev_type *default_type)
> +{
> +	return ERR_PTR(-EINVAL);
> +}
>  #endif	/* CONFIG_NUMA */
>  #endif  /* _LINUX_MEMORY_TIERS_H */
> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
> index c9854a394d9b..109be75fa554 100644
> --- a/mm/memory-tiers.c
> +++ b/mm/memory-tiers.c
> @@ -1,6 +1,4 @@
>  // SPDX-License-Identifier: GPL-2.0
> -#include <linux/types.h>
> -#include <linux/nodemask.h>
>  #include <linux/slab.h>
>  #include <linux/lockdep.h>
>  #include <linux/memory.h>
> @@ -19,16 +17,6 @@ struct memory_tier {
>  	int adistance_start;
>  };
>  
> -struct memory_dev_type {
> -	/* list of memory types that are are part of same tier as this type */
> -	struct list_head tier_sibiling;
> -	/* abstract distance for this specific memory type */
> -	int adistance;
> -	/* Nodes of same abstract distance */
> -	nodemask_t nodes;
> -	struct memory_tier *memtier;
> -};
> -
>  static DEFINE_MUTEX(memory_tier_lock);
>  static LIST_HEAD(memory_tiers);
>  struct memory_dev_type *node_memory_types[MAX_NUMNODES];
> @@ -141,6 +129,22 @@ static void clear_node_memory_tier(int node)
>  	mutex_unlock(&memory_tier_lock);
>  }
>  
> +struct memory_dev_type *init_node_memory_type(int node, struct memory_dev_type *default_type)
> +{
> +	struct memory_dev_type *mem_type;
> +
> +	mutex_lock(&memory_tier_lock);
> +	if (node_memory_types[node]) {
> +		mem_type = node_memory_types[node];
> +	} else {
> +		node_memory_types[node] = default_type;
> +		node_set(node, default_type->nodes);
> +		mem_type = default_type;
> +	}
> +	mutex_unlock(&memory_tier_lock);
> +	return mem_type;
> +}
> +
>  static int __meminit memtier_hotplug_callback(struct notifier_block *self,
>  					      unsigned long action, void *_arg)
>  {