Re: [PATCH v7 2/3] mm/memory_hotplug: split memmap_on_memory requests across memblocks

[David Hildenbrand]

On 26.10.23 00:44, Vishal Verma wrote:
> The MHP_MEMMAP_ON_MEMORY flag for hotplugged memory is restricted to
> 'memblock_size' chunks of memory being added. Adding a larger span of
> memory precludes memmap_on_memory semantics.
>
> For users of hotplug such as kmem, large amounts of memory might get
> added from the CXL subsystem. In some cases, this amount may exceed the
> available 'main memory' to store the memmap for the memory being added.
> In this case, it is useful to have a way to place the memmap on the
> memory being added, even if it means splitting the addition into
> memblock-sized chunks.
>
> Change add_memory_resource() to loop over memblock-sized chunks of
> memory if caller requested memmap_on_memory, and if other conditions for
> it are met. Teach try_remove_memory() to also expect that a memory
> range being removed might have been split up into memblock sized chunks,
> and to loop through those as needed.
>
> This does preclude being able to use PUD mappings in the direct map; a
> proposal to how this could be optimized in the future is laid out
> here[1].


Almost there, I think :)

>   
> +static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,
> +					    u64 start, u64 size)
> +{
> +	unsigned long memblock_size = memory_block_size_bytes();
> +	u64 cur_start;
> +	int ret;
> +
> +	for (cur_start = start; cur_start < start + size;
> +	     cur_start += memblock_size) {
> +		struct mhp_params params = { .pgprot =
> +						     pgprot_mhp(PAGE_KERNEL) };
> +		struct vmem_altmap mhp_altmap = {
> +			.base_pfn = PHYS_PFN(cur_start),
> +			.end_pfn = PHYS_PFN(cur_start + memblock_size - 1),
> +		};
> +
> +		mhp_altmap.free = memory_block_memmap_on_memory_pages();
> +		params.altmap = kmemdup(&mhp_altmap, sizeof(struct vmem_altmap),
> +					GFP_KERNEL);
> +		if (!params.altmap)
> +			return -ENOMEM;

Best to cleanup here instead of handling it in the caller [as noted by 
Vishal, we might not be doing that yet]. Using 
remove_memory_blocks_and_altmaps() on the fully processed range sounds 
reasonable.

maybe something like

ret = arch_add_memory(nid, cur_start, memblock_size, &params);
if (ret) {
	kfree(params.altmap);
	goto out;
}

ret = create_memory_block_devices(cur_start, memblock_size,
				   params.altmap, group);
if (ret) {
	arch_remove_memory(cur_start, memblock_size, NULL);
	kfree(params.altmap);
	goto out;
}

if (ret && cur_start != start)
	remove_memory_blocks_and_altmaps(start, cur_start - start);
return ret;

> +
> +		/* call arch's memory hotadd */
> +		ret = arch_add_memory(nid, cur_start, memblock_size, &params);
> +		if (ret < 0) {
> +			kfree(params.altmap);
> +			return ret;
> +		}
> +
> +		/* create memory block devices after memory was added */
> +		ret = create_memory_block_devices(cur_start, memblock_size,
> +						  params.altmap, group);
> +		if (ret) {
> +			arch_remove_memory(cur_start, memblock_size, NULL);
> +			kfree(params.altmap);
> +			return ret;
> +		}
> +	}
> +
> +	return 0;
> +}
> +

[...]

>   static int check_cpu_on_node(int nid)
>   {
>   	int cpu;
> @@ -2146,11 +2186,69 @@ void try_offline_node(int nid)
>   }
>   EXPORT_SYMBOL(try_offline_node);
>   
> -static int __ref try_remove_memory(u64 start, u64 size)
> +static void __ref remove_memory_blocks_and_altmaps(u64 start, u64 size)
>   {
> -	struct memory_block *mem;
> -	int rc = 0, nid = NUMA_NO_NODE;
> +	unsigned long memblock_size = memory_block_size_bytes();
>   	struct vmem_altmap *altmap = NULL;
> +	struct memory_block *mem;
> +	u64 cur_start;
> +	int rc;
> +
> +	/*
> +	 * For memmap_on_memory, the altmaps could have been added on
> +	 * a per-memblock basis. Loop through the entire range if so,
> +	 * and remove each memblock and its altmap.
> +	 */

/*
 * altmaps where added on a per-memblock basis; we have to process
 * each individual memory block.
 */

> +	for (cur_start = start; cur_start < start + size;
> +	     cur_start += memblock_size) {
> +		rc = walk_memory_blocks(cur_start, memblock_size, &mem,
> +					test_has_altmap_cb);
> +		if (rc) {
> +			altmap = mem->altmap;
> +			/*
> +			 * Mark altmap NULL so that we can add a debug
> +			 * check on memblock free.
> +			 */
> +			mem->altmap = NULL;
> +		}

Simpler (especially, we know that there must be an altmap):

mem = find_memory_block(pfn_to_section_nr(cur_start));
altmap = mem->altmap;
mem->altmap = NULL;

I think we might be able to remove test_has_altmap_cb() then.

> +
> +		remove_memory_block_devices(cur_start, memblock_size);
> +
> +		arch_remove_memory(cur_start, memblock_size, altmap);
> +
> +		/* Verify that all vmemmap pages have actually been freed. */
> +		if (altmap) {

There must be an altmap, so this can be done unconditionally.

> +			WARN(altmap->alloc, "Altmap not fully unmapped");
> +			kfree(altmap);
> +		}
> +	}
> +}


--
Cheers,

David / dhildenb