Re: [PATCH v3] mm/page_alloc: use existing highatomic reserves on the buddy fastpath

[Vlastimil Babka (SUSE)]

On 6/23/26 02:46, JP Kobryn wrote:
> ALLOC_HIGHATOMIC currently provides both access to MIGRATE_HIGHATOMIC free
> pages and permission to create new highatomic pageblock reserves. This
> makes it unsuitable for the fastpath.
> 
> However, the fastpath can reach rmqueue_buddy() while MIGRATE_HIGHATOMIC
> reserves have free pages available. In this situation, the allocation can
> fall back to other migratetypes without trying those reserves first.
> 
> Allow high-priority non-blocking allocations to use existing
> MIGRATE_HIGHATOMIC reserves on the buddy fastpath without growing them.
> First tighten the criteria for reserving pageblocks so that growth may only
> occur in the slowpath. Then allow fastpath usage by enabling
> ALLOC_HIGHATOMIC when the GFP mask describes a non-blocking high-priority
> allocation. This logic has been factored out from gfp_to_alloc_flags() to a
> new function gfp_to_alloc_flags_nonblocking().
> 
> A UDP receive workload was run with free MIGRATE_HIGHATOMIC pageblocks
> available in the target zone. Before this patch, the workload did not
> consume these blocks. With this patch, eligible order-1 allocations
> reaching the buddy path consumed existing MIGRATE_HIGHATOMIC pageblocks,
> with no highatomic misses observed. The workload did not grow highatomic
> reserves and NAPI page-frag allocations remained healthy with no failures
> or order-0 fallbacks.
> 
> Signed-off-by: JP Kobryn <jp.kobryn@xxxxxxxxx>

Reviewed-by: Vlastimil Babka (SUSE) <vbabka@xxxxxxxxxx>

> ---
> v3:
>   - remove ALLOC_HIGHATOMIC_RESERVE and let ALLOC_HIGHATOMIC keep original behavior
>   - use ALLOC_WMARK_MIN to identify slowpath before growing reserve
>   - factor out non-blocking logic from gfp_to_alloc_flags() into *_nonblocking() helper
>   - dropped reviewed-by tag
> 
> v2: https://lore.kernel.org/linux-mm/20260617234958.150339-1-jp.kobryn@xxxxxxxxx/
>   - decouple use semantics from ALLOC_HIGHATOMIC_RESERVE
>   - update changelog to reflect above change and reword test paragraph
>   - adjust comment in PCP path
> 
> v1: https://lore.kernel.org/linux-mm/20260616191420.52556-1-jp.kobryn@xxxxxxxxx/
> 
>  mm/page_alloc.c | 44 ++++++++++++++++++++++++++++++--------------
>  1 file changed, 30 insertions(+), 14 deletions(-)
> 
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index f7db8f049bd2..7330f22e3f8f 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -3247,10 +3247,11 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,
>  	} while (check_new_pages(page, order));
>  
>  	/*
> -	 * If this is a high-order atomic allocation then check
> -	 * if the pageblock should be reserved for the future
> +	 * Slowpath (precarious) high-atomic allocations may reserve
> +	 * a pageblock for future use.
>  	 */
> -	if (unlikely(alloc_flags & ALLOC_HIGHATOMIC))
> +	if (unlikely((alloc_flags & ALLOC_HIGHATOMIC) &&
> +			((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN)))
>  		reserve_highatomic_pageblock(page, order, zone);
>  
>  	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
> @@ -4473,6 +4474,29 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
>  	}
>  }
>  
> +static inline unsigned int
> +gfp_to_alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order)
> +{
> +	unsigned int alloc_flags = 0;
> +
> +	if (gfp_mask & __GFP_DIRECT_RECLAIM)
> +		return 0;
> +
> +	/*
> +	 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
> +	 * if it can't schedule.
> +	 */
> +	if (gfp_mask & __GFP_NOMEMALLOC)
> +		return 0;
> +
> +	alloc_flags |= ALLOC_NON_BLOCK;
> +
> +	if (order > 0 && (gfp_mask & __GFP_HIGH))
> +		alloc_flags |= ALLOC_HIGHATOMIC;
> +
> +	return alloc_flags;
> +}
> +
>  static inline unsigned int
>  gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
>  {
> @@ -4495,18 +4519,9 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
>  	alloc_flags |= (__force int)
>  		(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
>  
> -	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
> -		/*
> -		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
> -		 * if it can't schedule.
> -		 */
> -		if (!(gfp_mask & __GFP_NOMEMALLOC)) {
> -			alloc_flags |= ALLOC_NON_BLOCK;
> -
> -			if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE))
> -				alloc_flags |= ALLOC_HIGHATOMIC;
> -		}
> +	alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp_mask, order);
>  
> +	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
>  		/*
>  		 * Ignore cpuset mems for non-blocking __GFP_HIGH (probably
>  		 * GFP_ATOMIC) rather than fail, see the comment for
> @@ -5299,6 +5314,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
>  	 * memory until all local zones are considered.
>  	 */
>  	alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp);
> +	alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC;
>  
>  	/* First allocation attempt */
>  	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);