Re: [patch] mm, page_alloc: wakeup kcompactd even if kswapd cannot free more memory

[Vlastimil Babka]

On 03/12/2018 01:00 AM, David Rientjes wrote:
> Kswapd will not wakeup if per-zone watermarks are not failing or if too
> many previous attempts at background reclaim have failed.
> 
> This can be true if there is a lot of free memory available.  For high-
> order allocations, kswapd is responsible for waking up kcompactd for
> background compaction.  If the zone is now below its watermarks or
                                         not ?

> reclaim has recently failed (lots of free memory, nothing left to
> reclaim), kcompactd does not get woken up.
> 
> When __GFP_DIRECT_RECLAIM is not allowed, allow kcompactd to still be
> woken up even if kswapd will not reclaim.  This allows high-order
> allocations, such as thp, to still trigger background compaction even
> when the zone has an abundance of free memory.
> 
> Signed-off-by: David Rientjes <rientjes@xxxxxxxxxx>

Acked-by: Vlastimil Babka <vbabka@xxxxxxx>

> ---
>  .../postprocess/trace-vmscan-postprocess.pl   |  4 +--
>  include/linux/mmzone.h                        |  3 +-
>  include/trace/events/vmscan.h                 | 17 ++++++----
>  mm/page_alloc.c                               | 14 ++++----
>  mm/vmscan.c                                   | 32 +++++++++++++------
>  5 files changed, 45 insertions(+), 25 deletions(-)
> 
> diff --git a/Documentation/trace/postprocess/trace-vmscan-postprocess.pl b/Documentation/trace/postprocess/trace-vmscan-postprocess.pl
> --- a/Documentation/trace/postprocess/trace-vmscan-postprocess.pl
> +++ b/Documentation/trace/postprocess/trace-vmscan-postprocess.pl
> @@ -111,7 +111,7 @@ my $regex_direct_begin_default = 'order=([0-9]*) may_writepage=([0-9]*) gfp_flag
>  my $regex_direct_end_default = 'nr_reclaimed=([0-9]*)';
>  my $regex_kswapd_wake_default = 'nid=([0-9]*) order=([0-9]*)';
>  my $regex_kswapd_sleep_default = 'nid=([0-9]*)';
> -my $regex_wakeup_kswapd_default = 'nid=([0-9]*) zid=([0-9]*) order=([0-9]*)';
> +my $regex_wakeup_kswapd_default = 'nid=([0-9]*) zid=([0-9]*) order=([0-9]*) gfp_flags=([A-Z_|]*)';
>  my $regex_lru_isolate_default = 'isolate_mode=([0-9]*) classzone_idx=([0-9]*) order=([0-9]*) nr_requested=([0-9]*) nr_scanned=([0-9]*) nr_skipped=([0-9]*) nr_taken=([0-9]*) lru=([a-z_]*)';
>  my $regex_lru_shrink_inactive_default = 'nid=([0-9]*) nr_scanned=([0-9]*) nr_reclaimed=([0-9]*) nr_dirty=([0-9]*) nr_writeback=([0-9]*) nr_congested=([0-9]*) nr_immediate=([0-9]*) nr_activate=([0-9]*) nr_ref_keep=([0-9]*) nr_unmap_fail=([0-9]*) priority=([0-9]*) flags=([A-Z_|]*)';
>  my $regex_lru_shrink_active_default = 'lru=([A-Z_]*) nr_scanned=([0-9]*) nr_rotated=([0-9]*) priority=([0-9]*)';
> @@ -201,7 +201,7 @@ $regex_kswapd_sleep = generate_traceevent_regex(
>  $regex_wakeup_kswapd = generate_traceevent_regex(
>  			"vmscan/mm_vmscan_wakeup_kswapd",
>  			$regex_wakeup_kswapd_default,
> -			"nid", "zid", "order");
> +			"nid", "zid", "order", "gfp_flags");
>  $regex_lru_isolate = generate_traceevent_regex(
>  			"vmscan/mm_vmscan_lru_isolate",
>  			$regex_lru_isolate_default,
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -775,7 +775,8 @@ static inline bool is_dev_zone(const struct zone *zone)
>  #include <linux/memory_hotplug.h>
>  
>  void build_all_zonelists(pg_data_t *pgdat);
> -void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx);
> +void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order,
> +		   enum zone_type classzone_idx);
>  bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
>  			 int classzone_idx, unsigned int alloc_flags,
>  			 long free_pages);
> diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h
> --- a/include/trace/events/vmscan.h
> +++ b/include/trace/events/vmscan.h
> @@ -78,26 +78,29 @@ TRACE_EVENT(mm_vmscan_kswapd_wake,
>  
>  TRACE_EVENT(mm_vmscan_wakeup_kswapd,
>  
> -	TP_PROTO(int nid, int zid, int order),
> +	TP_PROTO(int nid, int zid, int order, gfp_t gfp_flags),
>  
> -	TP_ARGS(nid, zid, order),
> +	TP_ARGS(nid, zid, order, gfp_flags),
>  
>  	TP_STRUCT__entry(
> -		__field(	int,		nid	)
> -		__field(	int,		zid	)
> -		__field(	int,		order	)
> +		__field(	int,	nid		)
> +		__field(	int,	zid		)
> +		__field(	int,	order		)
> +		__field(	gfp_t,	gfp_flags	)
>  	),
>  
>  	TP_fast_assign(
>  		__entry->nid		= nid;
>  		__entry->zid		= zid;
>  		__entry->order		= order;
> +		__entry->gfp_flags	= gfp_flags;
>  	),
>  
> -	TP_printk("nid=%d zid=%d order=%d",
> +	TP_printk("nid=%d zid=%d order=%d gfp_flags=%s",
>  		__entry->nid,
>  		__entry->zid,
> -		__entry->order)
> +		__entry->order,
> +		show_gfp_flags(__entry->gfp_flags))
>  );
>  
>  DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_begin_template,
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -3683,16 +3683,18 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
>  	return page;
>  }
>  
> -static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
> +static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
> +			     const struct alloc_context *ac)
>  {
>  	struct zoneref *z;
>  	struct zone *zone;
>  	pg_data_t *last_pgdat = NULL;
> +	enum zone_type high_zoneidx = ac->high_zoneidx;
>  
> -	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
> -					ac->high_zoneidx, ac->nodemask) {
> +	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx,
> +					ac->nodemask) {
>  		if (last_pgdat != zone->zone_pgdat)
> -			wakeup_kswapd(zone, order, ac->high_zoneidx);
> +			wakeup_kswapd(zone, gfp_mask, order, high_zoneidx);
>  		last_pgdat = zone->zone_pgdat;
>  	}
>  }
> @@ -3971,7 +3973,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
>  		goto nopage;
>  
>  	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
> -		wake_all_kswapds(order, ac);
> +		wake_all_kswapds(order, gfp_mask, ac);
>  
>  	/*
>  	 * The adjusted alloc_flags might result in immediate success, so try
> @@ -4029,7 +4031,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
>  retry:
>  	/* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
>  	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
> -		wake_all_kswapds(order, ac);
> +		wake_all_kswapds(order, gfp_mask, ac);
>  
>  	reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
>  	if (reserve_flags)
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -3546,16 +3546,21 @@ static int kswapd(void *p)
>  }
>  
>  /*
> - * A zone is low on free memory, so wake its kswapd task to service it.
> + * A zone is low on free memory or too fragmented for high-order memory.  If
> + * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
> + * pgdat.  It will wake up kcompactd after reclaiming memory.  If kswapd reclaim
> + * has failed or is not needed, still wake up kcompactd if only compaction is
> + * needed.
>   */
> -void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
> +void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
> +		   enum zone_type classzone_idx)
>  {
>  	pg_data_t *pgdat;
>  
>  	if (!managed_zone(zone))
>  		return;
>  
> -	if (!cpuset_zone_allowed(zone, GFP_KERNEL | __GFP_HARDWALL))
> +	if (!cpuset_zone_allowed(zone, gfp_flags))
>  		return;
>  	pgdat = zone->zone_pgdat;
>  	pgdat->kswapd_classzone_idx = kswapd_classzone_idx(pgdat,
> @@ -3564,14 +3569,23 @@ void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
>  	if (!waitqueue_active(&pgdat->kswapd_wait))
>  		return;
>  
> -	/* Hopeless node, leave it to direct reclaim */
> -	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
> -		return;
> -
> -	if (pgdat_balanced(pgdat, order, classzone_idx))
> +	/* Hopeless node, leave it to direct reclaim if possible */
> +	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
> +	    pgdat_balanced(pgdat, order, classzone_idx)) {
> +		/*
> +		 * There may be plenty of free memory available, but it's too
> +		 * fragmented for high-order allocations.  Wake up kcompactd
> +		 * and rely on compaction_suitable() to determine if it's
> +		 * needed.  If it fails, it will defer subsequent attempts to
> +		 * ratelimit its work.
> +		 */
> +		if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
> +			wakeup_kcompactd(pgdat, order, classzone_idx);
>  		return;
> +	}
>  
> -	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, classzone_idx, order);
> +	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, classzone_idx, order,
> +				      gfp_flags);
>  	wake_up_interruptible(&pgdat->kswapd_wait);
>  }
>  
>