RE: [PATCH 06/12] percpu: set PCPU_BITMAP_BLOCK_SIZE to PAGE_SIZE

[Peng Fan]

> -----Original Message-----
> From: owner-linux-mm@xxxxxxxxx [mailto:owner-linux-mm@xxxxxxxxx] On
> Behalf Of Dennis Zhou
> Sent: 2019年2月28日 10:19
> To: Dennis Zhou <dennis@xxxxxxxxxx>; Tejun Heo <tj@xxxxxxxxxx>; Christoph
> Lameter <cl@xxxxxxxxx>
> Cc: Vlad Buslov <vladbu@xxxxxxxxxxxx>; kernel-team@xxxxxx;
> linux-mm@xxxxxxxxx; linux-kernel@xxxxxxxxxxxxxxx
> Subject: [PATCH 06/12] percpu: set PCPU_BITMAP_BLOCK_SIZE to PAGE_SIZE
> 
> Previously, block size was flexible based on the constraint that the
> GCD(PCPU_BITMAP_BLOCK_SIZE, PAGE_SIZE) > 1. However, this carried the
> overhead that keeping a floating number of populated free pages required
> scanning over the free regions of a chunk.
> 
> Setting the block size to be fixed at PAGE_SIZE lets us know when an empty
> page becomes used as we will break a full contig_hint of a block.
> This means we no longer have to scan the whole chunk upon breaking a
> contig_hint which empty page management piggybacks off. A later patch
> takes advantage of this to optimize the allocation path by only scanning
> forward using the scan_hint introduced later too.
> 
> Signed-off-by: Dennis Zhou <dennis@xxxxxxxxxx>
> ---
>  include/linux/percpu.h |  12 ++---
>  mm/percpu-km.c         |   2 +-
>  mm/percpu.c            | 111 +++++++++++++++++------------------------
>  3 files changed, 49 insertions(+), 76 deletions(-)
> 
> diff --git a/include/linux/percpu.h b/include/linux/percpu.h index
> 70b7123f38c7..9909dc0e273a 100644
> --- a/include/linux/percpu.h
> +++ b/include/linux/percpu.h
> @@ -26,16 +26,10 @@
>  #define PCPU_MIN_ALLOC_SHIFT		2
>  #define PCPU_MIN_ALLOC_SIZE		(1 << PCPU_MIN_ALLOC_SHIFT)
> 
> -/* number of bits per page, used to trigger a scan if blocks are > PAGE_SIZE
> */
> -#define PCPU_BITS_PER_PAGE		(PAGE_SIZE >>
> PCPU_MIN_ALLOC_SHIFT)
> -
>  /*
> - * This determines the size of each metadata block.  There are several
> subtle
> - * constraints around this constant.  The reserved region must be a multiple
> of
> - * PCPU_BITMAP_BLOCK_SIZE.  Additionally, PCPU_BITMAP_BLOCK_SIZE
> must be a
> - * multiple of PAGE_SIZE or PAGE_SIZE must be a multiple of
> - * PCPU_BITMAP_BLOCK_SIZE to align with the populated page map. The
> unit_size
> - * also has to be a multiple of PCPU_BITMAP_BLOCK_SIZE to ensure full
> blocks.
> + * The PCPU_BITMAP_BLOCK_SIZE must be the same size as PAGE_SIZE as
> the
> + * updating of hints is used to manage the nr_empty_pop_pages in both
> + * the chunk and globally.
>   */
>  #define PCPU_BITMAP_BLOCK_SIZE		PAGE_SIZE
>  #define PCPU_BITMAP_BLOCK_BITS		(PCPU_BITMAP_BLOCK_SIZE >>
> 	\
> diff --git a/mm/percpu-km.c b/mm/percpu-km.c index
> 0f643dc2dc65..c10bf7466596 100644
> --- a/mm/percpu-km.c
> +++ b/mm/percpu-km.c
> @@ -70,7 +70,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t
> gfp)
>  	chunk->base_addr = page_address(pages) - pcpu_group_offsets[0];
> 
>  	spin_lock_irqsave(&pcpu_lock, flags);
> -	pcpu_chunk_populated(chunk, 0, nr_pages, false);
> +	pcpu_chunk_populated(chunk, 0, nr_pages);
>  	spin_unlock_irqrestore(&pcpu_lock, flags);
> 
>  	pcpu_stats_chunk_alloc();
> diff --git a/mm/percpu.c b/mm/percpu.c
> index 3d7deece9556..967c9cc3a928 100644
> --- a/mm/percpu.c
> +++ b/mm/percpu.c
> @@ -527,37 +527,21 @@ static void pcpu_chunk_relocate(struct
> pcpu_chunk *chunk, int oslot)
>  		__pcpu_chunk_move(chunk, nslot, oslot < nslot);  }
> 
> -/**
> - * pcpu_cnt_pop_pages- counts populated backing pages in range
> +/*
> + * pcpu_update_empty_pages - update empty page counters
>   * @chunk: chunk of interest
> - * @bit_off: start offset
> - * @bits: size of area to check
> + * @nr: nr of empty pages
>   *
> - * Calculates the number of populated pages in the region
> - * [page_start, page_end).  This keeps track of how many empty populated
> - * pages are available and decide if async work should be scheduled.
> - *
> - * RETURNS:
> - * The nr of populated pages.
> + * This is used to keep track of the empty pages now based on the
> + premise
> + * a pcpu_block_md covers a page.  The hint update functions recognize
> + if
> + * a block is made full or broken to calculate deltas for keeping track
> + of
> + * free pages.
>   */
> -static inline int pcpu_cnt_pop_pages(struct pcpu_chunk *chunk, int bit_off,
> -				     int bits)
> +static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk,
> +int nr)
>  {
> -	int page_start = PFN_UP(bit_off * PCPU_MIN_ALLOC_SIZE);
> -	int page_end = PFN_DOWN((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);
> -
> -	if (page_start >= page_end)
> -		return 0;
> -
> -	/*
> -	 * bitmap_weight counts the number of bits set in a bitmap up to
> -	 * the specified number of bits.  This is counting the populated
> -	 * pages up to page_end and then subtracting the populated pages
> -	 * up to page_start to count the populated pages in
> -	 * [page_start, page_end).
> -	 */
> -	return bitmap_weight(chunk->populated, page_end) -
> -	       bitmap_weight(chunk->populated, page_start);
> +	chunk->nr_empty_pop_pages += nr;
> +	if (chunk != pcpu_reserved_chunk)
> +		pcpu_nr_empty_pop_pages += nr;
>  }
> 
>  /*
> @@ -611,36 +595,19 @@ static void pcpu_chunk_update(struct pcpu_chunk
> *chunk, int bit_off, int bits)
>   * Updates:
>   *      chunk->contig_bits
>   *      chunk->contig_bits_start
> - *      nr_empty_pop_pages (chunk and global)
>   */
>  static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk)  {
> -	int bit_off, bits, nr_empty_pop_pages;
> +	int bit_off, bits;
> 
>  	/* clear metadata */
>  	chunk->contig_bits = 0;
> 
>  	bit_off = chunk->first_bit;
> -	bits = nr_empty_pop_pages = 0;
> +	bits = 0;
>  	pcpu_for_each_md_free_region(chunk, bit_off, bits) {
>  		pcpu_chunk_update(chunk, bit_off, bits);
> -
> -		nr_empty_pop_pages += pcpu_cnt_pop_pages(chunk, bit_off, bits);
>  	}
> -
> -	/*
> -	 * Keep track of nr_empty_pop_pages.
> -	 *
> -	 * The chunk maintains the previous number of free pages it held,
> -	 * so the delta is used to update the global counter.  The reserved
> -	 * chunk is not part of the free page count as they are populated
> -	 * at init and are special to serving reserved allocations.
> -	 */
> -	if (chunk != pcpu_reserved_chunk)
> -		pcpu_nr_empty_pop_pages +=
> -			(nr_empty_pop_pages - chunk->nr_empty_pop_pages);
> -
> -	chunk->nr_empty_pop_pages = nr_empty_pop_pages;
>  }
> 
>  /**
> @@ -712,6 +679,7 @@ static void pcpu_block_refresh_hint(struct
> pcpu_chunk *chunk, int index)  static void
> pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
>  					 int bits)
>  {
> +	int nr_empty_pages = 0;
>  	struct pcpu_block_md *s_block, *e_block, *block;
>  	int s_index, e_index;	/* block indexes of the freed allocation */
>  	int s_off, e_off;	/* block offsets of the freed allocation */
> @@ -736,6 +704,9 @@ static void pcpu_block_update_hint_alloc(struct
> pcpu_chunk *chunk, int bit_off,
>  	 * If the allocation breaks the contig_hint, a scan is required to
>  	 * restore this hint.
>  	 */
> +	if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
> +		nr_empty_pages++;
> +
>  	if (s_off == s_block->first_free)
>  		s_block->first_free = find_next_zero_bit(
>  					pcpu_index_alloc_map(chunk, s_index), @@ -763,6
> +734,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk
> *chunk, int bit_off,
>  	 * Update e_block.
>  	 */
>  	if (s_index != e_index) {
> +		if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
> +			nr_empty_pages++;
> +
>  		/*
>  		 * When the allocation is across blocks, the end is along
>  		 * the left part of the e_block.
> @@ -787,6 +761,7 @@ static void pcpu_block_update_hint_alloc(struct
> pcpu_chunk *chunk, int bit_off,
>  		}
> 
>  		/* update in-between md_blocks */
> +		nr_empty_pages += (e_index - s_index - 1);
>  		for (block = s_block + 1; block < e_block; block++) {
>  			block->contig_hint = 0;
>  			block->left_free = 0;
> @@ -794,6 +769,9 @@ static void pcpu_block_update_hint_alloc(struct
> pcpu_chunk *chunk, int bit_off,
>  		}
>  	}
> 
> +	if (nr_empty_pages)
> +		pcpu_update_empty_pages(chunk, -1 * nr_empty_pages);
> +
>  	/*
>  	 * The only time a full chunk scan is required is if the chunk
>  	 * contig hint is broken.  Otherwise, it means a smaller space @@
> -826,6 +804,7 @@ static void pcpu_block_update_hint_alloc(struct
> pcpu_chunk *chunk, int bit_off,  static void
> pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
>  					int bits)
>  {
> +	int nr_empty_pages = 0;
>  	struct pcpu_block_md *s_block, *e_block, *block;
>  	int s_index, e_index;	/* block indexes of the freed allocation */
>  	int s_off, e_off;	/* block offsets of the freed allocation */
> @@ -879,14 +858,19 @@ static void pcpu_block_update_hint_free(struct
> pcpu_chunk *chunk, int bit_off,
> 
>  	/* update s_block */
>  	e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS;
> +	if (!start && e_off == PCPU_BITMAP_BLOCK_BITS)
> +		nr_empty_pages++;
>  	pcpu_block_update(s_block, start, e_off);
> 
>  	/* freeing in the same block */
>  	if (s_index != e_index) {
>  		/* update e_block */
> +		if (end == PCPU_BITMAP_BLOCK_BITS)
> +			nr_empty_pages++;
>  		pcpu_block_update(e_block, 0, end);
> 
>  		/* reset md_blocks in the middle */
> +		nr_empty_pages += (e_index - s_index - 1);
>  		for (block = s_block + 1; block < e_block; block++) {
>  			block->first_free = 0;
>  			block->contig_hint_start = 0;
> @@ -896,15 +880,16 @@ static void pcpu_block_update_hint_free(struct
> pcpu_chunk *chunk, int bit_off,
>  		}
>  	}
> 
> +	if (nr_empty_pages)
> +		pcpu_update_empty_pages(chunk, nr_empty_pages);
> +
>  	/*
> -	 * Refresh chunk metadata when the free makes a page free, a block
> -	 * free, or spans across blocks.  The contig hint may be off by up to
> -	 * a page, but if the hint is contained in a block, it will be accurate
> -	 * with the else condition below.
> +	 * Refresh chunk metadata when the free makes a block free or spans
> +	 * across blocks.  The contig_hint may be off by up to a page, but if
> +	 * the contig_hint is contained in a block, it will be accurate with
> +	 * the else condition below.
>  	 */
> -	if ((ALIGN_DOWN(end, min(PCPU_BITS_PER_PAGE,
> PCPU_BITMAP_BLOCK_BITS)) >
> -	     ALIGN(start, min(PCPU_BITS_PER_PAGE,
> PCPU_BITMAP_BLOCK_BITS))) ||
> -	    s_index != e_index)
> +	if (((end - start) >= PCPU_BITMAP_BLOCK_BITS) || s_index != e_index)
>  		pcpu_chunk_refresh_hint(chunk);
>  	else
>  		pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start),
> @@ -1164,9 +1149,7 @@ static struct pcpu_chunk * __init
> pcpu_alloc_first_chunk(unsigned long tmp_addr,
>  	chunk->immutable = true;
>  	bitmap_fill(chunk->populated, chunk->nr_pages);
>  	chunk->nr_populated = chunk->nr_pages;
> -	chunk->nr_empty_pop_pages =
> -		pcpu_cnt_pop_pages(chunk, start_offset / PCPU_MIN_ALLOC_SIZE,
> -				   map_size / PCPU_MIN_ALLOC_SIZE);
> +	chunk->nr_empty_pop_pages = chunk->nr_pages;
> 
>  	chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE;
>  	chunk->free_bytes = map_size;
> @@ -1261,7 +1244,6 @@ static void pcpu_free_chunk(struct pcpu_chunk
> *chunk)
>   * @chunk: pcpu_chunk which got populated
>   * @page_start: the start page
>   * @page_end: the end page
> - * @for_alloc: if this is to populate for allocation
>   *
>   * Pages in [@page_start,@page_end) have been populated to @chunk.
> Update
>   * the bookkeeping information accordingly.  Must be called after each
> @@ -1271,7 +1253,7 @@ static void pcpu_free_chunk(struct pcpu_chunk
> *chunk)
>   * is to serve an allocation in that area.
>   */
>  static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,
> -				 int page_end, bool for_alloc)
> +				 int page_end)
>  {
>  	int nr = page_end - page_start;
> 
> @@ -1281,10 +1263,7 @@ static void pcpu_chunk_populated(struct
> pcpu_chunk *chunk, int page_start,
>  	chunk->nr_populated += nr;
>  	pcpu_nr_populated += nr;
> 
> -	if (!for_alloc) {
> -		chunk->nr_empty_pop_pages += nr;
> -		pcpu_nr_empty_pop_pages += nr;
> -	}
> +	pcpu_update_empty_pages(chunk, nr);
>  }
> 
>  /**
> @@ -1306,9 +1285,9 @@ static void pcpu_chunk_depopulated(struct
> pcpu_chunk *chunk,
> 
>  	bitmap_clear(chunk->populated, page_start, nr);
>  	chunk->nr_populated -= nr;
> -	chunk->nr_empty_pop_pages -= nr;
> -	pcpu_nr_empty_pop_pages -= nr;
>  	pcpu_nr_populated -= nr;
> +
> +	pcpu_update_empty_pages(chunk, -1 * nr);
>  }
> 
>  /*
> @@ -1523,7 +1502,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t
> align, bool reserved,
>  				err = "failed to populate";
>  				goto fail_unlock;
>  			}
> -			pcpu_chunk_populated(chunk, rs, re, true);
> +			pcpu_chunk_populated(chunk, rs, re);
>  			spin_unlock_irqrestore(&pcpu_lock, flags);
>  		}
> 
> @@ -1722,7 +1701,7 @@ static void pcpu_balance_workfn(struct
> work_struct *work)
>  			if (!ret) {
>  				nr_to_pop -= nr;
>  				spin_lock_irq(&pcpu_lock);
> -				pcpu_chunk_populated(chunk, rs, rs + nr, false);
> +				pcpu_chunk_populated(chunk, rs, rs + nr);
>  				spin_unlock_irq(&pcpu_lock);
>  			} else {
>  				nr_to_pop = 0;
> --

Reviewed-by: Peng Fan <peng.fan@xxxxxxx>

> 2.17.1