Re: [PATCH v2] mm: make folio_pte_batch available outside of mm/memory.c

[Ryan Roberts]

On 27/02/2024 10:42, Barry Song wrote:
> From: Barry Song <v-songbaohua@xxxxxxxx>
> 
> madvise, mprotect and some others might need folio_pte_batch to check if
> a range of PTEs are completely mapped to a large folio with contiguous
> physical addresses. Let's make it available in mm/internal.h.
> 
> Suggested-by: David Hildenbrand <david@xxxxxxxxxx>
> Cc: Lance Yang <ioworker0@xxxxxxxxx>
> Cc: Ryan Roberts <ryan.roberts@xxxxxxx>
> Cc: Yin Fengwei <fengwei.yin@xxxxxxxxx>
> [david@xxxxxxxxxx: improve the doc for the exported func]
> Signed-off-by: David Hildenbrand <david@xxxxxxxxxx>
> Signed-off-by: Barry Song <v-songbaohua@xxxxxxxx>

Reviewed-by: Ryan Roberts <ryan.roberts@xxxxxxx>

> ---
>  -v2:
>  * inline folio_pte_batch according to Ryan and David;
>  * improve the doc, thanks to David's work on this;
>  * fix tags of David and add David's s-o-b;
>  -v1:
>  https://lore.kernel.org/all/20240227024050.244567-1-21cnbao@xxxxxxxxx/
> 
>  mm/internal.h | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++
>  mm/memory.c   | 76 -------------------------------------------
>  2 files changed, 90 insertions(+), 76 deletions(-)
> 
> diff --git a/mm/internal.h b/mm/internal.h
> index 13b59d384845..fa9e2f7db506 100644
> --- a/mm/internal.h
> +++ b/mm/internal.h
> @@ -83,6 +83,96 @@ static inline void *folio_raw_mapping(struct folio *folio)
>  	return (void *)(mapping & ~PAGE_MAPPING_FLAGS);
>  }
>  
> +/* Flags for folio_pte_batch(). */
> +typedef int __bitwise fpb_t;
> +
> +/* Compare PTEs after pte_mkclean(), ignoring the dirty bit. */
> +#define FPB_IGNORE_DIRTY		((__force fpb_t)BIT(0))
> +
> +/* Compare PTEs after pte_clear_soft_dirty(), ignoring the soft-dirty bit. */
> +#define FPB_IGNORE_SOFT_DIRTY		((__force fpb_t)BIT(1))
> +
> +static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
> +{
> +	if (flags & FPB_IGNORE_DIRTY)
> +		pte = pte_mkclean(pte);
> +	if (likely(flags & FPB_IGNORE_SOFT_DIRTY))
> +		pte = pte_clear_soft_dirty(pte);
> +	return pte_wrprotect(pte_mkold(pte));
> +}
> +
> +/**
> + * folio_pte_batch - detect a PTE batch for a large folio
> + * @folio: The large folio to detect a PTE batch for.
> + * @addr: The user virtual address the first page is mapped at.
> + * @start_ptep: Page table pointer for the first entry.
> + * @pte: Page table entry for the first page.
> + * @max_nr: The maximum number of table entries to consider.
> + * @flags: Flags to modify the PTE batch semantics.
> + * @any_writable: Optional pointer to indicate whether any entry except the
> + *		  first one is writable.
> + *
> + * Detect a PTE batch: consecutive (present) PTEs that map consecutive
> + * pages of the same large folio.
> + *
> + * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,
> + * the accessed bit, writable bit, dirty bit (with FPB_IGNORE_DIRTY) and
> + * soft-dirty bit (with FPB_IGNORE_SOFT_DIRTY).
> + *
> + * start_ptep must map any page of the folio. max_nr must be at least one and
> + * must be limited by the caller so scanning cannot exceed a single page table.
> + *
> + * Return: the number of table entries in the batch.
> + */
> +static inline int folio_pte_batch(struct folio *folio, unsigned long addr,
> +		pte_t *start_ptep, pte_t pte, int max_nr, fpb_t flags,
> +		bool *any_writable)
> +{
> +	unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio);
> +	const pte_t *end_ptep = start_ptep + max_nr;
> +	pte_t expected_pte, *ptep;
> +	bool writable;
> +	int nr;
> +
> +	if (any_writable)
> +		*any_writable = false;
> +
> +	VM_WARN_ON_FOLIO(!pte_present(pte), folio);
> +	VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio);
> +	VM_WARN_ON_FOLIO(page_folio(pfn_to_page(pte_pfn(pte))) != folio, folio);
> +
> +	nr = pte_batch_hint(start_ptep, pte);
> +	expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags);
> +	ptep = start_ptep + nr;
> +
> +	while (ptep < end_ptep) {
> +		pte = ptep_get(ptep);
> +		if (any_writable)
> +			writable = !!pte_write(pte);
> +		pte = __pte_batch_clear_ignored(pte, flags);
> +
> +		if (!pte_same(pte, expected_pte))
> +			break;
> +
> +		/*
> +		 * Stop immediately once we reached the end of the folio. In
> +		 * corner cases the next PFN might fall into a different
> +		 * folio.
> +		 */
> +		if (pte_pfn(pte) >= folio_end_pfn)
> +			break;
> +
> +		if (any_writable)
> +			*any_writable |= writable;
> +
> +		nr = pte_batch_hint(ptep, pte);
> +		expected_pte = pte_advance_pfn(expected_pte, nr);
> +		ptep += nr;
> +	}
> +
> +	return min(ptep - start_ptep, max_nr);
> +}
> +
>  void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
>  						int nr_throttled);
>  static inline void acct_reclaim_writeback(struct folio *folio)
> diff --git a/mm/memory.c b/mm/memory.c
> index 1c45b6a42a1b..a7bcc39de56b 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -953,82 +953,6 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
>  	set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
>  }
>  
> -/* Flags for folio_pte_batch(). */
> -typedef int __bitwise fpb_t;
> -
> -/* Compare PTEs after pte_mkclean(), ignoring the dirty bit. */
> -#define FPB_IGNORE_DIRTY		((__force fpb_t)BIT(0))
> -
> -/* Compare PTEs after pte_clear_soft_dirty(), ignoring the soft-dirty bit. */
> -#define FPB_IGNORE_SOFT_DIRTY		((__force fpb_t)BIT(1))
> -
> -static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
> -{
> -	if (flags & FPB_IGNORE_DIRTY)
> -		pte = pte_mkclean(pte);
> -	if (likely(flags & FPB_IGNORE_SOFT_DIRTY))
> -		pte = pte_clear_soft_dirty(pte);
> -	return pte_wrprotect(pte_mkold(pte));
> -}
> -
> -/*
> - * Detect a PTE batch: consecutive (present) PTEs that map consecutive
> - * pages of the same folio.
> - *
> - * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,
> - * the accessed bit, writable bit, dirty bit (with FPB_IGNORE_DIRTY) and
> - * soft-dirty bit (with FPB_IGNORE_SOFT_DIRTY).
> - *
> - * If "any_writable" is set, it will indicate if any other PTE besides the
> - * first (given) PTE is writable.
> - */
> -static inline int folio_pte_batch(struct folio *folio, unsigned long addr,
> -		pte_t *start_ptep, pte_t pte, int max_nr, fpb_t flags,
> -		bool *any_writable)
> -{
> -	unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio);
> -	const pte_t *end_ptep = start_ptep + max_nr;
> -	pte_t expected_pte, *ptep;
> -	bool writable;
> -	int nr;
> -
> -	if (any_writable)
> -		*any_writable = false;
> -
> -	VM_WARN_ON_FOLIO(!pte_present(pte), folio);
> -
> -	nr = pte_batch_hint(start_ptep, pte);
> -	expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags);
> -	ptep = start_ptep + nr;
> -
> -	while (ptep < end_ptep) {
> -		pte = ptep_get(ptep);
> -		if (any_writable)
> -			writable = !!pte_write(pte);
> -		pte = __pte_batch_clear_ignored(pte, flags);
> -
> -		if (!pte_same(pte, expected_pte))
> -			break;
> -
> -		/*
> -		 * Stop immediately once we reached the end of the folio. In
> -		 * corner cases the next PFN might fall into a different
> -		 * folio.
> -		 */
> -		if (pte_pfn(pte) >= folio_end_pfn)
> -			break;
> -
> -		if (any_writable)
> -			*any_writable |= writable;
> -
> -		nr = pte_batch_hint(ptep, pte);
> -		expected_pte = pte_advance_pfn(expected_pte, nr);
> -		ptep += nr;
> -	}
> -
> -	return min(ptep - start_ptep, max_nr);
> -}
> -
>  /*
>   * Copy one present PTE, trying to batch-process subsequent PTEs that map
>   * consecutive pages of the same folio by copying them as well.