Re: [PATCH v11 09/21] dax,ext2: Replace the XIP page fault handler with the DAX page fault handler

[Mathieu Desnoyers]

On 25-Sep-2014 04:33:26 PM, Matthew Wilcox wrote:
> Instead of calling aops->get_xip_mem from the fault handler, the
> filesystem passes a get_block_t that is used to find the appropriate
> blocks.
> 
> Signed-off-by: Matthew Wilcox <matthew.r.wilcox@xxxxxxxxx>
> Reviewed-by: Jan Kara <jack@xxxxxxx>
> ---
>  fs/dax.c           | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  fs/ext2/file.c     |  35 +++++++-
>  include/linux/fs.h |   4 +-
>  mm/filemap_xip.c   | 206 -----------------------------------------------
>  4 files changed, 268 insertions(+), 209 deletions(-)
> 
> diff --git a/fs/dax.c b/fs/dax.c
> index 02e226f..ac5d3a6 100644
> --- a/fs/dax.c
> +++ b/fs/dax.c
> @@ -19,9 +19,13 @@
>  #include <linux/buffer_head.h>
>  #include <linux/fs.h>
>  #include <linux/genhd.h>
> +#include <linux/highmem.h>
> +#include <linux/memcontrol.h>
> +#include <linux/mm.h>
>  #include <linux/mutex.h>
>  #include <linux/sched.h>
>  #include <linux/uio.h>
> +#include <linux/vmstat.h>
>  
>  int dax_clear_blocks(struct inode *inode, sector_t block, long size)
>  {
> @@ -228,3 +232,231 @@ ssize_t dax_do_io(int rw, struct kiocb *iocb, struct inode *inode,
>  	return retval;
>  }
>  EXPORT_SYMBOL_GPL(dax_do_io);
> +
> +/*
> + * The user has performed a load from a hole in the file.  Allocating
> + * a new page in the file would cause excessive storage usage for
> + * workloads with sparse files.  We allocate a page cache page instead.
> + * We'll kick it out of the page cache if it's ever written to,
> + * otherwise it will simply fall out of the page cache under memory
> + * pressure without ever having been dirtied.

Nice trick :)

> + */
> +static int dax_load_hole(struct address_space *mapping, struct page *page,
> +							struct vm_fault *vmf)
> +{
> +	unsigned long size;
> +	struct inode *inode = mapping->host;

missing newline.

> +	if (!page)
> +		page = find_or_create_page(mapping, vmf->pgoff,
> +						GFP_KERNEL | __GFP_ZERO);
> +	if (!page)
> +		return VM_FAULT_OOM;
> +	/* Recheck i_size under page lock to avoid truncate race */
> +	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
> +	if (vmf->pgoff >= size) {
> +		unlock_page(page);
> +		page_cache_release(page);
> +		return VM_FAULT_SIGBUS;
> +	}
> +
> +	vmf->page = page;
> +	return VM_FAULT_LOCKED;
> +}
> +
> +static int copy_user_bh(struct page *to, struct buffer_head *bh,
> +			unsigned blkbits, unsigned long vaddr)
> +{
> +	void *vfrom, *vto;

missing newline.

> +	if (dax_get_addr(bh, &vfrom, blkbits) < 0)
> +		return -EIO;
> +	vto = kmap_atomic(to);
> +	copy_user_page(vto, vfrom, vaddr, to);
> +	kunmap_atomic(vto);
> +	return 0;
> +}
> +
> +static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh,
> +			struct vm_area_struct *vma, struct vm_fault *vmf)
> +{
> +	struct address_space *mapping = inode->i_mapping;
> +	sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9);
> +	unsigned long vaddr = (unsigned long)vmf->virtual_address;
> +	void *addr;
> +	unsigned long pfn;
> +	pgoff_t size;
> +	int error;
> +
> +	mutex_lock(&mapping->i_mmap_mutex);
> +
> +	/*
> +	 * Check truncate didn't happen while we were allocating a block.
> +	 * If it did, this block may or may not be still allocated to the
> +	 * file.  We can't tell the filesystem to free it because we can't
> +	 * take i_mutex here.  In the worst case, the file still has blocks
> +	 * allocated past the end of the file.
> +	 */
> +	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
> +	if (unlikely(vmf->pgoff >= size)) {
> +		error = -EIO;
> +		goto out;
> +	}
> +
> +	error = bdev_direct_access(bh->b_bdev, sector, &addr, &pfn, bh->b_size);
> +	if (error < 0)
> +		goto out;
> +	if (error < PAGE_SIZE) {
> +		error = -EIO;
> +		goto out;
> +	}
> +
> +	if (buffer_unwritten(bh) || buffer_new(bh))
> +		clear_page(addr);
> +
> +	error = vm_insert_mixed(vma, vaddr, pfn);
> +
> + out:
> +	mutex_unlock(&mapping->i_mmap_mutex);
> +
> +	if (bh->b_end_io)
> +		bh->b_end_io(bh, 1);
> +
> +	return error;
> +}
> +
> +static int do_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
> +			get_block_t get_block)
> +{
> +	struct file *file = vma->vm_file;
> +	struct inode *inode = file_inode(file);
> +	struct address_space *mapping = file->f_mapping;
> +	struct page *page;
> +	struct buffer_head bh;
> +	unsigned long vaddr = (unsigned long)vmf->virtual_address;
> +	unsigned blkbits = inode->i_blkbits;

unsigned -> unsigned int

> +	sector_t block;
> +	pgoff_t size;
> +	int error;
> +	int major = 0;
> +
> +	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
> +	if (vmf->pgoff >= size)
> +		return VM_FAULT_SIGBUS;
> +
> +	memset(&bh, 0, sizeof(bh));
> +	block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits);
> +	bh.b_size = PAGE_SIZE;
> +
> + repeat:
> +	page = find_get_page(mapping, vmf->pgoff);
> +	if (page) {
> +		if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) {
> +			page_cache_release(page);
> +			return VM_FAULT_RETRY;
> +		}
> +		if (unlikely(page->mapping != mapping)) {
> +			unlock_page(page);
> +			page_cache_release(page);
> +			goto repeat;
> +		}
> +	}
> +
> +	error = get_block(inode, block, &bh, 0);
> +	if (!error && (bh.b_size < PAGE_SIZE))
> +		error = -EIO;
> +	if (error)
> +		goto unlock_page;
> +
> +	if (!buffer_mapped(&bh) && !buffer_unwritten(&bh) && !vmf->cow_page) {
> +		if (vmf->flags & FAULT_FLAG_WRITE) {
> +			error = get_block(inode, block, &bh, 1);
> +			count_vm_event(PGMAJFAULT);
> +			mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
> +			major = VM_FAULT_MAJOR;
> +			if (!error && (bh.b_size < PAGE_SIZE))
> +				error = -EIO;
> +			if (error)
> +				goto unlock_page;
> +		} else {
> +			return dax_load_hole(mapping, page, vmf);
> +		}
> +	}
> +
> +	if (vmf->cow_page) {
> +		struct page *new_page = vmf->cow_page;

add newline.

> +		if (buffer_written(&bh))
> +			error = copy_user_bh(new_page, &bh, blkbits, vaddr);
> +		else
> +			clear_user_highpage(new_page, vaddr);
> +		if (error)
> +			goto unlock_page;
> +		vmf->page = page;
> +		if (!page) {
> +			mutex_lock(&mapping->i_mmap_mutex);
> +			/* Check we didn't race with truncate */
> +			size = (i_size_read(inode) + PAGE_SIZE - 1) >>
> +								PAGE_SHIFT;
> +			if (vmf->pgoff >= size) {
> +				mutex_unlock(&mapping->i_mmap_mutex);
> +				error = -EIO;
> +				goto out;
> +			}
> +		}

If page is non-NULL, is it possible that we return VM_FAULT_LOCKED
without actually holding i_mmap_mutex ? Is it on purpose ?

> +		return VM_FAULT_LOCKED;
> +	}

Thanks,

Mathieu

> +
> +	/* Check we didn't race with a read fault installing a new page */
> +	if (!page && major)
> +		page = find_lock_page(mapping, vmf->pgoff);
> +
> +	if (page) {
> +		unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
> +							PAGE_CACHE_SIZE, 0);
> +		delete_from_page_cache(page);
> +		unlock_page(page);
> +		page_cache_release(page);
> +	}
> +
> +	error = dax_insert_mapping(inode, &bh, vma, vmf);
> +
> + out:
> +	if (error == -ENOMEM)
> +		return VM_FAULT_OOM | major;
> +	/* -EBUSY is fine, somebody else faulted on the same PTE */
> +	if ((error < 0) && (error != -EBUSY))
> +		return VM_FAULT_SIGBUS | major;
> +	return VM_FAULT_NOPAGE | major;
> +
> + unlock_page:
> +	if (page) {
> +		unlock_page(page);
> +		page_cache_release(page);
> +	}
> +	goto out;
> +}
> +
> +/**
> + * dax_fault - handle a page fault on a DAX file
> + * @vma: The virtual memory area where the fault occurred
> + * @vmf: The description of the fault
> + * @get_block: The filesystem method used to translate file offsets to blocks
> + *
> + * When a page fault occurs, filesystems may call this helper in their
> + * fault handler for DAX files.
> + */
> +int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
> +			get_block_t get_block)
> +{
> +	int result;
> +	struct super_block *sb = file_inode(vma->vm_file)->i_sb;
> +
> +	if (vmf->flags & FAULT_FLAG_WRITE) {
> +		sb_start_pagefault(sb);
> +		file_update_time(vma->vm_file);
> +	}
> +	result = do_dax_fault(vma, vmf, get_block);
> +	if (vmf->flags & FAULT_FLAG_WRITE)
> +		sb_end_pagefault(sb);
> +
> +	return result;
> +}
> +EXPORT_SYMBOL_GPL(dax_fault);
> diff --git a/fs/ext2/file.c b/fs/ext2/file.c
> index a247123..da8dc64 100644
> --- a/fs/ext2/file.c
> +++ b/fs/ext2/file.c
> @@ -25,6 +25,37 @@
>  #include "xattr.h"
>  #include "acl.h"
>  
> +#ifdef CONFIG_EXT2_FS_XIP
> +static int ext2_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
> +{
> +	return dax_fault(vma, vmf, ext2_get_block);
> +}
> +
> +static int ext2_dax_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
> +{
> +	return dax_mkwrite(vma, vmf, ext2_get_block);
> +}
> +
> +static const struct vm_operations_struct ext2_dax_vm_ops = {
> +	.fault		= ext2_dax_fault,
> +	.page_mkwrite	= ext2_dax_mkwrite,
> +	.remap_pages	= generic_file_remap_pages,
> +};
> +
> +static int ext2_file_mmap(struct file *file, struct vm_area_struct *vma)
> +{
> +	if (!IS_DAX(file_inode(file)))
> +		return generic_file_mmap(file, vma);
> +
> +	file_accessed(file);
> +	vma->vm_ops = &ext2_dax_vm_ops;
> +	vma->vm_flags |= VM_MIXEDMAP;
> +	return 0;
> +}
> +#else
> +#define ext2_file_mmap	generic_file_mmap
> +#endif
> +
>  /*
>   * Called when filp is released. This happens when all file descriptors
>   * for a single struct file are closed. Note that different open() calls
> @@ -70,7 +101,7 @@ const struct file_operations ext2_file_operations = {
>  #ifdef CONFIG_COMPAT
>  	.compat_ioctl	= ext2_compat_ioctl,
>  #endif
> -	.mmap		= generic_file_mmap,
> +	.mmap		= ext2_file_mmap,
>  	.open		= dquot_file_open,
>  	.release	= ext2_release_file,
>  	.fsync		= ext2_fsync,
> @@ -89,7 +120,7 @@ const struct file_operations ext2_xip_file_operations = {
>  #ifdef CONFIG_COMPAT
>  	.compat_ioctl	= ext2_compat_ioctl,
>  #endif
> -	.mmap		= xip_file_mmap,
> +	.mmap		= ext2_file_mmap,
>  	.open		= dquot_file_open,
>  	.release	= ext2_release_file,
>  	.fsync		= ext2_fsync,
> diff --git a/include/linux/fs.h b/include/linux/fs.h
> index c04d371..338f04b 100644
> --- a/include/linux/fs.h
> +++ b/include/linux/fs.h
> @@ -49,6 +49,7 @@ struct swap_info_struct;
>  struct seq_file;
>  struct workqueue_struct;
>  struct iov_iter;
> +struct vm_fault;
>  
>  extern void __init inode_init(void);
>  extern void __init inode_init_early(void);
> @@ -2491,10 +2492,11 @@ extern int nonseekable_open(struct inode * inode, struct file * filp);
>  
>  #ifdef CONFIG_FS_XIP
>  int dax_clear_blocks(struct inode *, sector_t block, long size);
> -extern int xip_file_mmap(struct file * file, struct vm_area_struct * vma);
>  extern int xip_truncate_page(struct address_space *mapping, loff_t from);
>  ssize_t dax_do_io(int rw, struct kiocb *, struct inode *, struct iov_iter *,
>  		loff_t, get_block_t, dio_iodone_t, int flags);
> +int dax_fault(struct vm_area_struct *, struct vm_fault *, get_block_t);
> +#define dax_mkwrite(vma, vmf, gb)	dax_fault(vma, vmf, gb)
>  #else
>  static inline int dax_clear_blocks(struct inode *i, sector_t blk, long sz)
>  {
> diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c
> index f7c37a1..9dd45f3 100644
> --- a/mm/filemap_xip.c
> +++ b/mm/filemap_xip.c
> @@ -22,212 +22,6 @@
>  #include <asm/io.h>
>  
>  /*
> - * We do use our own empty page to avoid interference with other users
> - * of ZERO_PAGE(), such as /dev/zero
> - */
> -static DEFINE_MUTEX(xip_sparse_mutex);
> -static seqcount_t xip_sparse_seq = SEQCNT_ZERO(xip_sparse_seq);
> -static struct page *__xip_sparse_page;
> -
> -/* called under xip_sparse_mutex */
> -static struct page *xip_sparse_page(void)
> -{
> -	if (!__xip_sparse_page) {
> -		struct page *page = alloc_page(GFP_HIGHUSER | __GFP_ZERO);
> -
> -		if (page)
> -			__xip_sparse_page = page;
> -	}
> -	return __xip_sparse_page;
> -}
> -
> -/*
> - * __xip_unmap is invoked from xip_unmap and
> - * xip_write
> - *
> - * This function walks all vmas of the address_space and unmaps the
> - * __xip_sparse_page when found at pgoff.
> - */
> -static void
> -__xip_unmap (struct address_space * mapping,
> -		     unsigned long pgoff)
> -{
> -	struct vm_area_struct *vma;
> -	struct mm_struct *mm;
> -	unsigned long address;
> -	pte_t *pte;
> -	pte_t pteval;
> -	spinlock_t *ptl;
> -	struct page *page;
> -	unsigned count;
> -	int locked = 0;
> -
> -	count = read_seqcount_begin(&xip_sparse_seq);
> -
> -	page = __xip_sparse_page;
> -	if (!page)
> -		return;
> -
> -retry:
> -	mutex_lock(&mapping->i_mmap_mutex);
> -	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
> -		mm = vma->vm_mm;
> -		address = vma->vm_start +
> -			((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
> -		BUG_ON(address < vma->vm_start || address >= vma->vm_end);
> -		pte = page_check_address(page, mm, address, &ptl, 1);
> -		if (pte) {
> -			/* Nuke the page table entry. */
> -			flush_cache_page(vma, address, pte_pfn(*pte));
> -			pteval = ptep_clear_flush(vma, address, pte);
> -			page_remove_rmap(page);
> -			dec_mm_counter(mm, MM_FILEPAGES);
> -			BUG_ON(pte_dirty(pteval));
> -			pte_unmap_unlock(pte, ptl);
> -			/* must invalidate_page _before_ freeing the page */
> -			mmu_notifier_invalidate_page(mm, address);
> -			page_cache_release(page);
> -		}
> -	}
> -	mutex_unlock(&mapping->i_mmap_mutex);
> -
> -	if (locked) {
> -		mutex_unlock(&xip_sparse_mutex);
> -	} else if (read_seqcount_retry(&xip_sparse_seq, count)) {
> -		mutex_lock(&xip_sparse_mutex);
> -		locked = 1;
> -		goto retry;
> -	}
> -}
> -
> -/*
> - * xip_fault() is invoked via the vma operations vector for a
> - * mapped memory region to read in file data during a page fault.
> - *
> - * This function is derived from filemap_fault, but used for execute in place
> - */
> -static int xip_file_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
> -{
> -	struct file *file = vma->vm_file;
> -	struct address_space *mapping = file->f_mapping;
> -	struct inode *inode = mapping->host;
> -	pgoff_t size;
> -	void *xip_mem;
> -	unsigned long xip_pfn;
> -	struct page *page;
> -	int error;
> -
> -	/* XXX: are VM_FAULT_ codes OK? */
> -again:
> -	size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
> -	if (vmf->pgoff >= size)
> -		return VM_FAULT_SIGBUS;
> -
> -	error = mapping->a_ops->get_xip_mem(mapping, vmf->pgoff, 0,
> -						&xip_mem, &xip_pfn);
> -	if (likely(!error))
> -		goto found;
> -	if (error != -ENODATA)
> -		return VM_FAULT_OOM;
> -
> -	/* sparse block */
> -	if ((vma->vm_flags & (VM_WRITE | VM_MAYWRITE)) &&
> -	    (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) &&
> -	    (!(mapping->host->i_sb->s_flags & MS_RDONLY))) {
> -		int err;
> -
> -		/* maybe shared writable, allocate new block */
> -		mutex_lock(&xip_sparse_mutex);
> -		error = mapping->a_ops->get_xip_mem(mapping, vmf->pgoff, 1,
> -							&xip_mem, &xip_pfn);
> -		mutex_unlock(&xip_sparse_mutex);
> -		if (error)
> -			return VM_FAULT_SIGBUS;
> -		/* unmap sparse mappings at pgoff from all other vmas */
> -		__xip_unmap(mapping, vmf->pgoff);
> -
> -found:
> -		/* We must recheck i_size under i_mmap_mutex */
> -		mutex_lock(&mapping->i_mmap_mutex);
> -		size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >>
> -							PAGE_CACHE_SHIFT;
> -		if (unlikely(vmf->pgoff >= size)) {
> -			mutex_unlock(&mapping->i_mmap_mutex);
> -			return VM_FAULT_SIGBUS;
> -		}
> -		err = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
> -							xip_pfn);
> -		mutex_unlock(&mapping->i_mmap_mutex);
> -		if (err == -ENOMEM)
> -			return VM_FAULT_OOM;
> -		/*
> -		 * err == -EBUSY is fine, we've raced against another thread
> -		 * that faulted-in the same page
> -		 */
> -		if (err != -EBUSY)
> -			BUG_ON(err);
> -		return VM_FAULT_NOPAGE;
> -	} else {
> -		int err, ret = VM_FAULT_OOM;
> -
> -		mutex_lock(&xip_sparse_mutex);
> -		write_seqcount_begin(&xip_sparse_seq);
> -		error = mapping->a_ops->get_xip_mem(mapping, vmf->pgoff, 0,
> -							&xip_mem, &xip_pfn);
> -		if (unlikely(!error)) {
> -			write_seqcount_end(&xip_sparse_seq);
> -			mutex_unlock(&xip_sparse_mutex);
> -			goto again;
> -		}
> -		if (error != -ENODATA)
> -			goto out;
> -
> -		/* We must recheck i_size under i_mmap_mutex */
> -		mutex_lock(&mapping->i_mmap_mutex);
> -		size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >>
> -							PAGE_CACHE_SHIFT;
> -		if (unlikely(vmf->pgoff >= size)) {
> -			ret = VM_FAULT_SIGBUS;
> -			goto unlock;
> -		}
> -		/* not shared and writable, use xip_sparse_page() */
> -		page = xip_sparse_page();
> -		if (!page)
> -			goto unlock;
> -		err = vm_insert_page(vma, (unsigned long)vmf->virtual_address,
> -							page);
> -		if (err == -ENOMEM)
> -			goto unlock;
> -
> -		ret = VM_FAULT_NOPAGE;
> -unlock:
> -		mutex_unlock(&mapping->i_mmap_mutex);
> -out:
> -		write_seqcount_end(&xip_sparse_seq);
> -		mutex_unlock(&xip_sparse_mutex);
> -
> -		return ret;
> -	}
> -}
> -
> -static const struct vm_operations_struct xip_file_vm_ops = {
> -	.fault	= xip_file_fault,
> -	.page_mkwrite	= filemap_page_mkwrite,
> -	.remap_pages = generic_file_remap_pages,
> -};
> -
> -int xip_file_mmap(struct file * file, struct vm_area_struct * vma)
> -{
> -	BUG_ON(!file->f_mapping->a_ops->get_xip_mem);
> -
> -	file_accessed(file);
> -	vma->vm_ops = &xip_file_vm_ops;
> -	vma->vm_flags |= VM_MIXEDMAP;
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(xip_file_mmap);
> -
> -/*
>   * truncate a page used for execute in place
>   * functionality is analog to block_truncate_page but does use get_xip_mem
>   * to get the page instead of page cache
> -- 
> 2.1.0
> 
> --
> To unsubscribe, send a message with 'unsubscribe linux-mm' in
> the body to majordomo@xxxxxxxxxx  For more info on Linux MM,
> see: http://www.linux-mm.org/ .
> Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>
> 
> 

-- 
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
Key fingerprint: 2A0B 4ED9 15F2 D3FA 45F5  B162 1728 0A97 8118 6ACF
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/