include/linux/mm.h | 3 ++- mm/memory.c | 46 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index dc7b87310c10..e82a604339c0 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1648,7 +1648,8 @@ int invalidate_inode_page(struct page *page); #ifdef CONFIG_MMU extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, - unsigned long address, unsigned int flags); + unsigned long address, unsigned int flags, + struct pt_regs *); extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, unsigned long address, unsigned int fault_flags, bool *unlocked); diff --git a/mm/memory.c b/mm/memory.c index dc7f3543b1fd..f15056151fb1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -72,6 +72,7 @@ #include #include +#include #include #include @@ -4353,7 +4354,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, * return value. See filemap_fault() and __lock_page_or_retry(). */ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, - unsigned int flags) + unsigned int flags, struct pt_regs *regs) { vm_fault_t ret; @@ -4394,6 +4395,49 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, mem_cgroup_oom_synchronize(false); } + if (ret & VM_FAULT_RETRY) + return ret; + + /* + * Do accounting in the common code, to avoid unnecessary + * architecture differences or duplicated code. + * + * We arbitrarily make the rules be: + * + * - faults that never even got here (because the address + * wasn't valid). That includes arch_vma_access_permitted() + * failing above. + * + * So this is expressly not a "this many hardware page + * faults" counter. Use the hw profiling for that. + * + * - incomplete faults (ie RETRY) do not count (see above). + * They will only count once completed. + * + * - the fault counts as a "major" fault when the final + * successful fault is VM_FAULT_MAJOR, or if it was a + * retry (which implies that we couldn't handle it + * immediately previously). + * + * - if the fault is done for GUP, regs wil be NULL and + * no accounting will be done (but you _could_ pass in + * your own regs and it would be accounted to the thread + * doing the fault, not to the target!) + */ + + if (!regs) + return ret; + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); + + if ((ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED)) { + current->maj_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); + } else { + current->min_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); + } + return ret; } EXPORT_SYMBOL_GPL(handle_mm_fault);