[PATCH 4/5] mm: add and use mmap_read_lock[_killable]_nosteal() on contention

[Lorenzo Stoakes (ARM)]

With down_read[_killable]_nosteal() available, it becomes possible to
release the mmap read lock in such a way that a queued write lock always
takes precedence.

Expose this to mm through two new wrappers - mmap_read_lock_nosteal() and
mmap_read_lock_killable_nosteal().

Make use of this in scenarios where the mmap read lock is dropped
explicitly for the purposes of relieving mmap_lock contention - the BPF
task VMA iterator, and lock_next_vma_under_mmap_lock(), which is where
/proc/$pid/smaps_rollup re-acquires the mmap lock after dropping it on
contention and falling back from per-VMA locking.

Signed-off-by: Lorenzo Stoakes (ARM) <ljs@xxxxxxxxxx>
---
 include/linux/mmap_lock.h | 44 ++++++++++++++++++++++++++++++++++++++++++++
 kernel/bpf/task_iter.c    |  2 +-
 mm/mmap_lock.c            |  2 +-
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h
index 9ccbbcbd7aca..88906ecd6ca3 100644
--- a/include/linux/mmap_lock.h
+++ b/include/linux/mmap_lock.h
@@ -627,6 +627,31 @@ static inline int __must_check mmap_read_lock_killable(struct mm_struct *mm)
 	return ret;
 }
 
+/**
+ * mmap_read_lock_killable_nosteal() - Acquire a killable mmap read lock, but
+ * yield to any queued write locks.
+ * @mm: The mm whose mmap read lock is required.
+ *
+ * If an mmap write lock is queued behind readers, but its handoff timeout has
+ * yet to expire, releasing then re-acquiring a read lock can steal the lock
+ * from it.
+ *
+ * mmap_read_lock_killable_nosteal() will queue in this case instead, which is
+ * useful when releasing the mmap read lock specifically to relieve lock
+ * contention.
+ *
+ * Returns: 0 on success, or an error otherwise.
+ */
+static inline int __must_check mmap_read_lock_killable_nosteal(struct mm_struct *mm)
+{
+	int ret;
+
+	__mmap_lock_trace_start_locking(mm, false);
+	ret = down_read_killable_nosteal(&mm->mmap_lock);
+	__mmap_lock_trace_acquire_returned(mm, false, ret == 0);
+	return ret;
+}
+
 static inline bool __must_check mmap_read_trylock(struct mm_struct *mm)
 {
 	bool ret;
@@ -637,6 +662,25 @@ static inline bool __must_check mmap_read_trylock(struct mm_struct *mm)
 	return ret;
 }
 
+/**
+ * mmap_read_lock_nosteal() - Acquire the mmap read lock, but yield to
+ * any queued write locks.
+ * @mm: The mm whose mmap read lock is required.
+ *
+ * If an mmap write lock is queued behind readers, but its handoff timeout has
+ * yet to expire, releasing then re-acquiring a read lock can steal the lock
+ * from it.
+ *
+ * mmap_read_lock_nosteal() will queue in this case instead, which is useful when
+ * releasing the mmap read lock specifically to relieve lock contention.
+ */
+static inline void mmap_read_lock_nosteal(struct mm_struct *mm)
+{
+	__mmap_lock_trace_start_locking(mm, false);
+	down_read_nosteal(&mm->mmap_lock);
+	__mmap_lock_trace_acquire_returned(mm, false, true);
+}
+
 static inline void mmap_read_unlock(struct mm_struct *mm)
 {
 	__mmap_lock_trace_released(mm, false);
diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c
index 2ee7b3b3c10b..b455306a6b52 100644
--- a/kernel/bpf/task_iter.c
+++ b/kernel/bpf/task_iter.c
@@ -486,7 +486,7 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info)
 			info->prev_vm_end = curr_vma->vm_end;
 			op = task_vma_iter_find_vma;
 			mmap_read_unlock(curr_mm);
-			if (mmap_read_lock_killable(curr_mm)) {
+			if (mmap_read_lock_killable_nosteal(curr_mm)) {
 				mmput(curr_mm);
 				goto finish;
 			}
diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c
index 834bbf86e004..41b11b843211 100644
--- a/mm/mmap_lock.c
+++ b/mm/mmap_lock.c
@@ -412,7 +412,7 @@ static struct vm_area_struct *lock_next_vma_under_mmap_lock(struct mm_struct *mm
 	struct vm_area_struct *vma;
 	int ret;
 
-	ret = mmap_read_lock_killable(mm);
+	ret = mmap_read_lock_killable_nosteal(mm);
 	if (ret)
 		return ERR_PTR(ret);
 

-- 
2.56.0