[PATCH] simplification

[Lorenzo Stoakes (ARM)]

---
 mm/mmap_lock.c | 47 +++++++++++++++++++++++++++++++++++++----------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c
index 83b7d3ded0dc..6568dda0aba4 100644
--- a/mm/mmap_lock.c
+++ b/mm/mmap_lock.c
@@ -428,6 +428,16 @@ static struct vm_area_struct *lock_next_vma_under_mmap_lock(struct mm_struct *mm
 	return vma;
 }

+static bool vma_range_is_empty(struct mm_struct *mm, unsigned long start,
+		unsigned long last)
+{
+	MA_STATE(mas, &mm->mm_mt, start, start);
+
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
+
+	return !mas_walk(&mas) && mas.last >= last;
+}
+
 struct vm_area_struct *lock_next_vma(struct mm_struct *mm,
 				     struct vma_iterator *vmi,
 				     unsigned long from_addr)
@@ -438,11 +448,29 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm,

 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu read lock held");
 retry:
+	/*
+	 * No lock was held since the last call, so the iterator's cached range
+	 * may no longer be valid. Always restart by address.
+	 *
+	 * The vma_range_is_empty() checks below avoid unnecessary mmap_lock
+	 * fallbacks.
+	 */
+	vma_iter_set(vmi, from_addr);
 	/* Start mmap_lock speculation in case we need to verify the vma later */
 	mmap_unlocked = mmap_lock_speculate_try_begin(mm, &mm_wr_seq);
 	vma = vma_next(vmi);
-	if (!vma)
-		return NULL;
+	if (!vma) {
+		/*
+		 * There is no reason to fallback if there really isn't anything
+		 * in the remaining range.
+		 */
+		if ((mmap_unlocked && !mmap_lock_speculate_retry(mm, mm_wr_seq)) ||
+		    vma_range_is_empty(mm, from_addr, ULONG_MAX))
+			return NULL;
+
+		rcu_read_unlock();
+		goto fallback;
+	}

 	vma = vma_start_read(mm, vma);
 	if (IS_ERR_OR_NULL(vma)) {
@@ -470,15 +498,14 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm,
 	 * it was not shrunk after we found it and another vma has not been
 	 * installed ahead of it. Otherwise we might observe a gap that should
 	 * not be there.
+	 *
+	 * If speculation fails, avoid taking the mmap_lock when a direct lookup
+	 * confirms that the gap is empty.
 	 */
-	if (from_addr < vma->vm_start) {
-		/* Verify only if the address space might have changed since vma lookup. */
-		if (!mmap_unlocked || mmap_lock_speculate_retry(mm, mm_wr_seq)) {
-			vma_iter_set(vmi, from_addr);
-			if (vma != vma_next(vmi))
-				goto fallback_unlock;
-		}
-	}
+	if (from_addr < vma->vm_start &&
+	    (!mmap_unlocked || mmap_lock_speculate_retry(mm, mm_wr_seq)) &&
+	    !vma_range_is_empty(mm, from_addr, vma->vm_start - 1))
+		goto fallback_unlock;

 	return vma;

--
2.56.0