[PATCH v4 07/11] futex: Allow to make the number of slots invariant.

[Sebastian Andrzej Siewior]

Add an option to freeze the number of hash buckets. The idea is to have
fixed once a certain size is acceptable so that it can be avoided to
acquire mm_struct::futex_hash_lock on certain operations.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@xxxxxxxxxxxxx>
---
 include/uapi/linux/prctl.h |  2 ++
 kernel/futex/core.c        | 54 ++++++++++++++++++++++++++++++++++++--
 kernel/futex/futex.h       |  1 +
 kernel/futex/pi.c          |  2 +-
 kernel/futex/requeue.c     |  3 ++-
 5 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 55b843644c51a..d1f4b3dea565c 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -357,5 +357,7 @@ struct prctl_mm_map {
 #define PR_FUTEX_HASH			77
 # define PR_FUTEX_HASH_SET_SLOTS	1
 # define PR_FUTEX_HASH_GET_SLOTS	2
+# define PR_FUTEX_HASH_SET_INVARIANT	3
+# define PR_FUTEX_HASH_GET_INVARIANT	4
 
 #endif /* _LINUX_PRCTL_H */
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 0dd7100e36419..1abea8f9abd22 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -61,6 +61,7 @@ struct futex_hash_bucket_private {
 	rcuref_t	users;
 	unsigned int	hash_mask;
 	struct rcu_head rcu;
+	bool		slots_invariant;
 	struct futex_hash_bucket queues[];
 };
 
@@ -1266,6 +1267,7 @@ static int futex_hash_allocate(unsigned int hash_slots)
 	struct futex_hash_bucket_private *hb_p, *hb_p_old = NULL;
 	struct mm_struct *mm;
 	size_t alloc_size;
+	int ret = 0;
 	int i;
 
 	if (hash_slots == 0)
@@ -1291,20 +1293,30 @@ static int futex_hash_allocate(unsigned int hash_slots)
 
 	rcuref_init(&hb_p->users, 1);
 	hb_p->hash_mask = hash_slots - 1;
+	hb_p->slots_invariant = false;
 
 	for (i = 0; i < hash_slots; i++)
 		futex_hash_bucket_init(&hb_p->queues[i], i + 1);
 
 	mm = current->mm;
 	scoped_guard(rwsem_write, &mm->futex_hash_lock) {
+
 		hb_p_old = rcu_dereference_check(mm->futex_hash_bucket,
 						 lockdep_is_held(&mm->futex_hash_lock));
-		rcu_assign_pointer(mm->futex_hash_bucket, hb_p);
+		if (hb_p_old && hb_p_old->slots_invariant)
+			ret = -EINVAL;
+		else
+			rcu_assign_pointer(mm->futex_hash_bucket, hb_p);
 	}
+	if (ret) {
+		kvfree(hb_p);
+		return ret;
+	}
+
 	if (hb_p_old)
 		futex_put_old_hb_p(hb_p_old);
 
-	return 0;
+	return ret;
 }
 
 int futex_hash_allocate_default(void)
@@ -1323,6 +1335,36 @@ static int futex_hash_get_slots(void)
 	return 0;
 }
 
+static int futex_hash_set_invariant(void)
+{
+	struct futex_hash_bucket_private *hb_p;
+	struct mm_struct *mm;
+
+	mm = current->mm;
+	guard(rwsem_write)(&mm->futex_hash_lock);
+	hb_p = rcu_dereference_check(mm->futex_hash_bucket,
+				     lockdep_is_held(&mm->futex_hash_lock));
+	if (!hb_p)
+		return -EINVAL;
+	if (hb_p->slots_invariant)
+		return -EALREADY;
+	hb_p->slots_invariant = true;
+	return 0;
+}
+
+bool futex_hash_is_invariant(void)
+{
+	struct futex_hash_bucket_private *hb_p;
+	struct mm_struct *mm;
+
+	mm = current->mm;
+	guard(rcu)();
+	hb_p = rcu_dereference(mm->futex_hash_bucket);
+	if (!hb_p)
+		return -EINVAL;
+	return hb_p->slots_invariant;
+}
+
 int futex_hash_prctl(unsigned long arg2, unsigned long arg3,
 		     unsigned long arg4, unsigned long arg5)
 {
@@ -1337,6 +1379,14 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3,
 		ret = futex_hash_get_slots();
 		break;
 
+	case PR_FUTEX_HASH_SET_INVARIANT:
+		ret = futex_hash_set_invariant();
+		break;
+
+	case PR_FUTEX_HASH_GET_INVARIANT:
+		ret = futex_hash_is_invariant();
+		break;
+
 	default:
 		ret = -EINVAL;
 		break;
diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h
index 503f56643a966..e81820a393027 100644
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -208,6 +208,7 @@ extern void futex_hash_get(struct futex_hash_bucket *hb);
 extern bool futex_check_hb_valid(struct futex_hash_bucket *hb);
 extern bool check_pi_lock_owner(u32 __user *uaddr);
 extern void reset_pi_state_owner(struct futex_pi_state *pi_state);
+extern bool futex_hash_is_invariant(void);
 
 static inline struct futex_hash_bucket *futex_hb_from_futex_q(struct futex_q *q)
 {
diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c
index b4156d1cc6608..9df320be750c3 100644
--- a/kernel/futex/pi.c
+++ b/kernel/futex/pi.c
@@ -1202,7 +1202,7 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
 	if (!IS_ENABLED(CONFIG_FUTEX_PI))
 		return -ENOSYS;
 
-	if (!(flags & FLAGS_SHARED))
+	if (!(flags & FLAGS_SHARED) && !futex_hash_is_invariant())
 		futex_hash_lock = &current->mm->futex_hash_lock;
 retry:
 	if (get_user(uval, uaddr))
diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c
index 6b3c4413fbf47..904c68abfb8f3 100644
--- a/kernel/futex/requeue.c
+++ b/kernel/futex/requeue.c
@@ -431,7 +431,8 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
 		if (refill_pi_state_cache())
 			return -ENOMEM;
 
-		if (!(flags1 & FLAGS_SHARED) || !(flags2 & FLAGS_SHARED))
+		if ((!(flags1 & FLAGS_SHARED) || !(flags2 & FLAGS_SHARED)) &&
+		    !futex_hash_is_invariant())
 			futex_hash_lock = &current->mm->futex_hash_lock;
 	}
 
-- 
2.45.2