[PATCH 3/3] FUTEX : NUMA friendly global hashtable

[Eric Dumazet]

[PATCH 3/3] FUTEX : NUMA friendly global hashtable

On NUMA machines, we should get better performance using a big futex 
hashtable, allocated with vmalloc() so that it is spreaded on several nodes.

I chose a static size of four pages. (Very big NUMA machines have 64k page 
size)

This patch should have a temporary effect, as most futexes are expected to be 
stored in process private tables. We probably can drop it  in five years :)


Signed-off-by: Eric Dumazet <dada1@xxxxxxxxxxxxx>
---
 kernel/futex.c |   26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)
--- linux-2.6.21-rc3/kernel/futex.c	2007-03-15 18:54:47.000000000 +0100
+++ linux-2.6.21-rc3-ed/kernel/futex.c	2007-03-15 18:54:47.000000000 +0100
@@ -152,8 +152,13 @@ struct futex_hash_bucket {
 # define FUTEX_HASH_SLOTS	16
 # define FUTEX_NOPRIVHASH	/* no private hashtable, only one global */
 #else
-# define FUTEX_HASH_SLOTS	256
-# define FUTEX_PRIVHASH_SLOTS	64
+# ifdef CONFIG_NUMA
+#  define FUTEX_HASH_SLOTS	((4*PAGE_SIZE)/sizeof(struct futex_hash_bucket))
+#  define FUTEX_PRIVHASH_SLOTS	(4096 / sizeof(struct futex_hash_bucket))
+# else
+#  define FUTEX_HASH_SLOTS	256
+#  define FUTEX_PRIVHASH_SLOTS	64
+# endif
 #endif
 
 #define FUTEX_PRIVHASH_SIZE \
@@ -166,8 +171,14 @@ struct futex_hash_bucket {
  * PTHREAD_PROCESS_PRIVATE futexes may be hashed into this table too if the 
  * owner process failed to allocate its private hashtable (or CONFIG_BASE_SMALL)
  *
+ * On NUMA configs, table is allocated with vmalloc() to spread this hash table
+ * up to 4 nodes (we use 4 pages)
  */
+#ifdef CONFIG_NUMA
+static struct futex_hash_bucket *futex_queues __read_mostly;
+#else
 static struct futex_hash_bucket futex_queues[FUTEX_HASH_SLOTS];
+#endif
 
 /* Futex-fs vfsmount entry: */
 static struct vfsmount *futex_mnt;
@@ -2051,7 +2062,16 @@ static int __init init(void)
 		return PTR_ERR(futex_mnt);
 	}
 
-	for (i = 0; i < ARRAY_SIZE(futex_queues); i++) {
+#ifdef CONFIG_NUMA
+	/*
+	 * vmalloc() is supposed to obey mempolicy and spread our 4 pages
+	 * on several nodes
+	 */
+	futex_queues = vmalloc(FUTEX_HASH_SLOTS * sizeof(*futex_queues));
+	if (!futex_queues)
+		panic("Failed to allocate futex hash table\n");
+#endif
+	for (i = 0; i < FUTEX_HASH_SLOTS; i++) {
 		INIT_LIST_HEAD(&futex_queues[i].chain);
 		spin_lock_init(&futex_queues[i].lock);
 	}