[PATCH 1/2] generic smp function call: add multiple queues for scaling

From: Jeremy Fitzhardinge
Date: Tue Jul 29 2008 - 19:33:27 EST

This patch allows an architecture to have multiple smp_call_function
queues, in order to reduce contention on a single queue and lock. By
default there is still one queue, so this patch makes no functional
change.

However, an architecture can set CONFIG_GENERIC_SMP_QUEUES to enable a
certain number of queues. It's expected it will will set up an IPI
vector for each queue, and it should pass the appropriate queue number
into generic_smp_call_function_interrupt.

Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@xxxxxxxxxx>
---
arch/alpha/kernel/smp.c | 2 -
arch/arm/kernel/smp.c | 2 -
arch/ia64/kernel/smp.c | 2 -
arch/m32r/kernel/smp.c | 2 -
arch/mips/kernel/smp.c | 2 -
arch/parisc/kernel/smp.c | 2 -
arch/powerpc/kernel/smp.c | 2 -
arch/sparc64/kernel/smp.c | 2 -
arch/x86/kernel/smp.c | 2 -
arch/x86/mach-voyager/voyager_smp.c | 2 -
arch/x86/xen/smp.c | 2 -
include/linux/smp.h | 2 -
kernel/smp.c | 70 ++++++++++++++++++++++++++++-------
13 files changed, 69 insertions(+), 25 deletions(-)

===================================================================
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -588,7 +588,7 @@
break;

case IPI_CALL_FUNC:
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
break;

case IPI_CALL_FUNC_SINGLE:
===================================================================
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -481,7 +481,7 @@
break;

case IPI_CALL_FUNC:
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
break;

case IPI_CALL_FUNC_SINGLE:
===================================================================
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -114,7 +114,7 @@
stop_this_cpu();
break;
case IPI_CALL_FUNC:
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
break;
case IPI_CALL_FUNC_SINGLE:
generic_smp_call_function_single_interrupt();
===================================================================
--- a/arch/m32r/kernel/smp.c
+++ b/arch/m32r/kernel/smp.c
@@ -576,7 +576,7 @@
void smp_call_function_interrupt(void)
{
irq_enter();
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
irq_exit();
}

===================================================================
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -151,7 +151,7 @@
{
irq_enter();
generic_smp_call_function_single_interrupt();
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
irq_exit();
}

===================================================================
--- a/arch/parisc/kernel/smp.c
+++ b/arch/parisc/kernel/smp.c
@@ -179,7 +179,7 @@

case IPI_CALL_FUNC:
smp_debug(100, KERN_DEBUG "CPU%d IPI_CALL_FUNC\n", this_cpu);
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
break;

case IPI_CALL_FUNC_SINGLE:
===================================================================
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -95,7 +95,7 @@
{
switch(msg) {
case PPC_MSG_CALL_FUNCTION:
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
break;
case PPC_MSG_RESCHEDULE:
/* XXX Do we have to do this? */
===================================================================
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -811,7 +811,7 @@
void smp_call_function_client(int irq, struct pt_regs *regs)
{
clear_softint(1 << irq);
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
}

void smp_call_function_single_client(int irq, struct pt_regs *regs)
===================================================================
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -189,7 +189,7 @@
{
ack_APIC_irq();
irq_enter();
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
#ifdef CONFIG_X86_32
__get_cpu_var(irq_stat).irq_call_count++;
#else
===================================================================
--- a/arch/x86/mach-voyager/voyager_smp.c
+++ b/arch/x86/mach-voyager/voyager_smp.c
@@ -957,7 +957,7 @@
static void smp_call_function_interrupt(void)
{
irq_enter();
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
__get_cpu_var(irq_stat).irq_call_count++;
irq_exit();
}
===================================================================
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -401,7 +401,7 @@
static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
{
irq_enter();
- generic_smp_call_function_interrupt();
+ generic_smp_call_function_interrupt(0);
#ifdef CONFIG_X86_32
__get_cpu_var(irq_stat).irq_call_count++;
#else
===================================================================
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -73,7 +73,7 @@
*/
#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
void generic_smp_call_function_single_interrupt(void);
-void generic_smp_call_function_interrupt(void);
+void generic_smp_call_function_interrupt(unsigned queue);
void ipi_call_lock(void);
void ipi_call_unlock(void);
void ipi_call_lock_irq(void);
===================================================================
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -11,9 +11,19 @@
#include <linux/rculist.h>
#include <linux/smp.h>

+#ifdef CONFIG_GENERIC_SMP_QUEUES
+#define NQUEUES CONFIG_GENERIC_SMP_QUEUES
+#else
+#define NQUEUES 1
+#endif
+
static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
-static LIST_HEAD(call_function_queue);
-__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);
+struct queue {
+ struct list_head list;
+ spinlock_t lock;
+};
+
+static __cacheline_aligned_in_smp struct queue call_function_queues[NQUEUES];

enum {
CSD_FLAG_WAIT = 0x01,
@@ -96,17 +106,20 @@
* Invoked by arch to handle an IPI for call function. Must be called with
* interrupts disabled.
*/
-void generic_smp_call_function_interrupt(void)
+void generic_smp_call_function_interrupt(unsigned queue_no)
{
struct call_function_data *data;
+ struct queue *queue;
int cpu = get_cpu();
+
+ queue = &call_function_queues[queue_no];

/*
* It's ok to use list_for_each_rcu() here even though we may delete
* 'pos', since list_del_rcu() doesn't clear ->next
*/
rcu_read_lock();
- list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
+ list_for_each_entry_rcu(data, &queue->list, csd.list) {
int refs;

if (!cpu_isset(cpu, data->cpumask))
@@ -124,9 +137,9 @@
if (refs)
continue;

- spin_lock(&call_function_lock);
+ spin_lock(&queue->lock);
list_del_rcu(&data->csd.list);
- spin_unlock(&call_function_lock);
+ spin_unlock(&queue->lock);

if (data->csd.flags & CSD_FLAG_WAIT) {
/*
@@ -285,6 +298,8 @@
cpumask_t allbutself;
unsigned long flags;
int cpu, num_cpus;
+ unsigned queue_no;
+ struct queue *queue;

/* Can deadlock when called with interrupts disabled */
WARN_ON(irqs_disabled());
@@ -294,6 +309,9 @@
cpu_clear(cpu, allbutself);
cpus_and(mask, mask, allbutself);
num_cpus = cpus_weight(mask);
+
+ queue_no = cpu % NQUEUES;
+ queue = &call_function_queues[queue_no];

/*
* If zero CPUs, return. If just a single CPU, turn this request
@@ -323,9 +341,9 @@
data->refs = num_cpus;
data->cpumask = mask;

- spin_lock_irqsave(&call_function_lock, flags);
- list_add_tail_rcu(&data->csd.list, &call_function_queue);
- spin_unlock_irqrestore(&call_function_lock, flags);
+ spin_lock_irqsave(&queue->lock, flags);
+ list_add_tail_rcu(&data->csd.list, &queue->list);
+ spin_unlock_irqrestore(&queue->lock, flags);

/* Send a message to all CPUs in the map */
arch_send_call_function_ipi(mask);
@@ -366,20 +384,46 @@

void ipi_call_lock(void)
{
- spin_lock(&call_function_lock);
+ int i;
+
+ for(i = 0; i < NQUEUES; i++)
+ spin_lock(&call_function_queues[i].lock);
}

void ipi_call_unlock(void)
{
- spin_unlock(&call_function_lock);
+ int i;
+
+ for(i = 0; i < NQUEUES; i++)
+ spin_unlock(&call_function_queues[i].lock);
}

void ipi_call_lock_irq(void)
{
- spin_lock_irq(&call_function_lock);
+ int i;
+
+ for(i = 0; i < NQUEUES; i++)
+ spin_lock_irq(&call_function_queues[i].lock);
}

void ipi_call_unlock_irq(void)
{
- spin_unlock_irq(&call_function_lock);
+ int i;
+
+ for(i = 0; i < NQUEUES; i++)
+ spin_unlock_irq(&call_function_queues[i].lock);
}
+
+
+static int __init init_smp_function_call(void)
+{
+ int i;
+
+ for(i = 0; i < NQUEUES; i++) {
+ INIT_LIST_HEAD(&call_function_queues[i].list);
+ spin_lock_init(&call_function_queues[i].lock);
+ }
+
+ return 0;
+}
+early_initcall(init_smp_function_call);


--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/