[PATCH 36/37] locking: Introduce spin_[un]lock_bh_mask()

From: Frederic Weisbecker
Date: Thu Feb 28 2019 - 12:15:12 EST

It allows us to extend the coverage of vector finegrained masking
throughout softirq safe locking. This is especially interesting with
networking that makes extensive use of it.

It works the same way as local_bh_disable_mask():

bh = spin_lock_bh_mask(lock, BIT(NET_RX_SOFTIRQ));
[...]
spin_unlock_bh_mask(lock, bh);

Suggested-by: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Reviewed-by: David S. Miller <davem@xxxxxxxxxxxxx>
Signed-off-by: Frederic Weisbecker <frederic@xxxxxxxxxx>
Cc: Mauro Carvalho Chehab <mchehab+samsung@xxxxxxxxxx>
Cc: Joel Fernandes <joel@xxxxxxxxxxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Pavan Kondeti <pkondeti@xxxxxxxxxxxxxx>
Cc: Paul E . McKenney <paulmck@xxxxxxxxxxxxxxxxxx>
Cc: David S . Miller <davem@xxxxxxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Sebastian Andrzej Siewior <bigeasy@xxxxxxxxxxxxx>
Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
---
include/linux/spinlock.h | 14 ++++++++++++++
include/linux/spinlock_api_smp.h | 28 ++++++++++++++++++++++++++++
include/linux/spinlock_api_up.h | 13 +++++++++++++
kernel/locking/spinlock.c | 19 +++++++++++++++++++
4 files changed, 74 insertions(+)

diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index e089157dcf97..57dd73ed202d 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -270,6 +270,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)

#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
+#define raw_spin_lock_bh_mask(lock, mask) _raw_spin_lock_bh_mask(lock, mask)
#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)

@@ -279,6 +280,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
_raw_spin_unlock_irqrestore(lock, flags); \
} while (0)
#define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock)
+#define raw_spin_unlock_bh_mask(lock, mask) _raw_spin_unlock_bh_mask(lock, mask)

#define raw_spin_trylock_bh(lock) \
__cond_lock(lock, _raw_spin_trylock_bh(lock))
@@ -334,6 +336,12 @@ static __always_inline void spin_lock_bh(spinlock_t *lock)
raw_spin_lock_bh(&lock->rlock);
}

+static __always_inline unsigned int spin_lock_bh_mask(spinlock_t *lock,
+ unsigned int mask)
+{
+ return raw_spin_lock_bh_mask(&lock->rlock, mask);
+}
+
static __always_inline int spin_trylock(spinlock_t *lock)
{
return raw_spin_trylock(&lock->rlock);
@@ -374,6 +382,12 @@ static __always_inline void spin_unlock_bh(spinlock_t *lock)
raw_spin_unlock_bh(&lock->rlock);
}

+static __always_inline void spin_unlock_bh_mask(spinlock_t *lock,
+ unsigned int mask)
+{
+ raw_spin_unlock_bh_mask(&lock->rlock, mask);
+}
+
static __always_inline void spin_unlock_irq(spinlock_t *lock)
{
raw_spin_unlock_irq(&lock->rlock);
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index 42dfab89e740..473641abbc5c 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -26,6 +26,8 @@ void __lockfunc
_raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map)
__acquires(lock);
void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock);
+unsigned int __lockfunc
+_raw_spin_lock_bh_mask(raw_spinlock_t *lock, unsigned int mask) __acquires(lock);
void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
__acquires(lock);

@@ -38,6 +40,9 @@ int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock);
int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock);
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock);
+void __lockfunc
+_raw_spin_unlock_bh_mask(raw_spinlock_t *lock, unsigned int mask)
+ __releases(lock);
void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock);
void __lockfunc
_raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
@@ -49,6 +54,7 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)

#ifdef CONFIG_INLINE_SPIN_LOCK_BH
#define _raw_spin_lock_bh(lock) __raw_spin_lock_bh(lock)
+#define _raw_spin_lock_bh_mask(lock, mask) __raw_spin_lock_bh_mask(lock, mask)
#endif

#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ
@@ -73,6 +79,7 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)

#ifdef CONFIG_INLINE_SPIN_UNLOCK_BH
#define _raw_spin_unlock_bh(lock) __raw_spin_unlock_bh(lock)
+#define _raw_spin_unlock_bh_mask(lock, mask) __raw_spin_unlock_bh_mask(lock, mask)
#endif

#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ
@@ -136,6 +143,19 @@ static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
}

+static inline unsigned int __raw_spin_lock_bh_mask(raw_spinlock_t *lock,
+ unsigned int mask)
+{
+ unsigned int old_mask;
+
+ old_mask = local_bh_disable_mask(_RET_IP_, SOFTIRQ_LOCK_OFFSET, mask);
+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+ LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+
+ return old_mask;
+}
+
+
static inline void __raw_spin_lock(raw_spinlock_t *lock)
{
preempt_disable();
@@ -176,6 +196,14 @@ static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock)
__local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
}

+static inline void __raw_spin_unlock_bh_mask(raw_spinlock_t *lock,
+ unsigned int mask)
+{
+ spin_release(&lock->dep_map, 1, _RET_IP_);
+ do_raw_spin_unlock(lock);
+ local_bh_enable_mask(_RET_IP_, SOFTIRQ_LOCK_OFFSET, mask);
+}
+
static inline int __raw_spin_trylock_bh(raw_spinlock_t *lock)
{
__local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h
index d0d188861ad6..b900dcf46b26 100644
--- a/include/linux/spinlock_api_up.h
+++ b/include/linux/spinlock_api_up.h
@@ -33,6 +33,13 @@
#define __LOCK_BH(lock) \
do { __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_LOCK_OFFSET); ___LOCK(lock); } while (0)

+#define __LOCK_BH_MASK(lock, mask) ({ \
+ unsigned int ____old_mask; \
+ ____old_mask = local_bh_disable_mask(_THIS_IP_, SOFTIRQ_LOCK_OFFSET, mask); \
+ ___LOCK(lock); \
+ ____old_mask; \
+})
+
#define __LOCK_IRQ(lock) \
do { local_irq_disable(); __LOCK(lock); } while (0)

@@ -49,6 +56,10 @@
do { __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_LOCK_OFFSET); \
___UNLOCK(lock); } while (0)

+#define __UNLOCK_BH_MASK(lock, mask) \
+ do { local_bh_enable_mask(_THIS_IP_, SOFTIRQ_LOCK_OFFSET, mask); \
+ ___UNLOCK(lock); } while (0)
+
#define __UNLOCK_IRQ(lock) \
do { local_irq_enable(); __UNLOCK(lock); } while (0)

@@ -60,6 +71,7 @@
#define _raw_read_lock(lock) __LOCK(lock)
#define _raw_write_lock(lock) __LOCK(lock)
#define _raw_spin_lock_bh(lock) __LOCK_BH(lock)
+#define _raw_spin_lock_bh_mask(lock, mask) __LOCK_BH_MASK(lock, mask)
#define _raw_read_lock_bh(lock) __LOCK_BH(lock)
#define _raw_write_lock_bh(lock) __LOCK_BH(lock)
#define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock)
@@ -76,6 +88,7 @@
#define _raw_read_unlock(lock) __UNLOCK(lock)
#define _raw_write_unlock(lock) __UNLOCK(lock)
#define _raw_spin_unlock_bh(lock) __UNLOCK_BH(lock)
+#define _raw_spin_unlock_bh_mask(lock, mask) __UNLOCK_BH_MASK(lock, mask)
#define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock)
#define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock)
#define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock)
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index 936f3d14dd6b..4245cb3cda5a 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -170,6 +170,16 @@ void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_lock_bh);
#endif

+#ifndef CONFIG_INLINE_SPIN_LOCK_BH
+unsigned int __lockfunc _raw_spin_lock_bh_mask(raw_spinlock_t *lock,
+ unsigned int mask)
+{
+ return __raw_spin_lock_bh_mask(lock, mask);
+}
+EXPORT_SYMBOL(_raw_spin_lock_bh_mask);
+#endif
+
+
#ifdef CONFIG_UNINLINE_SPIN_UNLOCK
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock)
{
@@ -202,6 +212,15 @@ void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_unlock_bh);
#endif

+#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
+void __lockfunc _raw_spin_unlock_bh_mask(raw_spinlock_t *lock,
+ unsigned int mask)
+{
+ __raw_spin_unlock_bh_mask(lock, mask);
+}
+EXPORT_SYMBOL(_raw_spin_unlock_bh_mask);
+#endif
+
#ifndef CONFIG_INLINE_READ_TRYLOCK
int __lockfunc _raw_read_trylock(rwlock_t *lock)
{
--
2.21.0