[PATCH v2] x86: Introduce ASM flags to bitops
From: Uros Bizjak
Date: Mon Jul 27 2015 - 13:49:42 EST
From: Uros Bizjak <ubizjak@xxxxxxxxx>
This patch introduces GCC ASM flags to bitops.
The new functionality depends on __GCC_ASM_FLAG_OUTPUTS__ preprocessor
symbol, which is automatically set by GCC version 6 or later when
ASM flags feature is supported.
The improvement can be illustrated with following code snipped.
Instead of:
136d7: 48 0f a3 3d 00 00 00 bt %rdi,0x0(%rip)
136de: 00
136df: 19 ff sbb %edi,%edi
136e1: 85 ff test %edi,%edi
136e3: 0f 95 c0 setne %al
following code is generated:
13767: 48 0f a3 3d 00 00 00 bt %rdi,0x0(%rip)
1376e: 00
1376f: 0f 92 c0 setb %al
Similar improvement can be seen in following code:
7a6c: 48 0f a3 11 bt %rdx,(%rcx)
7a70: 19 d2 sbb %edx,%edx
7a72: 85 d2 test %edx,%edx
7a74: 74 eb je 7a61
which becomes:
7a8c: 48 0f a3 11 bt %rdx,(%rcx)
7a90: 73 ef jae 7a81
v2:
- Conditionally define CC_HAVE_ASM_FLAG_OUTPUTS in compiler-gcc.h
and use it instead of __GCC_ASM_FLAG_OUTPUTS__
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Signed-off-by: Uros Bizjak <ubizjak@xxxxxxxxx>
---
arch/x86/include/asm/bitops.h | 26 ++++++++++++++++++++++++--
arch/x86/include/asm/percpu.h | 18 +++++++++++++++++-
arch/x86/include/asm/signal.h | 6 ++++++
arch/x86/include/asm/sync_bitops.h | 18 ++++++++++++++++++
include/linux/compiler-gcc.h | 9 +++++++++
5 files changed, 74 insertions(+), 3 deletions(-)
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index cfe3b95..3b66d88 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -232,10 +232,16 @@ static inline int __test_and_set_bit(long nr, volatile unsigned long *addr)
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm("bts %2,%1"
+ : "=@ccc" (oldbit), ADDR
+ : "Ir" (nr));
+#else
asm("bts %2,%1\n\t"
"sbb %0,%0"
: "=r" (oldbit), ADDR
: "Ir" (nr));
+#endif
return oldbit;
}
@@ -272,10 +278,16 @@ static inline int __test_and_clear_bit(long nr, volatile unsigned long *addr)
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm volatile("btr %2,%1"
+ : "=@ccc" (oldbit), ADDR
+ : "Ir" (nr));
+#else
asm volatile("btr %2,%1\n\t"
"sbb %0,%0"
: "=r" (oldbit), ADDR
: "Ir" (nr));
+#endif
return oldbit;
}
@@ -284,11 +296,16 @@ static inline int __test_and_change_bit(long nr, volatile unsigned long *addr)
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm volatile("btc %2,%1"
+ : "=@ccc" (oldbit), ADDR
+ : "Ir" (nr) : "memory");
+#else
asm volatile("btc %2,%1\n\t"
"sbb %0,%0"
: "=r" (oldbit), ADDR
: "Ir" (nr) : "memory");
-
+#endif
return oldbit;
}
@@ -315,11 +332,16 @@ static inline int variable_test_bit(long nr, volatile const unsigned long *addr)
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm volatile("bt %2,%1"
+ : "=@ccc" (oldbit)
+ : "m" (*(unsigned long *)addr), "Ir" (nr));
+#else
asm volatile("bt %2,%1\n\t"
"sbb %0,%0"
: "=r" (oldbit)
: "m" (*(unsigned long *)addr), "Ir" (nr));
-
+#endif
return oldbit;
}
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index e0ba66c..4879f36 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -508,6 +508,16 @@ do { \
#endif
/* This is not atomic against other CPUs -- CPU preemption needs to be off */
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+#define x86_test_and_clear_bit_percpu(bit, var) \
+({ \
+ int old__; \
+ asm volatile("btr %2,"__percpu_arg(1) \
+ : "=@ccc" (old__), "+m" (var) \
+ : "dIr" (bit)); \
+ old__; \
+})
+#else
#define x86_test_and_clear_bit_percpu(bit, var) \
({ \
int old__; \
@@ -516,6 +526,7 @@ do { \
: "dIr" (bit)); \
old__; \
})
+#endif
static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
const unsigned long __percpu *addr)
@@ -534,11 +545,16 @@ static inline int x86_this_cpu_variable_test_bit(int nr,
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm volatile("bt "__percpu_arg(2)",%1"
+ : "=@ccc" (oldbit)
+ : "m" (*(unsigned long *)addr), "Ir" (nr));
+#else
asm volatile("bt "__percpu_arg(2)",%1\n\t"
"sbb %0,%0"
: "=r" (oldbit)
: "m" (*(unsigned long *)addr), "Ir" (nr));
-
+#endif
return oldbit;
}
diff --git a/arch/x86/include/asm/signal.h b/arch/x86/include/asm/signal.h
index 31eab86..1792de8 100644
--- a/arch/x86/include/asm/signal.h
+++ b/arch/x86/include/asm/signal.h
@@ -82,8 +82,14 @@ static inline int __const_sigismember(sigset_t *set, int _sig)
static inline int __gen_sigismember(sigset_t *set, int _sig)
{
int ret;
+
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm("btl %2,%1"
+ : "=@ccc"(ret) : "m"(*set), "Ir"(_sig-1));
+#else
asm("btl %2,%1\n\tsbbl %0,%0"
: "=r"(ret) : "m"(*set), "Ir"(_sig-1) : "cc");
+#endif
return ret;
}
diff --git a/arch/x86/include/asm/sync_bitops.h b/arch/x86/include/asm/sync_bitops.h
index f28a24b..0af45ee 100644
--- a/arch/x86/include/asm/sync_bitops.h
+++ b/arch/x86/include/asm/sync_bitops.h
@@ -81,9 +81,15 @@ static inline int sync_test_and_set_bit(long nr, volatile unsigned long *addr)
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm volatile("lock; bts %2,%1"
+ : "=@ccc" (oldbit), "+m" (ADDR)
+ : "Ir" (nr) : "memory");
+#else
asm volatile("lock; bts %2,%1\n\tsbbl %0,%0"
: "=r" (oldbit), "+m" (ADDR)
: "Ir" (nr) : "memory");
+#endif
return oldbit;
}
@@ -99,9 +105,15 @@ static inline int sync_test_and_clear_bit(long nr, volatile unsigned long *addr)
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm volatile("lock; btr %2,%1"
+ : "=@ccc" (oldbit), "+m" (ADDR)
+ : "Ir" (nr) : "memory");
+#else
asm volatile("lock; btr %2,%1\n\tsbbl %0,%0"
: "=r" (oldbit), "+m" (ADDR)
: "Ir" (nr) : "memory");
+#endif
return oldbit;
}
@@ -117,9 +129,15 @@ static inline int sync_test_and_change_bit(long nr, volatile unsigned long *addr
{
int oldbit;
+#ifdef CC_HAVE_ASM_FLAG_OUTPUTS
+ asm volatile("lock; btc %2,%1"
+ : "=@ccc" (oldbit), "+m" (ADDR)
+ : "Ir" (nr) : "memory");
+#else
asm volatile("lock; btc %2,%1\n\tsbbl %0,%0"
: "=r" (oldbit), "+m" (ADDR)
: "Ir" (nr) : "memory");
+#endif
return oldbit;
}
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index dfaa7b3..0feb83e 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -122,6 +122,15 @@
#define __maybe_unused __attribute__((unused))
#define __always_unused __attribute__((unused))
+/*
+ * The compiler defines __GCC_ASM_FLAG_OUTPUTS__ when a special form
+ * of output operand exists by which conditions in the flags register
+ * may be outputs of the asm.
+ */
+#ifdef __GCC_ASM_FLAG_OUTPUTS__
+# define CC_HAVE_ASM_FLAG_OUTPUTS
+#endif
+
/* gcc version specific checks */
#if GCC_VERSION < 30200
--
2.4.3
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/