[PATCH 2/2] x86/bitops: Fix false output register dependency of TZCNT insn

[Uros Bizjak]

On Haswell and later Intel processors, the TZCNT instruction appears
to have a false dependency on the destination register. Even though
the instruction only writes to it, the instruction will wait until
destination is ready before executing. This false dependency
was fixed for Skylake (and later) processors.

Fix false dependency by clearing the destination register first.

The x86_64 defconfig object size increases by 4215 bytes:

	    text           data     bss      dec            hex filename
	27342396        4642999  814852 32800247        1f47df7 vmlinux-old.o
	27346611        4643015  814852 32804478        1f48e7e vmlinux-new.o

Signed-off-by: Uros Bizjak <ubizjak@xxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Borislav Petkov <bp@xxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>
Cc: "H. Peter Anvin" <hpa@xxxxxxxxx>
---
 arch/x86/include/asm/bitops.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index bbaf75ea6703..7e3d1cc97c5a 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -248,8 +248,9 @@ arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
 
 static __always_inline unsigned long variable__ffs(unsigned long word)
 {
-	asm("tzcnt %1,%0"
-		: "=r" (word)
+	asm("xor %k0,%k0\n\t" /* avoid false dependency on dest register */
+	    "tzcnt %1,%0"
+		: "=&r" (word)
 		: ASM_INPUT_RM (word));
 	return word;
 }
@@ -267,8 +268,9 @@ static __always_inline unsigned long variable__ffs(unsigned long word)
 
 static __always_inline unsigned long variable_ffz(unsigned long word)
 {
-	asm("tzcnt %1,%0"
-		: "=r" (word)
+	asm("xor %k0,%k0\n\t" /* avoid false dependency on dest register */
+	    "tzcnt %1,%0"
+		: "=&r" (word)
 		: "r" (~word));
 	return word;
 }
-- 
2.42.0