[PATCH v6 6/8] string: introduce memcpy_nt() helpers

[Li Zhe]

Introduce memcpy_nt() and memcpy_nt_drain() for write-once copy sites
that want a named non-temporal copy primitive plus an explicit drain
step.

On x86_64, override both helpers in arch/x86/include/asm/string_64.h
using the usual self-macro pattern, next to the existing
memcpy_flushcache() backend that memcpy_nt() wraps. The x86_64
implementation maps memcpy_nt() to memcpy_flushcache() and uses wmb()
for memcpy_nt_drain(), because that backend issues MOVNTI stores and
callers need an ordering point before later normal stores that depend
on those writes becoming visible.

include/linux/string.h provides the generic fallback under
memcpy_nt() as plain memcpy() and leaves memcpy_nt_drain() empty, so
architectures that do not override memcpy_nt() do not pay an
unconditional barrier. Architectures that later grow a specialized
memcpy_nt() backend can override memcpy_nt_drain() with whatever
drain primitive their memory-ordering rules require.

The immediate user is the ZONE_DEVICE template-copy path. It populates
struct page descriptors in a write-once pattern, so a regular cached
memcpy() can incur avoidable write-allocate traffic and cache
pollution for data with little near-term reuse.

Signed-off-by: Li Zhe <lizhe.67@xxxxxxxxxxxxx>
---
 arch/x86/include/asm/string_64.h | 22 ++++++++++++++++++++++
 include/linux/string.h           | 23 +++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 4635616863f5..6cb9e0ac7fa0 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -4,6 +4,7 @@
 
 #ifdef __KERNEL__
 #include <linux/jump_label.h>
+#include <asm/barrier.h>
 
 /* Written 2002 by Andi Kleen */
 
@@ -100,6 +101,27 @@ static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t
 	}
 	__memcpy_flushcache(dst, src, cnt);
 }
+
+#define memcpy_nt memcpy_nt
+/*
+ * Reuse the existing x86 flushcache backend as the nt copy primitive.
+ * Callers pair it with memcpy_nt_drain() when later stores must be
+ * ordered after the copy.
+ */
+static __always_inline void memcpy_nt(void *dst, const void *src, size_t cnt)
+{
+	memcpy_flushcache(dst, src, cnt);
+}
+
+#define memcpy_nt_drain memcpy_nt_drain
+static __always_inline void memcpy_nt_drain(void)
+{
+	/*
+	 * Order the prior MOVNTI stores issued by memcpy_flushcache()
+	 * before later normal stores.
+	 */
+	wmb();
+}
 #endif
 
 #endif /* __KERNEL__ */
diff --git a/include/linux/string.h b/include/linux/string.h
index 5702daca4326..a109b2f86ca6 100644
--- a/include/linux/string.h
+++ b/include/linux/string.h
@@ -278,6 +278,29 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
 }
 #endif
 
+#ifndef memcpy_nt
+/*
+ * memcpy_nt() requests a non-temporal copy when the architecture has a
+ * suitable backend. Architectures that do not override it fall back to
+ * memcpy().
+ */
+static inline void memcpy_nt(void *dst, const void *src, size_t cnt)
+{
+	memcpy(dst, src, cnt);
+}
+#endif
+
+#ifndef memcpy_nt_drain
+/*
+ * Callers use memcpy_nt_drain() before later normal stores that need to
+ * be ordered after memcpy_nt(). Architectures without a specialized
+ * backend can leave it empty.
+ */
+static inline void memcpy_nt_drain(void)
+{
+}
+#endif
+
 void *memchr_inv(const void *s, int c, size_t n);
 char *strreplace(char *str, char old, char new);
 
-- 
2.20.1