[PATCH 1/2] lib/crc: loongarch: Use hardware unaligned load in CRC32(C) loops

[Huacai Chen]

With CONFIG_ARCH_STRICT_ALIGN=y the kernel is built with -mstrict-align,
so get_unaligned_le{16,32,64}() compiles into a byte-by-byte load/shift/
or sequence (~22 instructions per 8 bytes). On a Loongson-3C6000 machine
this throttles the CRC pipeline to ~5.7 cycles per 8 bytes (2.8 GB/s),
while the crcc.* instruction itself runs at ~1.1 cycles (latency) / ~0.8
cycles (throughput) -- the CRC unit sits idle waiting for loads.

CPU cores with the UAL feature handle unaligned accesses in hardware,
so issue real ld.{h,w,d} instructions when possible and keep the generic
get_unaligned*() path for cores without UAL.

Measured on Loongson-3C6000 (single core):

    before: 2.8 GB/s
    after:  8.4 GB/s   (~3x)

Signed-off-by: Tianyang Zhang <zhangtianyang@xxxxxxxxxxx>
Signed-off-by: Huacai Chen <chenhuacai@xxxxxxxxxxx>
---
 lib/crc/loongarch/crc32.h | 39 +++++++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/lib/crc/loongarch/crc32.h b/lib/crc/loongarch/crc32.h
index d34fa4c68632..692a4165eca2 100644
--- a/lib/crc/loongarch/crc32.h
+++ b/lib/crc/loongarch/crc32.h
@@ -24,15 +24,40 @@ do {							\
 #define CRC32(crc, value, size)		_CRC32(crc, value, size, crc)
 #define CRC32C(crc, value, size)	_CRC32(crc, value, size, crcc)
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ual);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
 
+static inline u16 get_le16(const void *p)
+{
+	if (static_branch_likely(&have_ual))
+		return *((__le16 *)p);
+	else
+		return get_unaligned_le16(p);
+}
+
+static inline u32 get_le32(const void *p)
+{
+	if (static_branch_likely(&have_ual))
+		return *((__le32 *)p);
+	else
+		return get_unaligned_le32(p);
+}
+
+static inline u64 get_le64(const void *p)
+{
+	if (static_branch_likely(&have_ual))
+		return *((__le64 *)p);
+	else
+		return get_unaligned_le64(p);
+}
+
 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 {
 	if (!static_branch_likely(&have_crc32))
 		return crc32_le_base(crc, p, len);
 
 	while (len >= sizeof(u64)) {
-		u64 value = get_unaligned_le64(p);
+		u64 value = get_le64(p);
 
 		CRC32(crc, value, d);
 		p += sizeof(u64);
@@ -40,14 +65,14 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 	}
 
 	if (len & sizeof(u32)) {
-		u32 value = get_unaligned_le32(p);
+		u32 value = get_le32(p);
 
 		CRC32(crc, value, w);
 		p += sizeof(u32);
 	}
 
 	if (len & sizeof(u16)) {
-		u16 value = get_unaligned_le16(p);
+		u16 value = get_le16(p);
 
 		CRC32(crc, value, h);
 		p += sizeof(u16);
@@ -68,7 +93,7 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 		return crc32c_base(crc, p, len);
 
 	while (len >= sizeof(u64)) {
-		u64 value = get_unaligned_le64(p);
+		u64 value = get_le64(p);
 
 		CRC32C(crc, value, d);
 		p += sizeof(u64);
@@ -76,14 +101,14 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 	}
 
 	if (len & sizeof(u32)) {
-		u32 value = get_unaligned_le32(p);
+		u32 value = get_le32(p);
 
 		CRC32C(crc, value, w);
 		p += sizeof(u32);
 	}
 
 	if (len & sizeof(u16)) {
-		u16 value = get_unaligned_le16(p);
+		u16 value = get_le16(p);
 
 		CRC32C(crc, value, h);
 		p += sizeof(u16);
@@ -103,6 +128,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 #define crc32_mod_init_arch crc32_mod_init_arch
 static void crc32_mod_init_arch(void)
 {
+	if (cpu_has_ual)
+		static_branch_enable(&have_ual);
 	if (cpu_has_crc32)
 		static_branch_enable(&have_crc32);
 }
-- 
2.52.0