Re:[PATCH 1/2] lib/crc: loongarch: Use hardware unaligned load in CRC32(C) loops

[Wentao Guan]

> With CONFIG_ARCH_STRICT_ALIGN=y the kernel is built with -mstrict-align,
> so get_unaligned_le{16,32,64}() compiles into a byte-by-byte load/shift/
> or sequence (~22 instructions per 8 bytes). On a Loongson-3C6000 machine
> this throttles the CRC pipeline to ~5.7 cycles per 8 bytes (2.8 GB/s),
> while the crcc.* instruction itself runs at ~1.1 cycles (latency) / ~0.8
> cycles (throughput) -- the CRC unit sits idle waiting for loads.
> 
> CPU cores with the UAL feature handle unaligned accesses in hardware,
> so issue real ld.{h,w,d} instructions when possible and keep the generic
> get_unaligned*() path for cores without UAL.
> 
> Measured on Loongson-3C6000 (single core):
> 
>     before: 2.8 GB/s
>     after:  8.4 GB/s   (~3x)
> 
> Signed-off-by: Tianyang Zhang <zhangtianyang@xxxxxxxxxxx>
> Signed-off-by: Huacai Chen <chenhuacai@xxxxxxxxxxx>
> ---
>  lib/crc/loongarch/crc32.h | 39 +++++++++++++++++++++++++++++++++------
>  1 file changed, 33 insertions(+), 6 deletions(-)
> 
> diff --git a/lib/crc/loongarch/crc32.h b/lib/crc/loongarch/crc32.h
> index d34fa4c68632..692a4165eca2 100644
> --- a/lib/crc/loongarch/crc32.h
> +++ b/lib/crc/loongarch/crc32.h
> @@ -24,15 +24,40 @@ do { \
>  #define CRC32(crc, value, size) _CRC32(crc, value, size, crc)
>  #define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc)
>  
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ual);
>  static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
>  
> +static inline u16 get_le16(const void *p)
> +{
> + if (static_branch_likely(&have_ual))
> + return *((__le16 *)p);
> + else
> + return get_unaligned_le16(p);
> +}
> +
> +static inline u32 get_le32(const void *p)
> +{
> + if (static_branch_likely(&have_ual))
> + return *((__le32 *)p);
> + else
> + return get_unaligned_le32(p);
> +}
> +
> +static inline u64 get_le64(const void *p)
> +{
> + if (static_branch_likely(&have_ual))
> + return *((__le64 *)p);
> + else
> + return get_unaligned_le64(p);
> +}
> +
>  static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
>  {
>   if (!static_branch_likely(&have_crc32))
>   return crc32_le_base(crc, p, len);
>  
>   while (len >= sizeof(u64)) {
> - u64 value = get_unaligned_le64(p);
> + u64 value = get_le64(p);
>  
>  CRC32(crc, value, d);
>  p += sizeof(u64);
> @@ -40,14 +65,14 @@ static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
>   }
>  
>   if (len & sizeof(u32)) {
> - u32 value = get_unaligned_le32(p);
> + u32 value = get_le32(p);
>  
>  CRC32(crc, value, w);
>  p += sizeof(u32);
>   }
>  
>   if (len & sizeof(u16)) {
> - u16 value = get_unaligned_le16(p);
> + u16 value = get_le16(p);
>  
>   CRC32(crc, value, h);
>   p += sizeof(u16);
> @@ -68,7 +93,7 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
>  return crc32c_base(crc, p, len);
>  
>   while (len >= sizeof(u64)) {
> - u64 value = get_unaligned_le64(p);
> + u64 value = get_le64(p);
>  
>   CRC32C(crc, value, d);
>   p += sizeof(u64);
> @@ -76,14 +101,14 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
>   }
>  
>   if (len & sizeof(u32)) {
> - u32 value = get_unaligned_le32(p);
> + u32 value = get_le32(p);
>  
>  CRC32C(crc, value, w);
>  p += sizeof(u32);
>   }
>  
>   if (len & sizeof(u16)) {
> - u16 value = get_unaligned_le16(p);
> + u16 value = get_le16(p);
>  
>   CRC32C(crc, value, h);
>   p += sizeof(u16);
> @@ -103,6 +128,8 @@ static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
>  #define crc32_mod_init_arch crc32_mod_init_arch
>  static void crc32_mod_init_arch(void)
>  {
> + if (cpu_has_ual)
> + static_branch_enable(&have_ual);
>   if (cpu_has_crc32)
>   static_branch_enable(&have_crc32);
>  }
> --
> 2.52.0

Reviewed-by: Wentao Guan <guanwentao@xxxxxxxxxxxxx>

Nice performance improved patch~.