Re: [PATCH v4 8/8] s390: Add support for DCACHE_WORD_ACCESS (again)

[Sven Schnelle]

Heiko Carstens <hca@xxxxxxxxxxxxx> writes:

> Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to
> speed up string operations in fs/dcache.c and fs/namei.c.
>
> With the secure storage access exception cases addressed, add support
> for DCACHE_WORD_ACCESS again.
>
> Tested-by: Christian Borntraeger <borntraeger@xxxxxxxxxxxxx>
> Signed-off-by: Heiko Carstens <hca@xxxxxxxxxxxxx>
> ---
>  arch/s390/Kconfig                      |  1 +
>  arch/s390/include/asm/asm-extable.h    |  4 ++++
>  arch/s390/include/asm/word-at-a-time.h | 22 ++++++++++++++++++++++
>  arch/s390/mm/extable.c                 | 18 ++++++++++++++++++
>  4 files changed, 45 insertions(+)
>
> diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
> index 84404e6778d5..fe81c539ef2f 100644
> --- a/arch/s390/Kconfig
> +++ b/arch/s390/Kconfig
> @@ -163,6 +163,7 @@ config S390
>  	select ARCH_WANTS_THP_SWAP
>  	select BUILDTIME_TABLE_SORT
>  	select CLONE_BACKWARDS2
> +	select DCACHE_WORD_ACCESS if !KMSAN
>  	select DYNAMIC_FTRACE if FUNCTION_TRACER
>  	select FUNCTION_ALIGNMENT_8B if CC_IS_GCC
>  	select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC
> diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h
> index 99748c20e767..d23ea0c94e4e 100644
> --- a/arch/s390/include/asm/asm-extable.h
> +++ b/arch/s390/include/asm/asm-extable.h
> @@ -12,6 +12,7 @@
>  #define EX_TYPE_UA_FAULT	3
>  #define EX_TYPE_UA_LOAD_REG	5
>  #define EX_TYPE_UA_LOAD_REGPAIR	6
> +#define EX_TYPE_ZEROPAD		7
>  #define EX_TYPE_FPC		8
>  #define EX_TYPE_UA_MVCOS_TO	9
>  #define EX_TYPE_UA_MVCOS_FROM	10
> @@ -79,6 +80,9 @@
>  #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero)	\
>  	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0)
>  
> +#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr)		\
> +	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0)
> +
>  #define EX_TABLE_FPC(_fault, _target)					\
>  	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0)
>  
> diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h
> index e9287036392d..eaa19dee7699 100644
> --- a/arch/s390/include/asm/word-at-a-time.h
> +++ b/arch/s390/include/asm/word-at-a-time.h
> @@ -4,6 +4,7 @@
>  
>  #include <linux/bitops.h>
>  #include <linux/wordpart.h>
> +#include <asm/asm-extable.h>
>  #include <asm/bitsperlong.h>
>  
>  struct word_at_a_time {
> @@ -40,4 +41,25 @@ static inline unsigned long zero_bytemask(unsigned long data)
>  	return ~1UL << data;
>  }
>  
> +/*
> + * Load an unaligned word from kernel space.
> + *
> + * In the (very unlikely) case of the word being a page-crosser
> + * and the next page not being mapped, take the exception and
> + * return zeroes in the non-existing part.
> + */
> +static inline unsigned long load_unaligned_zeropad(const void *addr)
> +{
> +	unsigned long data;
> +
> +	asm_inline volatile(
> +		"0:	lg	%[data],0(%[addr])\n"
> +		"1:	nopr	%%r7\n"
> +		EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])
> +		EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr])
> +		: [data] "=d" (data)
> +		: [addr] "a" (addr), "m" (*(unsigned long *)addr));
> +	return data;
> +}
> +
>  #endif /* _ASM_WORD_AT_A_TIME_H */
> diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c
> index 063b4346742d..7498e858c401 100644
> --- a/arch/s390/mm/extable.c
> +++ b/arch/s390/mm/extable.c
> @@ -50,6 +50,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex,
>  	return true;
>  }
>  
> +static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs)
> +{
> +	unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
> +	unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data);
> +	unsigned long data, addr, offset;
> +
> +	addr = regs->gprs[reg_addr];
> +	offset = addr & (sizeof(unsigned long) - 1);
> +	addr &= ~(sizeof(unsigned long) - 1);
> +	data = *(unsigned long *)addr;
> +	data <<= BITS_PER_BYTE * offset;
> +	regs->gprs[reg_data] = data;
> +	regs->psw.addr = extable_fixup(ex);
> +	return true;
> +}
> +
>  static bool ex_handler_fpc(const struct exception_table_entry *ex, struct pt_regs *regs)
>  {
>  	fpu_sfpc(0);
> @@ -118,6 +134,8 @@ bool fixup_exception(struct pt_regs *regs)
>  		return ex_handler_ua_load_reg(ex, false, regs);
>  	case EX_TYPE_UA_LOAD_REGPAIR:
>  		return ex_handler_ua_load_reg(ex, true, regs);
> +	case EX_TYPE_ZEROPAD:
> +		return ex_handler_zeropad(ex, regs);
>  	case EX_TYPE_FPC:
>  		return ex_handler_fpc(ex, regs);
>  	case EX_TYPE_UA_MVCOS_TO:

Reviewed-by: Sven Schnelle <svens@xxxxxxxxxxxxx>