[PATCH v2 07/17] KVM: arm64: Export kvm_handle_user_mem_abort()
From: Gavin Shan
Date: Tue Feb 09 2021 - 00:09:06 EST
The main work is handled by user_mem_abort(). After asynchronous
page fault is supported, one page fault need to be handled with
two calls to this function. It means the page fault needs to be
replayed asynchronously in that case. This renames the function
to kvm_handle_user_mem_abort() can exports it. Besides, there are
more changes introduced in order to accommodate asynchronous page
fault:
* Add arguments @esr and @prefault to user_mem_abort(). @esr
is the cached value of ESR_EL2 instead of fetching from the
current vCPU when the page fault is replayed in scenario of
asynchronous page fault. @prefault is used to indicate the
page fault is replayed one or not.
* Define helper functions esr_dbat_*() in asm/esr.h to extract
or check various fields of the passed ESR_EL2 value because
those helper functions defined in asm/kvm_emulate.h assumes
the ESR_EL2 value has been cached in vCPU struct. It won't
be true on handling the replayed page fault in scenario of
asynchronous page fault.
* Some helper functions defined in asm/kvm_emulate.h are used
by mmu.c only and seem not to be used by other source file
in near future. They are moved to mmu.c and renamed accordingly.
kvm_vcpu_trap_is_exec_fault()
is_exec_fault()
kvm_is_write_fault()
is_write_fault()
kvm_vcpu_trap_get_fault_level()
Replaced by esr_dabt_get_fault_level()
Signed-off-by: Gavin Shan <gshan@xxxxxxxxxx>
---
arch/arm64/include/asm/esr.h | 6 ++++
arch/arm64/include/asm/kvm_emulate.h | 27 ++---------------
arch/arm64/include/asm/kvm_host.h | 4 +++
arch/arm64/kvm/mmu.c | 43 ++++++++++++++++++++++------
4 files changed, 48 insertions(+), 32 deletions(-)
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 29f97eb3dad4..db46eb58c633 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -321,8 +321,14 @@
ESR_ELx_CP15_32_ISS_DIR_READ)
#ifndef __ASSEMBLY__
+#include <linux/bitfield.h>
#include <asm/types.h>
+#define esr_dabt_get_fault_type(esr) (esr & ESR_ELx_FSC_TYPE)
+#define esr_dabt_get_fault_level(esr) (FIELD_GET(ESR_ELx_FSC_LEVEL, esr))
+#define esr_dabt_is_wnr(esr) (!!(FIELD_GET(ESR_ELx_WNR, esr)))
+#define esr_dabt_is_s1ptw(esr) (!!(FIELD_GET(ESR_ELx_S1PTW, esr)))
+
static inline bool esr_is_data_abort(u32 esr)
{
const u32 ec = ESR_ELx_EC(esr);
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 0ef213b715a5..119b953828a2 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -282,13 +282,13 @@ static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
{
- return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW);
+ return esr_dabt_is_s1ptw(kvm_vcpu_get_esr(vcpu));
}
/* Always check for S1PTW *before* using this. */
static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
{
- return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR;
+ return esr_dabt_is_wnr(kvm_vcpu_get_esr(vcpu));
}
static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
@@ -317,11 +317,6 @@ static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
}
-static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
-{
- return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
-}
-
static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC;
@@ -329,12 +324,7 @@ static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
static __always_inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
{
- return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_TYPE;
-}
-
-static __always_inline u8 kvm_vcpu_trap_get_fault_level(const struct kvm_vcpu *vcpu)
-{
- return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_LEVEL;
+ return esr_dabt_get_fault_type(kvm_vcpu_get_esr(vcpu));
}
static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu)
@@ -362,17 +352,6 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
return ESR_ELx_SYS64_ISS_RT(esr);
}
-static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
-{
- if (kvm_vcpu_abt_iss1tw(vcpu))
- return true;
-
- if (kvm_vcpu_trap_is_iabt(vcpu))
- return false;
-
- return kvm_vcpu_dabt_iswrite(vcpu);
-}
-
static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
{
return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 01eda5c84600..00b30b7554e5 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -589,6 +589,10 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
struct kvm_vcpu_events *events);
#define KVM_ARCH_WANT_MMU_NOTIFIER
+int kvm_handle_user_mem_abort(struct kvm_vcpu *vcpu,
+ struct kvm_memory_slot *memslot,
+ phys_addr_t fault_ipa, unsigned long hva,
+ unsigned int esr, bool prefault);
int kvm_unmap_hva_range(struct kvm *kvm,
unsigned long start, unsigned long end, unsigned flags);
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 7d2257cc5438..ff8480b68fb8 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -738,9 +738,34 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot,
return PAGE_SIZE;
}
-static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
- struct kvm_memory_slot *memslot, unsigned long hva,
- unsigned long fault_status)
+static inline bool is_exec_fault(unsigned int esr)
+{
+ if (ESR_ELx_EC(esr) != ESR_ELx_EC_IABT_LOW)
+ return false;
+
+ if (esr_dabt_is_s1ptw(esr))
+ return false;
+
+ return true;
+}
+
+static inline bool is_write_fault(unsigned int esr)
+{
+ if (esr_dabt_is_s1ptw(esr))
+ return true;
+
+ if (ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW)
+ return false;
+
+ return esr_dabt_is_wnr(esr);
+}
+
+int kvm_handle_user_mem_abort(struct kvm_vcpu *vcpu,
+ struct kvm_memory_slot *memslot,
+ phys_addr_t fault_ipa,
+ unsigned long hva,
+ unsigned int esr,
+ bool prefault)
{
int ret = 0;
bool write_fault, writable, force_pte = false;
@@ -754,14 +779,15 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
gfn_t gfn;
kvm_pfn_t pfn;
bool logging_active = memslot_is_logging(memslot);
- unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);
+ unsigned long fault_status = esr_dabt_get_fault_type(esr);
+ unsigned long fault_level = esr_dabt_get_fault_level(esr);
unsigned long vma_pagesize, fault_granule;
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
struct kvm_pgtable *pgt;
fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level);
- write_fault = kvm_is_write_fault(vcpu);
- exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu);
+ write_fault = is_write_fault(kvm_vcpu_get_esr(vcpu));
+ exec_fault = is_exec_fault(kvm_vcpu_get_esr(vcpu));
VM_BUG_ON(write_fault && exec_fault);
if (fault_status == FSC_PERM && !write_fault && !exec_fault) {
@@ -993,7 +1019,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
gfn = fault_ipa >> PAGE_SHIFT;
memslot = gfn_to_memslot(vcpu->kvm, gfn);
hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
- write_fault = kvm_is_write_fault(vcpu);
+ write_fault = is_write_fault(kvm_vcpu_get_esr(vcpu));
if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
/*
* The guest has put either its instructions or its page-tables
@@ -1048,7 +1074,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
goto out_unlock;
}
- ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
+ ret = kvm_handle_user_mem_abort(vcpu, memslot, fault_ipa, hva,
+ kvm_vcpu_get_esr(vcpu), false);
if (ret == 0)
ret = 1;
out:
--
2.23.0