Re: [PATCH RESEND v4 2/3] KVM: arm64: Introduce KVM_PGTABLE_WALK_SKIP_LEVEL* walk flags
From: Oliver Upton
Date: Fri Nov 06 2026 - 13:14:04 EST
Hi Leo,
On Fri, Nov 06, 2026 at 04:12:37PM +0000, Leonardo Bras wrote:
> Add new walking flags KVM_PGTABLE_WALK_SKIP_LEVEL{0-3}.
> When used, they make the kvm_pgtable_walk() skip the given level, as well
> as levels higher than that.
>
> Being able to skip levels substancially decreases the runtime of a
> pagetable walk when the callback function does nothing on that level.
>
> Signed-off-by: Leonardo Bras <leo.bras@xxxxxxx>
I'm still not crazy about using a flag per level... This is what I have
stuffed on top locally. I've got a similar oneline fix for patch #3 to
initialize the field with an offset.
Any opinions?
Thanks,
Oliver
commit cad2070228c4305b5a83ce0a4cfa4aaa3dc0eb44
Author: Oliver Upton <oupton@xxxxxxxxxx>
Date: Fri Nov 6 10:03:14 2026 -0800
squash! KVM: arm64: Introduce KVM_PGTABLE_WALK_SKIP_LEVEL* walk flags
KVM: arm64: Add support for early termination of table walk recursion
Add a new field to the walk flags that represents an offset from the
last level of the page table hierarchy at which recursion should
terminate.
Signed-off-by: Leonardo Bras <leo.bras@xxxxxxx>
[oliver: pass a numeric value instead of a flag per level]
Signed-off-by: Oliver Upton <oupton@xxxxxxxxxx>
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 1d7a9125e522..18dcce3c506e 100644
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -326,10 +326,8 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,
* @KVM_PGTABLE_WALK_SKIP_CMO: Visit and update table entries
* without Cache maintenance
* operations required.
- * @KVM_PGTABLE_WALK_SKIP_LEVEL0: Skip visiting level 0+ entries
- * @KVM_PGTABLE_WALK_SKIP_LEVEL1: Skip visiting level 1+ entries
- * @KVM_PGTABLE_WALK_SKIP_LEVEL2: Skip visiting level 2+ entries
- * @KVM_PGTABLE_WALK_SKIP_LEVEL3: Skip visiting level 3 entries
+ * @KVM_PGTABLE_WALK_SKIP_OFFSET: Offset from KVM_PGTABLE_LAST_LEVEL at
+ * which the walk should stop descending
*/
enum kvm_pgtable_walk_flags {
KVM_PGTABLE_WALK_LEAF = BIT(0),
@@ -339,18 +337,9 @@ enum kvm_pgtable_walk_flags {
KVM_PGTABLE_WALK_IGNORE_EAGAIN = BIT(4),
KVM_PGTABLE_WALK_SKIP_BBM_TLBI = BIT(5),
KVM_PGTABLE_WALK_SKIP_CMO = BIT(6),
- /* SKIP_LEVEL* flags should not be split nor reordered */
- KVM_PGTABLE_WALK_SKIP_LEVEL0 = BIT(7),
- KVM_PGTABLE_WALK_SKIP_LEVEL1 = BIT(8),
- KVM_PGTABLE_WALK_SKIP_LEVEL2 = BIT(9),
- KVM_PGTABLE_WALK_SKIP_LEVEL3 = BIT(10),
+ KVM_PGTABLE_WALK_SKIP_OFFSET = GENMASK(9, 7),
};
-#define KVM_PGTABLE_WALK_SKIP_LEVELS (KVM_PGTABLE_WALK_SKIP_LEVEL0 | \
- KVM_PGTABLE_WALK_SKIP_LEVEL1 | \
- KVM_PGTABLE_WALK_SKIP_LEVEL2 | \
- KVM_PGTABLE_WALK_SKIP_LEVEL3 )
-
struct kvm_pgtable_visit_ctx {
kvm_pte_t *ptep;
kvm_pte_t old;
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 6aaf2696c99a..3008817da6c5 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -147,9 +147,12 @@ static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker,
static bool kvm_pgtable_skip_level(s8 level, enum kvm_pgtable_walk_flags flags)
{
- u32 skip = FIELD_GET(KVM_PGTABLE_WALK_SKIP_LEVELS, flags);
+ u8 offset = FIELD_GET(KVM_PGTABLE_WALK_SKIP_OFFSET, flags);
- return unlikely(skip && level >= ffs(skip) - 1);
+ BUILD_BUG_ON(!FIELD_FIT(KVM_PGTABLE_WALK_SKIP_OFFSET,
+ KVM_PGTABLE_LAST_LEVEL - KVM_PGTABLE_FIRST_LEVEL));
+
+ return unlikely(level > KVM_PGTABLE_LAST_LEVEL - offset);
}
static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
> ---
> arch/arm64/include/asm/kvm_pgtable.h | 14 ++++++++++++++
> arch/arm64/kvm/hyp/pgtable.c | 16 +++++++++++++---
> 2 files changed, 27 insertions(+), 3 deletions(-)
>
> diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
> index 41a8687938eb..5f37ceef8d63 100644
> --- a/arch/arm64/include/asm/kvm_pgtable.h
> +++ b/arch/arm64/include/asm/kvm_pgtable.h
> @@ -311,31 +311,45 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,
> * @KVM_PGTABLE_WALK_SHARED: Indicates the page-tables may be shared
> * with other software walkers.
> * @KVM_PGTABLE_WALK_IGNORE_EAGAIN: Don't terminate the walk early if
> * the walker returns -EAGAIN.
> * @KVM_PGTABLE_WALK_SKIP_BBM_TLBI: Visit and update table entries
> * without Break-before-make's
> * TLB invalidation.
> * @KVM_PGTABLE_WALK_SKIP_CMO: Visit and update table entries
> * without Cache maintenance
> * operations required.
> + * @KVM_PGTABLE_WALK_SKIP_LEVEL0: Skip visiting level 0+ entries
> + * @KVM_PGTABLE_WALK_SKIP_LEVEL1: Skip visiting level 1+ entries
> + * @KVM_PGTABLE_WALK_SKIP_LEVEL2: Skip visiting level 2+ entries
> + * @KVM_PGTABLE_WALK_SKIP_LEVEL3: Skip visiting level 3 entries
> */
> enum kvm_pgtable_walk_flags {
> KVM_PGTABLE_WALK_LEAF = BIT(0),
> KVM_PGTABLE_WALK_TABLE_PRE = BIT(1),
> KVM_PGTABLE_WALK_TABLE_POST = BIT(2),
> KVM_PGTABLE_WALK_SHARED = BIT(3),
> KVM_PGTABLE_WALK_IGNORE_EAGAIN = BIT(4),
> KVM_PGTABLE_WALK_SKIP_BBM_TLBI = BIT(5),
> KVM_PGTABLE_WALK_SKIP_CMO = BIT(6),
> + /* SKIP_LEVEL* flags should not be split nor reordered */
> + KVM_PGTABLE_WALK_SKIP_LEVEL0 = BIT(7),
> + KVM_PGTABLE_WALK_SKIP_LEVEL1 = BIT(8),
> + KVM_PGTABLE_WALK_SKIP_LEVEL2 = BIT(9),
> + KVM_PGTABLE_WALK_SKIP_LEVEL3 = BIT(10),
> };
>
> +#define KVM_PGTABLE_WALK_SKIP_LEVELS (KVM_PGTABLE_WALK_SKIP_LEVEL0 | \
> + KVM_PGTABLE_WALK_SKIP_LEVEL1 | \
> + KVM_PGTABLE_WALK_SKIP_LEVEL2 | \
> + KVM_PGTABLE_WALK_SKIP_LEVEL3 )
> +
> struct kvm_pgtable_visit_ctx {
> kvm_pte_t *ptep;
> kvm_pte_t old;
> void *arg;
> struct kvm_pgtable_mm_ops *mm_ops;
> u64 start;
> u64 addr;
> u64 end;
> s8 level;
> enum kvm_pgtable_walk_flags flags;
> diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
> index 7a51f78b6446..80744478c810 100644
> --- a/arch/arm64/kvm/hyp/pgtable.c
> +++ b/arch/arm64/kvm/hyp/pgtable.c
> @@ -137,20 +137,27 @@ static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker,
> * Ignore the return code altogether for walkers outside a fault handler
> * (e.g. write protecting a range of memory) and chug along with the
> * page table walk.
> */
> if (r == -EAGAIN)
> return walker->flags & KVM_PGTABLE_WALK_IGNORE_EAGAIN;
>
> return !r;
> }
>
> +static bool kvm_pgtable_skip_level(s8 level, enum kvm_pgtable_walk_flags flags)
> +{
> + u32 skip = FIELD_GET(KVM_PGTABLE_WALK_SKIP_LEVELS, flags);
> +
> + return unlikely(skip && level >= ffs(skip) - 1);
> +}
> +
> static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
> struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level);
>
> static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
> struct kvm_pgtable_mm_ops *mm_ops,
> kvm_pteref_t pteref, s8 level)
> {
> enum kvm_pgtable_walk_flags flags = data->walker->flags;
> kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref);
> struct kvm_pgtable_visit_ctx ctx = {
> @@ -185,35 +192,35 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
> * into a newly installed or replaced table.
> */
> if (reload) {
> ctx.old = READ_ONCE(*ptep);
> table = kvm_pte_table(ctx.old, level);
> }
>
> if (!kvm_pgtable_walk_continue(data->walker, ret))
> return ret;
>
> - if (!table) {
> + if (!table || kvm_pgtable_skip_level(level + 1, ctx.flags)) {
> data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level));
> data->addr += kvm_granule_size(level);
> goto out;
> }
>
> childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);
> ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);
> if (!kvm_pgtable_walk_continue(data->walker, ret))
> return ret;
>
> - if (ctx.flags & KVM_PGTABLE_WALK_TABLE_POST)
> +out:
> + if (table && ctx.flags & KVM_PGTABLE_WALK_TABLE_POST)
> ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_POST);
>
> -out:
> if (kvm_pgtable_walk_continue(data->walker, ret))
> return 0;
>
> return ret;
> }
>
> static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
> struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level)
> {
> u32 idx;
> @@ -242,20 +249,23 @@ static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_da
> u32 idx;
> int ret = 0;
> u64 limit = BIT(pgt->ia_bits);
>
> if (data->addr > limit || data->end > limit)
> return -ERANGE;
>
> if (!pgt->pgd)
> return -EINVAL;
>
> + if (kvm_pgtable_skip_level(pgt->start_level, data->walker->flags))
> + return 0;
> +
> for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {
> kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];
>
> ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);
> if (ret)
> break;
> }
>
> return ret;
> }
> --
> 2.55.0
>