[PATCH 17/17] irqchip/gic-v3: Implement FEAT_GICv3_NMI support
From: Jinjie Ruan
Date: Fri Jul 03 2026 - 06:13:47 EST
From: Lorenzo Pieralisi <lpieralisi@xxxxxxxxxx>
The FEAT_GICv3_NMI GIC feature coupled with the CPU FEAT_NMI enables
handling NMI interrupts in HW on aarch64, by adding a superpriority
interrupt to the existing GIC priority scheme.
Implement GIC driver support for the FEAT_GICv3_NMI feature.
Rename gic_supports_nmi() helper function to gic_supports_pseudo_nmis()
to make the pseudo NMIs code path clearer and more explicit.
Check, through the ARM64 capabilitity infrastructure, if support
for FEAT_NMI was detected on the core and the system has not overridden
the detection and forced pseudo-NMIs enablement.
If FEAT_NMI is detected, it was not overridden (check embedded in the
system_uses_nmi() call) and the GIC supports the FEAT_GICv3_NMI feature,
install an NMI handler and initialize NMIs related HW GIC registers.
Signed-off-by: Lorenzo Pieralisi <lpieralisi@xxxxxxxxxx>
Signed-off-by: Mark Brown <broonie@xxxxxxxxxx>
Signed-off-by: Marc Zyngier <maz@xxxxxxxxxx>
Link: https://lore.kernel.org/r/20221112151708.175147-15-broonie@xxxxxxxxxx
Signed-off-by: Jinjie Ruan <ruanjinjie@xxxxxxxxxx>
---
drivers/irqchip/irq-gic-v3.c | 146 ++++++++++++++++++++++++++---
include/linux/irqchip/arm-gic-v3.h | 4 +
2 files changed, 138 insertions(+), 12 deletions(-)
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 111cd882c7a9..6e48dd999021 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -64,6 +64,7 @@ struct gic_chip_data {
u32 nr_redist_regions;
u64 flags;
bool has_rss;
+ bool has_nmi;
unsigned int ppi_nr;
struct partition_affinity *parts;
unsigned int nr_parts;
@@ -252,6 +253,20 @@ enum gic_intid_range {
__INVALID_RANGE__
};
+#ifdef CONFIG_ARM64
+#include <linux/cpufeature.h>
+
+static inline bool has_v3_3_nmi(void)
+{
+ return gic_data.has_nmi && system_uses_nmi();
+}
+#else
+static inline bool has_v3_3_nmi(void)
+{
+ return false;
+}
+#endif
+
static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
{
switch (hwirq) {
@@ -467,6 +482,42 @@ static int gic_peek_irq(struct irq_data *d, u32 offset)
return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
}
+static DEFINE_RAW_SPINLOCK(irq_controller_lock);
+
+static void gic_irq_configure_nmi(struct irq_data *d, bool enable)
+{
+ void __iomem *base, *addr;
+ u32 offset, index, mask, val;
+
+ offset = convert_offset_index(d, GICD_INMIR, &index);
+ mask = 1 << (index % 32);
+
+ if (gic_irq_in_rdist(d))
+ base = gic_data_rdist_sgi_base();
+ else
+ base = gic_data.dist_base;
+
+ addr = base + offset + (index / 32) * 4;
+
+ raw_spin_lock(&irq_controller_lock);
+
+ val = readl_relaxed(addr);
+ val = enable ? (val | mask) : (val & ~mask);
+ writel_relaxed(val, addr);
+
+ raw_spin_unlock(&irq_controller_lock);
+}
+
+static void gic_irq_enable_nmi(struct irq_data *d)
+{
+ gic_irq_configure_nmi(d, true);
+}
+
+static void gic_irq_disable_nmi(struct irq_data *d)
+{
+ gic_irq_configure_nmi(d, false);
+}
+
static void gic_poke_irq(struct irq_data *d, u32 offset)
{
void __iomem *base;
@@ -512,7 +563,7 @@ static void gic_unmask_irq(struct irq_data *d)
gic_poke_irq(d, GICD_ISENABLER);
}
-static inline bool gic_supports_nmi(void)
+static inline bool gic_supports_pseudo_nmis(void)
{
return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
static_branch_likely(&supports_pseudo_nmis);
@@ -598,7 +649,7 @@ static int gic_irq_nmi_setup(struct irq_data *d)
{
struct irq_desc *desc = irq_to_desc(d->irq);
- if (!gic_supports_nmi())
+ if (!gic_supports_pseudo_nmis() && !has_v3_3_nmi())
return -EINVAL;
if (gic_peek_irq(d, GICD_ISENABLER)) {
@@ -617,7 +668,10 @@ static int gic_irq_nmi_setup(struct irq_data *d)
if (!gic_irq_in_rdist(d))
desc->handle_irq = handle_fasteoi_nmi;
- gic_irq_set_prio(d, dist_prio_nmi);
+ if (has_v3_3_nmi())
+ gic_irq_enable_nmi(d);
+ else
+ gic_irq_set_prio(d, dist_prio_nmi);
return 0;
}
@@ -626,7 +680,7 @@ static void gic_irq_nmi_teardown(struct irq_data *d)
{
struct irq_desc *desc = irq_to_desc(d->irq);
- if (WARN_ON(!gic_supports_nmi()))
+ if (WARN_ON(!gic_supports_pseudo_nmis() && !has_v3_3_nmi()))
return;
if (gic_peek_irq(d, GICD_ISENABLER)) {
@@ -645,7 +699,10 @@ static void gic_irq_nmi_teardown(struct irq_data *d)
if (!gic_irq_in_rdist(d))
desc->handle_irq = handle_fasteoi_irq;
- gic_irq_set_prio(d, dist_prio_irq);
+ if (has_v3_3_nmi())
+ gic_irq_disable_nmi(d);
+ else
+ gic_irq_set_prio(d, dist_prio_irq);
}
static bool gic_arm64_erratum_2941627_needed(struct irq_data *d)
@@ -804,7 +861,7 @@ static inline void gic_complete_ack(u32 irqnr)
static bool gic_rpr_is_nmi_prio(void)
{
- if (!gic_supports_nmi())
+ if (!gic_supports_pseudo_nmis())
return false;
return unlikely(gic_read_rpr() == GICV3_PRIO_NMI);
@@ -836,7 +893,8 @@ static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
gic_complete_ack(irqnr);
if (generic_handle_domain_nmi(gic_data.domain, irqnr)) {
- WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr);
+ WARN_ONCE(true, "Unexpected %sNMI (irqnr %u)\n",
+ gic_supports_pseudo_nmis() ? "pseudo-" : "", irqnr);
gic_deactivate_unhandled(irqnr);
}
}
@@ -857,6 +915,28 @@ static void __gic_handle_irq_from_irqson(struct pt_regs *regs)
bool is_nmi;
u32 irqnr;
+ /*
+ * We should enter here with interrupts disabled, otherwise we may met
+ * a race here with FEAT_NMI/FEAT_GICv3_NMI:
+ *
+ * [interrupt disabled]
+ * <- normal interrupt pending, for example timer interrupt
+ * <- NMI occurs, ISR_EL1.nmi = 1
+ * do_el1_interrupt()
+ * <- NMI withdraw, ISR_EL1.nmi = 0
+ * ISR_EL1.nmi = 0, not an NMI interrupt
+ * gic_handle_irq()
+ * __gic_handle_irq_from_irqson()
+ * irqnr = gic_read_iar() <- Oops, ack and handle an normal interrupt
+ * in interrupt disabled context!
+ *
+ * So if we met this case here, just return from the interrupt context.
+ * Since the interrupt is still pending, we can handle it once the
+ * interrupt re-enabled and it'll not be missing.
+ */
+ if (!interrupts_enabled(regs))
+ return;
+
irqnr = gic_read_iar();
is_nmi = gic_rpr_is_nmi_prio();
@@ -912,9 +992,37 @@ static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs)
__gic_handle_nmi(irqnr, regs);
}
+#ifdef CONFIG_ARM64
+static inline u64 gic_read_nmiar(void)
+{
+ u64 irqstat;
+
+ irqstat = read_sysreg_s(SYS_ICC_NMIAR1_EL1);
+
+ dsb(sy);
+
+ return irqstat;
+}
+
+static asmlinkage void __exception_irq_entry gic_handle_nmi_irq(struct pt_regs *regs)
+{
+ u32 irqnr = gic_read_nmiar();
+
+ __gic_handle_nmi(irqnr, regs);
+}
+
+static inline void gic_setup_nmi_handler(void)
+{
+ if (has_v3_3_nmi())
+ set_handle_nmi_irq(gic_handle_nmi_irq);
+}
+#else
+static inline void gic_setup_nmi_handler(void) { }
+#endif
+
static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
{
- if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs)))
+ if (unlikely(gic_supports_pseudo_nmis() && !interrupts_enabled(regs)))
__gic_handle_irq_from_irqsoff(regs);
else
__gic_handle_irq_from_irqson(regs);
@@ -1165,7 +1273,7 @@ static void gic_cpu_sys_reg_init(void)
/* Set priority mask register */
if (!gic_prio_masking_enabled()) {
write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
- } else if (gic_supports_nmi()) {
+ } else if (gic_supports_pseudo_nmis()) {
/*
* Check that all CPUs use the same priority space.
*
@@ -1958,15 +2066,27 @@ static const struct gic_quirk gic_quirks[] = {
}
};
-static void gic_enable_nmi_support(void)
+static void gic_enable_pseudo_nmis(void)
{
- if (!gic_prio_masking_enabled() || nmi_support_forbidden)
- return;
pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
gic_has_relaxed_pmr_sync() ? "relaxed" : "forced");
static_branch_enable(&supports_pseudo_nmis);
+}
+
+static void gic_enable_nmi_support(void)
+{
+ if ((!gic_prio_masking_enabled() || nmi_support_forbidden) &&
+ !has_v3_3_nmi())
+ return;
+
+ /*
+ * Initialize pseudo-NMIs only if GIC driver cannot take advantage
+ * of core (FEAT_NMI) and GIC (FEAT_GICv3_NMI) in HW
+ */
+ if (!has_v3_3_nmi())
+ gic_enable_pseudo_nmis();
if (static_branch_likely(&supports_deactivate_key))
gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
@@ -2035,6 +2155,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base,
irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
+ gic_data.has_nmi = !!(typer & GICD_TYPER_NMI);
if (typer & GICD_TYPER_MBIS) {
err = mbi_init(handle, gic_data.domain);
@@ -2043,6 +2164,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base,
}
set_handle_irq(gic_handle_irq);
+ gic_setup_nmi_handler();
gic_update_rdist_properties();
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index ea5fd2374ebe..ebccd76cadda 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -30,6 +30,7 @@
#define GICD_ICFGR 0x0C00
#define GICD_IGRPMODR 0x0D00
#define GICD_NSACR 0x0E00
+#define GICD_INMIR 0x0F80
#define GICD_IGROUPRnE 0x1000
#define GICD_ISENABLERnE 0x1200
#define GICD_ICENABLERnE 0x1400
@@ -39,6 +40,7 @@
#define GICD_ICACTIVERnE 0x1C00
#define GICD_IPRIORITYRnE 0x2000
#define GICD_ICFGRnE 0x3000
+#define GICD_INMIRnE 0x3B00
#define GICD_IROUTER 0x6000
#define GICD_IROUTERnE 0x8000
#define GICD_IDREGS 0xFFD0
@@ -83,6 +85,7 @@
#define GICD_TYPER_LPIS (1U << 17)
#define GICD_TYPER_MBIS (1U << 16)
#define GICD_TYPER_ESPI (1U << 8)
+#define GICD_TYPER_NMI (1U << 9)
#define GICD_TYPER_ID_BITS(typer) ((((typer) >> 19) & 0x1f) + 1)
#define GICD_TYPER_NUM_LPIS(typer) ((((typer) >> 11) & 0x1f) + 1)
@@ -238,6 +241,7 @@
#define GICR_ICFGR0 GICD_ICFGR
#define GICR_IGRPMODR0 GICD_IGRPMODR
#define GICR_NSACR GICD_NSACR
+#define GICR_INMIR0 GICD_INMIR
#define GICR_TYPER_PLPIS (1U << 0)
#define GICR_TYPER_VLPIS (1U << 1)
--
2.34.1