[RFC][PATCHv2 3/7] printk: introduce per-cpu alt_print seq buffer

From: Sergey Senozhatsky
Date: Fri Sep 30 2016 - 11:20:11 EST


This patch extends the idea of NMI per-cpu buffers to regions
that may cause recursive printk() calls and possible deadlocks.
Namely, printk() can't handle printk calls from schedule code
or printk() calls from lock debugging code (spin_dump() for instance);
because those may be called with `sem->lock' already taken or any
other `critical' locks (p->pi_lock, etc.). An example of deadlock
can be

vprintk_emit()
console_unlock()
up() << raw_spin_lock_irqsave(&sem->lock, flags);
wake_up_process()
try_to_wake_up()
ttwu_queue()
ttwu_activate()
activate_task()
enqueue_task()
enqueue_task_fair()
cfs_rq_of()
task_of()
WARN_ON_ONCE(!entity_is_task(se))
vprintk_emit()
console_trylock()
down_trylock()
raw_spin_lock_irqsave(&sem->lock, flags)
^^^^ deadlock

and some other cases.

Just like in NMI implementation, the solution uses a per-cpu
`printk_func' pointer to 'redirect' printk() calls to a 'safe'
callback, that store messages in a per-cpu buffer and flushes
them back to logbuf buffer later.

Usage example:

printk()
local_irq_save()
alt_printk_enter()
//
// any printk() call from here will endup in vprintk_alt(),
// that stores messages in a special per-CPU buffer.
//
alt_printk_exit()
local_irq_restore()

The 'redirection' mechanism, though, has been reworked, as suggested
by Petr Mladek. Instead of using a per-cpu @print_func callback we now
keep a per-cpu printk-context variable and call either default or nmi
vprintk function depending on its value. printk_nmi_entrer/exit and
alt_printk_enter/exit, thus, just set/celar corresponding bits in
printk-context functions.

The patch only adds alt_printk support, we don't use it yet.

Signed-off-by: Sergey Senozhatsky <sergey.senozhatsky@xxxxxxxxx>
---
kernel/printk/alt_printk.c | 117 +++++++++++++++++++++++++++++++++++----------
kernel/printk/internal.h | 43 ++++++++---------
kernel/printk/printk.c | 3 --
3 files changed, 111 insertions(+), 52 deletions(-)

diff --git a/kernel/printk/alt_printk.c b/kernel/printk/alt_printk.c
index 7178661..4bc1e7d 100644
--- a/kernel/printk/alt_printk.c
+++ b/kernel/printk/alt_printk.c
@@ -1,5 +1,5 @@
/*
- * alt_printk.c - Safe printk in NMI context
+ * alt_printk.c - Safe printk for printk-deadlock-prone contexts
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@@ -32,13 +32,13 @@
* is later flushed into the main ring buffer via IRQ work.
*
* The alternative implementation is chosen transparently
- * via @printk_func per-CPU variable.
+ * by examinig current printk() context mask stored in @alt_printk_ctx
+ * per-CPU variable.
*
* The implementation allows to flush the strings also from another CPU.
* There are situations when we want to make sure that all buffers
* were handled or when IRQs are blocked.
*/
-DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default;
static int alt_printk_irq_ready;
atomic_t nmi_message_lost;

@@ -52,25 +52,21 @@ struct alt_printk_seq_buf {
};
static DEFINE_PER_CPU(struct alt_printk_seq_buf, nmi_print_seq);

-/*
- * Safe printk() for NMI context. It uses a per-CPU buffer to
- * store the message. NMIs are not nested, so there is always only
- * one writer running. But the buffer might get flushed from another
- * CPU, so we need to be careful.
- */
-static int vprintk_nmi(const char *fmt, va_list args)
+static DEFINE_PER_CPU(struct alt_printk_seq_buf, alt_print_seq);
+static DEFINE_PER_CPU(int, alt_printk_ctx);
+static DEFINE_PER_CPU(unsigned long, alt_printk_irq_flags);
+
+static int alt_printk_log_store(struct alt_printk_seq_buf *s,
+ const char *fmt, va_list args)
{
- struct alt_printk_seq_buf *s = this_cpu_ptr(&nmi_print_seq);
- int add = 0;
+ int add;
size_t len;

again:
len = atomic_read(&s->len);

- if (len >= sizeof(s->buffer)) {
- atomic_inc(&nmi_message_lost);
+ if (len >= sizeof(s->buffer))
return 0;
- }

/*
* Make sure that all old data have been read before the buffer was
@@ -110,7 +106,6 @@ static void alt_printk_flush_line(const char *text, int len)
printk_deferred("%.*s", len, text);
else
printk("%.*s", len, text);
-
}

/*
@@ -240,6 +235,83 @@ void alt_printk_flush_on_panic(void)
alt_printk_flush();
}

+/*
+ * Safe printk() for NMI context. It uses a per-CPU buffer to
+ * store the message. NMIs are not nested, so there is always only
+ * one writer running. But the buffer might get flushed from another
+ * CPU, so we need to be careful.
+ */
+static int vprintk_nmi(const char *fmt, va_list args)
+{
+ struct alt_printk_seq_buf *s = this_cpu_ptr(&nmi_print_seq);
+ int add;
+
+ add = alt_printk_log_store(s, fmt, args);
+ if (!add)
+ atomic_inc(&nmi_message_lost);
+
+ return add;
+}
+
+void printk_nmi_enter(void)
+{
+ this_cpu_or(alt_printk_ctx, ALT_PRINTK_NMI_CONTEXT_MASK);
+}
+
+void printk_nmi_exit(void)
+{
+ this_cpu_and(alt_printk_ctx, ~ALT_PRINTK_NMI_CONTEXT_MASK);
+}
+
+/*
+ * Lockless printk(), to avoid deadlocks should the printk() recurse
+ * into itself. It uses a per-CPU buffer to store the message, just like
+ * NMI.
+ */
+static int vprintk_alt(const char *fmt, va_list args)
+{
+ struct alt_printk_seq_buf *s = this_cpu_ptr(&alt_print_seq);
+
+ return alt_printk_log_store(s, fmt, args);
+}
+
+/*
+ * Returns with local IRQs disabled.
+ * Can be preempted by NMI.
+ */
+void alt_printk_enter(void)
+{
+ unsigned long flags;
+ int entry_count;
+
+ local_irq_save(flags);
+ if (!(this_cpu_read(alt_printk_ctx) & ALT_PRINTK_CONTEXT_MASK))
+ this_cpu_write(alt_printk_irq_flags, flags);
+ this_cpu_inc(alt_printk_ctx);
+}
+
+/*
+ * Restores local IRQs state saved in alt_printk_enter().
+ * Can be preempted by NMI.
+ */
+void alt_printk_exit(void)
+{
+ this_cpu_dec(alt_printk_ctx);
+ if (!(this_cpu_read(alt_printk_ctx) & ALT_PRINTK_CONTEXT_MASK))
+ local_irq_restore(this_cpu_read(alt_printk_irq_flags));
+}
+
+__printf(1, 0) int vprintk_func(const char *fmt, va_list args)
+{
+ if (this_cpu_read(alt_printk_ctx) & ALT_PRINTK_NMI_CONTEXT_MASK)
+ return vprintk_nmi(fmt, args);
+
+ if (this_cpu_read(alt_printk_ctx) & ALT_PRINTK_CONTEXT_MASK)
+ return vprintk_alt(fmt, args);
+
+ return vprintk_default(fmt, args);
+}
+
void __init alt_printk_init(void)
{
int cpu;
@@ -248,6 +320,9 @@ void __init alt_printk_init(void)
struct alt_printk_seq_buf *s = &per_cpu(nmi_print_seq, cpu);

init_irq_work(&s->work, __alt_printk_flush);
+
+ s = &per_cpu(alt_print_seq, cpu);
+ init_irq_work(&s->work, __alt_printk_flush);
}

/* Make sure that IRQ works are initialized before enabling. */
@@ -257,13 +332,3 @@ void __init alt_printk_init(void)
/* Flush pending messages that did not have scheduled IRQ works. */
alt_printk_flush();
}
-
-void printk_nmi_enter(void)
-{
- this_cpu_write(printk_func, vprintk_nmi);
-}
-
-void printk_nmi_exit(void)
-{
- this_cpu_write(printk_func, vprintk_default);
-}
diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h
index 7fd2838..6338f5b 100644
--- a/kernel/printk/internal.h
+++ b/kernel/printk/internal.h
@@ -16,26 +16,8 @@
*/
#include <linux/percpu.h>

-typedef __printf(1, 0) int (*printk_func_t)(const char *fmt, va_list args);
-
-int __printf(1, 0) vprintk_default(const char *fmt, va_list args);
-
#ifdef CONFIG_PRINTK_NMI

-extern raw_spinlock_t logbuf_lock;
-
-/*
- * printk() could not take logbuf_lock in NMI context. Instead,
- * it temporary stores the strings into a per-CPU buffer.
- * The alternative implementation is chosen transparently
- * via per-CPU variable.
- */
-DECLARE_PER_CPU(printk_func_t, printk_func);
-static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
-{
- return this_cpu_read(printk_func)(fmt, args);
-}
-
extern atomic_t nmi_message_lost;
static inline int get_nmi_message_lost(void)
{
@@ -44,14 +26,29 @@ static inline int get_nmi_message_lost(void)

#else /* CONFIG_PRINTK_NMI */

-static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
-{
- return vprintk_default(fmt, args);
-}
-
static inline int get_nmi_message_lost(void)
{
return 0;
}

#endif /* CONFIG_PRINTK_NMI */
+
+#ifdef CONFIG_PRINTK
+
+#define ALT_PRINTK_CONTEXT_MASK 0x07ffffff
+#define ALT_PRINTK_NMI_CONTEXT_MASK 0x08000000
+
+extern raw_spinlock_t logbuf_lock;
+
+__printf(1, 0) int vprintk_default(const char *fmt, va_list args);
+__printf(1, 0) int vprintk_func(const char *fmt, va_list args);
+void alt_printk_enter(void);
+void alt_printk_exit(void);
+
+#else
+
+__printf(1, 0) int vprintk_func(const char *fmt, va_list args) { return 0; }
+void alt_printk_enter(void) { }
+void alt_printk_exit(void) { }
+
+#endif /* CONFIG_PRINTK */
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index c7ffbef..ac3d10e 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -2029,9 +2029,6 @@ static size_t msg_print_text(const struct printk_log *msg, enum log_flags prev,
static size_t cont_print_text(char *text, size_t size) { return 0; }
static bool suppress_message_printing(int level) { return false; }

-/* Still needs to be defined for users */
-DEFINE_PER_CPU(printk_func_t, printk_func);
-
#endif /* CONFIG_PRINTK */

#ifdef CONFIG_EARLY_PRINTK
--
2.10.0.372.g6fe1b14