[PATCH] console: consume APC, DM, DCS

[nick black]

From: nick black <nickblack@xxxxxxxxx>

console: consume APC, DM, DCS

The Linux console's VT102 implementation already
consumes OSC ("Operating System Command") sequences,
probably because that's how palette changes are
transmitted. In addition to OSC, there are three other
major clases of ANSI control strings: APC ("Application
Program Command"), PM ("Privacy Message", and DCS
("Device Control String"). They are handled similarly
to OSC in terms of termination.

Source: vt100.net

Add three new enumerated states, one for each of these
types. All three are handled the same way right now--they
simply consume input until terminated. I hope to expand
upon this firmament in the future. Add new predicate
ansi_control_string(), returning true for any of these
states. Replace explicit checks against ESosc with calls
to this function. Transition to these states appropriately
from the escape initiation (ESesc) state.

This was motivated by the following Notcurses bugs:

 https://github.com/dankamongmen/notcurses/issues/2050
 https://github.com/dankamongmen/notcurses/issues/1828
 https://github.com/dankamongmen/notcurses/issues/2069

where standard VT sequences are not consumed by the
Linux console. It's not necessary that the Linux console
*support* these sequences, but it ought *consume* these
well-specified classes of sequences.

Tested by sending a variety of escape sequences to the
console, and verifying that they still worked, or were
now properly consumed. Verified that the escapes were
properly terminated at a generic level. Verified that
the Notcurses tools continued to show expected output on
the Linux console, except now without escape bleedthrough.

Signed-off-by: Nick Black <nickblack@xxxxxxxxx>
---
 drivers/tty/vt/vt.c | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git drivers/tty/vt/vt.c drivers/tty/vt/vt.c
index ef981d3b7bb4..62b77ee3f8d1 100644
--- drivers/tty/vt/vt.c
+++ drivers/tty/vt/vt.c
@@ -2059,7 +2059,7 @@ static void restore_cur(struct vc_data *vc)
 
 enum { ESnormal, ESesc, ESsquare, ESgetpars, ESfunckey,
 	EShash, ESsetG0, ESsetG1, ESpercent, EScsiignore, ESnonstd,
-	ESpalette, ESosc };
+	ESpalette, ESosc, ESapc, ESpm, ESdcs };
 
 /* console_lock is held (except via vc_init()) */
 static void reset_terminal(struct vc_data *vc, int do_clear)
@@ -2133,20 +2133,28 @@ static void vc_setGx(struct vc_data *vc, unsigned int which, int c)
 		vc->vc_translate = set_translate(*charset, vc);
 }
 
+/* is this state an ANSI control string? */
+static bool ansi_control_string(unsigned state)
+{
+	if (state == ESosc || state == ESapc || state == ESpm || state == ESdcs)
+		return true;
+	return false;
+}
+
 /* console_lock is held */
 static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 {
 	/*
 	 *  Control characters can be used in the _middle_
-	 *  of an escape sequence.
+	 *  of an escape sequence, aside from ANSI control strings.
 	 */
-	if (vc->vc_state == ESosc && c>=8 && c<=13) /* ... except for OSC */
+	if (ansi_control_string(vc->vc_state) && c>=8 && c<=13)
 		return;
 	switch (c) {
 	case 0:
 		return;
 	case 7:
-		if (vc->vc_state == ESosc)
+		if (ansi_control_string(vc->vc_state))
 			vc->vc_state = ESnormal;
 		else if (vc->vc_bell_duration)
 			kd_mksound(vc->vc_bell_pitch, vc->vc_bell_duration);
@@ -2207,6 +2215,12 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 		case ']':
 			vc->vc_state = ESnonstd;
 			return;
+		case '_':
+			vc->vc_state = ESapc;
+			return;
+		case '^':
+			vc->vc_state = ESpm;
+			return;
 		case '%':
 			vc->vc_state = ESpercent;
 			return;
@@ -2224,6 +2238,9 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 			if (vc->state.x < VC_TABSTOPS_COUNT)
 				set_bit(vc->state.x, vc->vc_tab_stop);
 			return;
+		case 'P':
+			vc->vc_state = ESdcs;
+			return;
 		case 'Z':
 			respond_ID(tty);
 			return;
@@ -2520,8 +2537,14 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 		vc_setGx(vc, 1, c);
 		vc->vc_state = ESnormal;
 		return;
+	case ESapc:
+		return;
 	case ESosc:
 		return;
+	case ESpm:
+		return;
+	case ESdcs:
+		return;
 	default:
 		vc->vc_state = ESnormal;
 	}
-- 
2.33.0