Re: [PATCH 1/3] vt: avoid a VLA in the unicode screen scroll function

[Nicolas Pitre]

On Tue, 17 Jul 2018, Nicolas Pitre wrote:

> But still, if nr > 2 that means you need a temporary storage because the 
> destination memory has to be preserved before the source memory can be 
> moved there, and that destination memory content cannot be stored in the 
> vacated source memory until the source content is moved.

OK I'm an idiot.

After looking in the literature, I found out that there is indeed a 
better way to do this. So here's an updated patch:

----- >8

Subject: [PATCH v2 1/3] vt: avoid a VLA in the unicode screen scroll function

The nr argument is typically small: most often nr == 1. However this
could be abused with a very large explicit scroll in a resized screen.
Make the code scroll lines by performing an array rotation operation to
avoid the need for a large temporary space.

Requested-by: Kees Cook <keescook@xxxxxxxxxxxx>
Suggested-by: Adam Borowski <kilobyte@xxxxxxxxxx>
Signed-off-by: Nicolas Pitre <nico@xxxxxxxxxx>

diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index 2d14bb195d..d527184579 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -104,6 +104,7 @@
 #include <linux/kdb.h>
 #include <linux/ctype.h>
 #include <linux/bsearch.h>
+#include <linux/gcd.h>
 
 #define MAX_NR_CON_DRIVER 16
 
@@ -432,20 +433,29 @@ static void vc_uniscr_scroll(struct vc_data *vc, unsigned int t, unsigned int b,
 	struct uni_screen *uniscr = get_vc_uniscr(vc);
 
 	if (uniscr) {
-		unsigned int s, d, rescue, clear;
-		char32_t *save[nr];
-
-		s = clear = t;
-		d = t + nr;
-		rescue = b - nr;
-		if (dir == SM_UP) {
-			swap(s, d);
-			swap(clear, rescue);
+		unsigned int i, j, k, sz, d, clear;
+
+		sz = b - t;
+		clear = b - nr;
+		d = nr;
+		if (dir == SM_DOWN) {
+			clear = t;
+			d = sz - nr;
+		}
+		for (i = 0; i < gcd(d, sz); i++) {
+			char32_t *tmp = uniscr->lines[t + i];
+			j = i;
+			while (1) {
+				k = j + d;
+				if (k >= sz)
+					k -= sz;
+				if (k == i)
+					break;
+				uniscr->lines[t + j] = uniscr->lines[t + k];
+				j = k;
+			}
+			uniscr->lines[t + j] = tmp;
 		}
-		memcpy(save, uniscr->lines + rescue, nr * sizeof(*save));
-		memmove(uniscr->lines + d, uniscr->lines + s,
-			(b - t - nr) * sizeof(*uniscr->lines));
-		memcpy(uniscr->lines + clear, save, nr * sizeof(*save));
 		vc_uniscr_clear_lines(vc, clear, nr);
 	}
 }