scan_key: support more terminals.

Although we can get away with ignoring termcap/terminfo on the output
side by restricting ourselves to generally-supported escape sequences,
the input side is trickier because we need to support the sequences sent
by common terminals. Luckily, this isn't is as bad as it sounds because
only Home/End commonly differ. But it does mean we need a slightly
different implementation to deal with the many-to-one mapping.

Since we can't use TAGGED_ARRAY for this (without inflicting pain on all
the callers) I've also switched to OR-ing in the modifier keys, so we
have (say) KEY_UP|KEY_SHIFT rather than a separate KEY_SUP. This also
generalizes better should we ever need to support multiple modifiers at
once.

To reduce the number of #defines, I've also switched from KEY_F1,
KEY_F2, and so on to KEY_FN+1, KEY_FN+2, and so on. This isn't obviously
necessary, and easily undone if we'd rather have move #defines in return
for slightly more natural naming.

To enable all this, I've inverted scan_key and scan_key_getsize so that
scan_key_getsize is now the underlying function, and we don't waste all
the top bits encoding width and height between scan_key and
scan_key_getsize.

Tested by pressing Home and End in hexedit in all of the terminals
available to me.

2 files changed, 68 insertions, 49 deletions

diff --git a/lib/lib.h b/lib/lib.h
index e220962c..0f15484e 100644
--- a/lib/lib.h
+++ b/lib/lib.h
@@ -295,14 +295,28 @@ int draw_trim_esc(char *str, int padto, int width, char *escmore,
   int (*escout)(FILE *out, int cols,int wc));
 int draw_trim(char *str, int padto, int width);
 
-// interestingtimes.c
+// tty.c
 int tty_fd(void);
 int terminal_size(unsigned *xx, unsigned *yy);
 int terminal_probesize(unsigned *xx, unsigned *yy);
+#define KEY_UP 0
+#define KEY_DOWN 1
+#define KEY_RIGHT 2
+#define KEY_LEFT 3
+#define KEY_PGUP 4
+#define KEY_PGDN 5
+#define KEY_HOME 6
+#define KEY_END 7
+#define KEY_INSERT 8
+#define KEY_DELETE 9
+#define KEY_FN 10 // F1 = KEY_FN+1, F2 = KEY_FN+2, ...
+#define KEY_SHIFT (1<<16)
+#define KEY_CTRL (1<<17)
+#define KEY_ALT (1<<18)
+int scan_key(char *scratch, int timeout_ms);
 int scan_key_getsize(char *scratch, int timeout_ms, unsigned *xx, unsigned *yy);
 int set_terminal(int fd, int raw, int speed, struct termios *old);
 void xset_terminal(int fd, int raw, int speed, struct termios *old);
-int scan_key(char *scratch, int timeout_ms);
 void tty_esc(char *s);
 void tty_jump(int x, int y);
 void tty_reset(void);
diff --git a/lib/tty.c b/lib/tty.c
index cb613adc..a6b4576a 100644
--- a/lib/tty.c
+++ b/lib/tty.c
@@ -61,24 +61,6 @@ int terminal_probesize(unsigned *xx, unsigned *yy)
   return 0;
 }
 
-// Wrapper that parses results from ANSI probe to update screensize.
-// Otherwise acts like scan_key()
-int scan_key_getsize(char *scratch, int timeout_ms, unsigned *xx, unsigned *yy)
-{
-  int key;
-
-  if (512&(key = scan_key(scratch, timeout_ms))) {
-    if (key>0) {
-      if (xx) *xx = (key>>10)&1023;
-      if (yy) *yy = (key>>20)&1023;
-
-      return -3;
-    }
-  }
-
-  return key;
-}
-
 // Reset terminal to known state, saving copy of old state if old != NULL.
 int set_terminal(int fd, int raw, int speed, struct termios *old)
 {
@@ -137,33 +119,47 @@ void xset_terminal(int fd, int raw, int speed, struct termios *old)
 }
 
 struct scan_key_list {
-  char *name, *seq;
-} static const scan_key_list[] = TAGGED_ARRAY(KEY,
-  // up down right left pgup pgdn home end ins
-  {"UP", "\033[A"}, {"DOWN", "\033[B"}, {"RIGHT", "\033[C"}, {"LEFT", "\033[D"},
-  {"PGUP", "\033[5~"}, {"PGDN", "\033[6~"}, {"HOME", "\033OH"},
-  {"END", "\033OF"}, {"INSERT", "\033[2~"},
-
-  {"F1", "\033OP"}, {"F2", "\033OQ"}, {"F3", "\033OR"}, {"F4", "\033OS"},
-  {"F5", "\033[15~"}, {"F6", "\033[17~"}, {"F7", "\033[18~"},
-  {"F8", "\033[19~"}, {"F9", "\033[20~"},
-
-  {"SUP", "\033[1;2A"}, {"AUP", "\033[1;3A"}, {"CUP", "\033[1;5A"},
-  {"SDOWN", "\033[1;2B"}, {"ADOWN", "\033[1;3B"}, {"CDOWN", "\033[1;5B"},
-  {"SRIGHT", "\033[1;2C"}, {"ARIGHT", "\033[1;3C"}, {"CRIGHT", "\033[1;5C"},
-  {"SLEFT", "\033[1;2D"}, {"ALEFT", "\033[1;3D"}, {"CLEFT", "\033[1;5D"},
-
-  {"SF1", "\033O1;2P"}, {"AF1", "\033O1;3P"}, {"CF1", "\033[1;5P"}
-);
-
-// Scan stdin for a keypress, parsing known escape sequences
-// Blocks for timeout_ms milliseconds, none 0, forever if -1
-// Returns: 0-255=literal, -1=EOF, -2=TIMEOUT, 256-...=index into scan_key_list
-// >512 is x<<9+y<<21
-// scratch space is necessary because last char of !seq could start new seq
-// Zero out first byte of scratch before first call to scan_key
-// block=0 allows fetching multiple characters before updating display
-int scan_key(char *scratch, int timeout_ms)
+  int key;
+  char *seq;
+} static const scan_key_list[] = {
+  {KEY_UP, "\033[A"}, {KEY_DOWN, "\033[B"},
+  {KEY_RIGHT, "\033[C"}, {KEY_LEFT, "\033[D"},
+
+  {KEY_UP|KEY_SHIFT, "\033[1;2A"}, {KEY_DOWN|KEY_SHIFT, "\033[1;2B"},
+  {KEY_RIGHT|KEY_SHIFT, "\033[1;2C"}, {KEY_LEFT|KEY_SHIFT, "\033[1;2D"},
+
+  {KEY_UP|KEY_ALT, "\033[1;3A"}, {KEY_DOWN|KEY_ALT, "\033[1;3B"},
+  {KEY_RIGHT|KEY_ALT, "\033[1;3C"}, {KEY_LEFT|KEY_ALT, "\033[1;3D"},
+
+  {KEY_UP|KEY_CTRL, "\033[1;5A"}, {KEY_DOWN|KEY_CTRL, "\033[1;5B"},
+  {KEY_RIGHT|KEY_CTRL, "\033[1;5C"}, {KEY_LEFT|KEY_CTRL, "\033[1;5D"},
+
+  // VT102/VT220 escapes.
+  {KEY_HOME, "\033[1~"},
+  {KEY_INSERT, "\033[2~"},
+  {KEY_DELETE, "\033[3~"},
+  {KEY_END, "\033[4~"},
+  {KEY_PGUP, "\033[5~"},
+  {KEY_PGDN, "\033[6~"},
+  // "Normal" "PC" escapes (xterm).
+  {KEY_HOME, "\033OH"},
+  {KEY_END, "\033OF"},
+  // "Application" "PC" escapes (gnome-terminal).
+  {KEY_HOME, "\033[H"},
+  {KEY_END, "\033[F"},
+
+  {KEY_FN+1, "\033OP"}, {KEY_FN+2, "\033OQ"}, {KEY_FN+3, "\033OR"},
+  {KEY_FN+4, "\033OS"}, {KEY_FN+5, "\033[15~"}, {KEY_FN+6, "\033[17~"},
+  {KEY_FN+7, "\033[18~"}, {KEY_FN+8, "\033[19~"}, {KEY_FN+9, "\033[20~"},
+};
+
+// Scan stdin for a keypress, parsing known escape sequences, including
+// responses to screen size queries.
+// Blocks for timeout_ms milliseconds, 0=return immediately, -1=wait forever.
+// Returns 0-255=literal, -1=EOF, -2=TIMEOUT, -3=RESIZE, 256+= a KEY_ constant.
+// Scratch space is necessary because last char of !seq could start new seq.
+// Zero out first byte of scratch before first call to scan_key.
+int scan_key_getsize(char *scratch, int timeout_ms, unsigned *xx, unsigned *yy)
 {
   struct pollfd pfd;
   int maybe, i, j;
@@ -187,7 +183,9 @@ int scan_key(char *scratch, int timeout_ms)
       if (pos[5]) {
         // Recognized X/Y position, consume and return
         *scratch = 0;
-        return 512+(x<<10)+(y<<20);
+        if (xx) *xx = x;
+        if (yy) *yy = y;
+        return -3;
       } else for (i=0; i<6; i++) if (pos[i]==*scratch) maybe = 1;
 
       // Check sequences
@@ -199,7 +197,7 @@ int scan_key(char *scratch, int timeout_ms)
           if (!test[j]) {
             // We recognized current sequence: consume and return
             *scratch = 0;
-            return 256+i;
+            return 256+scan_key_list[i].key;
           }
         }
       }
@@ -229,6 +227,13 @@ int scan_key(char *scratch, int timeout_ms)
   return i;
 }
 
+// Wrapper that ignores results from ANSI probe to update screensize.
+// Otherwise acts like scan_key_getsize().
+int scan_key(char *scratch, int timeout_ms)
+{
+  return scan_key_getsize(scratch, timeout_ms, NULL, NULL);
+}
+
 void tty_esc(char *s)
 {
   printf("\033[%s", s);