Tweak utf8towc() to return -1 earlier sometimes (instead of -2), and add test

program to compare against libc output.

2 files changed, 45 insertions, 6 deletions

diff --git a/lib/lib.c b/lib/lib.c
index c482dcab..a4b7229b 100644
--- a/lib/lib.c
+++ b/lib/lib.c
@@ -345,20 +345,17 @@ int utf8towc(wchar_t *wc, char *str, unsigned len)
   if (len && *str<128) return !!(*wc = *str);
 
   result = first = *(s = str++);
+  if (result<0xc2 || result>0xf4) return -1;
   for (mask = 6; (first&0xc0)==0xc0; mask += 5, first <<= 1) {
-    if (mask>21) return -1;
     if (!--len) return -2;
-    c = *(str++);
-    if ((c&0xc0) != 0x80) return -1;
+    if (((c = *(str++))&0xc0) != 0x80) return -1;
     result = (result<<6)|(c&0x3f);
   }
   result &= (1<<mask)-1;
   c = str-s;
-  if (mask==6) return -1;
 
   // Avoid overlong encodings
-  if (mask==6 || mask>21 || result<(unsigned []){0x80,0x800,0x10000}[c-2])
-    return -1;
+  if (result<(unsigned []){0x80,0x800,0x10000}[c-2]) return -1;
 
   // Limit unicode so it can't encode anything UTF-16 can't.
   if (result>0x10ffff || (result>=0xd800 && result<=0xdfff)) return -1;
diff --git a/toys/example/test_utf8towc.c b/toys/example/test_utf8towc.c
new file mode 100644
index 00000000..f939eaa7
--- /dev/null
+++ b/toys/example/test_utf8towc.c
@@ -0,0 +1,42 @@
+/* test_utf8towc() against libc mbrtowc()
+ *
+ * Copyright 2017 Rob Landley <rob@landley.net>
+
+USE_TEST_UTF8TOWC(NEWTOY(test_utf8towc, 0, TOYFLAG_USR|TOYFLAG_BIN))
+
+config TEST_UTF8TOWC
+  bool "test_utf8towc"
+  default n
+  help
+    usage: test_utf8towc
+
+    Print differences between toybox's utf8 conversion routines vs libc du jour.
+*/
+
+#include "toys.h"
+
+void test_utf8towc_main(void)
+{
+  mbstate_t mb;
+  int len1, len2;
+  unsigned u, h;
+  wchar_t wc1, wc2;
+
+  setlocale(LC_ALL, "en_US.UTF-8");
+
+  memset(&mb, 0, sizeof(mb));
+  for (u=1; u; u++) {
+    char *str = (void *)&h;
+
+    wc1 = wc2 = 0;
+    len2 = 4;
+    h = htonl(u);
+    while (!*str) str++, len2--;
+
+    len1 = mbrtowc(&wc1, str, len2, &mb);
+    if (len1<0) memset(&mb, 0, sizeof(mb));
+    len2 = utf8towc(&wc2, str, len2);
+    if (len1 != len2 || wc1 != wc2)
+      printf("%x %d %x %d %x\n", u, len1, wc1, len2, wc2);
+  }
+}