py: Implement basic unicode functions.

py/misc.h

@@ -100,7 +100,9 @@ bool unichar_isupper(unichar c);
 bool unichar_islower(unichar c);
 unichar unichar_tolower(unichar c);
 unichar unichar_toupper(unichar c);
-#define unichar_charlen(s, bytelen) (bytelen)
+uint unichar_charlen(const char *str, uint len);
+#define UTF8_IS_NONASCII(ch) ((ch) & 0x80)
+#define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80)
 
 /** variable string *********************************************/
 

py/unicode.c

@@ -65,14 +65,39 @@ STATIC const uint8_t attr[] = {
     AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0
 };
 
-unichar utf8_get_char(const byte *s) {
-    return *s;
+unichar utf8_get_char(const char *s) {
+    unichar ord = *s++;
+    if (!UTF8_IS_NONASCII(ord)) return ord;
+    ord &= 0x7F;
+    for (unichar mask = 0x40; ord & mask; mask >>= 1) {
+        ord &= ~mask;
+    }
+    while (UTF8_IS_CONT(*s)) {
+        ord = (ord << 6) | (*s++ & 0x3F);
+    }
+    return ord;
+}
+
+char *utf8_next_char(const char *s) {
+    ++s;
+    while (UTF8_IS_CONT(*s)) {
+        ++s;
+    }
+    return (char *)s;
 }
 
-const byte *utf8_next_char(const byte *s) {
-    return s + 1;
+uint unichar_charlen(const char *str, uint len)
+{
+    uint charlen = 0;
+    for (const char *top = str + len; str < top; ++str) {
+        if (!UTF8_IS_CONT(*str)) {
+            ++charlen;
+        }
+    }
+    return charlen;
 }
 
+// Be aware: These unichar_is* functions are actually ASCII-only!
 bool unichar_isspace(unichar c) {
     return c < 128 && (attr[c] & FL_SPACE) != 0;
 }