Revamp qstrs: they now include length and hash.

Can now have null bytes in strings.  Can define ROM qstrs per port using
qstrdefsport.h

py/objslice.c

@@ -6,6 +6,7 @@
 #include "nlr.h"
 #include "misc.h"
 #include "mpconfig.h"
+#include "qstr.h"
 #include "obj.h"
 #include "runtime0.h"
 

py/objstr.c

@@ -7,7 +7,7 @@
 #include "nlr.h"
 #include "misc.h"
 #include "mpconfig.h"
-#include "mpqstr.h"
+#include "qstr.h"
 #include "obj.h"
 #include "runtime0.h"
 #include "runtime.h"
@@ -36,9 +36,30 @@ void str_print(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj
     mp_obj_str_print_qstr(print, env, self->qstr, kind);
 }
 
+// like strstr but with specified length and allows \0 bytes
+// TODO replace with something more efficient/standard
+static const byte *find_subbytes(const byte *haystack, uint hlen, const byte *needle, uint nlen) {
+    if (hlen >= nlen) {
+        for (uint i = 0; i <= hlen - nlen; i++) {
+            bool found = true;
+            for (uint j = 0; j < nlen; j++) {
+                if (haystack[i + j] != needle[j]) {
+                    found = false;
+                    break;
+                }
+            }
+            if (found) {
+                return haystack + i;
+            }
+        }
+    }
+    return NULL;
+}
+
 mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) {
     mp_obj_str_t *lhs = lhs_in;
-    const char *lhs_str = qstr_str(lhs->qstr);
+    uint lhs_len;
+    const byte *lhs_data = qstr_data(lhs->qstr, &lhs_len);
     switch (op) {
         case RT_BINARY_OP_SUBSCR:
             // TODO: need predicate to check for int-like type (bools are such for example)
@@ -46,31 +67,30 @@ mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) {
             if (MP_OBJ_IS_SMALL_INT(rhs_in)) {
                 // TODO: This implements byte string access for single index so far
                 // TODO: Handle negative indexes.
-                return mp_obj_new_int(lhs_str[mp_obj_get_int(rhs_in)]);
+                return mp_obj_new_int(lhs_data[mp_obj_get_int(rhs_in)]);
 #if MICROPY_ENABLE_SLICE
             } else if (MP_OBJ_IS_TYPE(rhs_in, &slice_type)) {
                 machine_int_t start, stop, step;
                 mp_obj_slice_get(rhs_in, &start, &stop, &step);
                 assert(step == 1);
-                int len = strlen(lhs_str);
                 if (start < 0) {
-                    start = len + start;
+                    start = lhs_len + start;
                     if (start < 0) {
                         start = 0;
                     }
-                } else if (start > len) {
-                    start = len;
+                } else if (start > lhs_len) {
+                    start = lhs_len;
                 }
                 if (stop <= 0) {
-                    stop = len + stop;
+                    stop = lhs_len + stop;
                     // CPython returns empty string in such case
                     if (stop < 0) {
                         stop = start;
                     }
-                } else if (stop > len) {
-                    stop = len;
+                } else if (stop > lhs_len) {
+                    stop = lhs_len;
                 }
-                return mp_obj_new_str(qstr_from_strn_copy(lhs_str + start, stop - start));
+                return mp_obj_new_str(qstr_from_strn((const char*)lhs_data + start, stop - start));
 #endif
             } else {
                 // Message doesn't match CPython, but we don't have so much bytes as they
@@ -82,24 +102,24 @@ mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) {
         case RT_BINARY_OP_INPLACE_ADD:
             if (MP_OBJ_IS_TYPE(rhs_in, &str_type)) {
                 // add 2 strings
-                const char *rhs_str = qstr_str(((mp_obj_str_t*)rhs_in)->qstr);
-                size_t lhs_len = strlen(lhs_str);
-                size_t rhs_len = strlen(rhs_str);
-                int alloc_len = lhs_len + rhs_len + 1;
-                char *val = m_new(char, alloc_len);
-                memcpy(val, lhs_str, lhs_len);
-                memcpy(val + lhs_len, rhs_str, rhs_len);
-                val[lhs_len + rhs_len] = '\0';
-                return mp_obj_new_str(qstr_from_str_take(val, alloc_len));
+                uint rhs_len;
+                const byte *rhs_data = qstr_data(((mp_obj_str_t*)rhs_in)->qstr, &rhs_len);
+                int alloc_len = lhs_len + rhs_len;
+                byte *q_ptr;
+                byte *val = qstr_build_start(alloc_len, &q_ptr);
+                memcpy(val, lhs_data, lhs_len);
+                memcpy(val + lhs_len, rhs_data, rhs_len);
+                return mp_obj_new_str(qstr_build_end(q_ptr));
             }
             break;
         case RT_COMPARE_OP_IN:
         case RT_COMPARE_OP_NOT_IN:
             /* NOTE `a in b` is `b.__contains__(a)` */
             if (MP_OBJ_IS_TYPE(rhs_in, &str_type)) {
-                const char *rhs_str = qstr_str(((mp_obj_str_t*)rhs_in)->qstr);
-                /* FIXME \0 in strs */
-                return MP_BOOL((op == RT_COMPARE_OP_IN) ^ (strstr(lhs_str, rhs_str) == NULL));
+                uint rhs_len;
+                const byte *rhs_data = qstr_data(((mp_obj_str_t*)rhs_in)->qstr, &rhs_len);
+                return MP_BOOL((op == RT_COMPARE_OP_IN) ^ (find_subbytes(lhs_data, lhs_len, rhs_data, rhs_len) == NULL));
+                return mp_const_false;
             }
             break;
     }
@@ -143,22 +163,22 @@ mp_obj_t str_join(mp_obj_t self_in, mp_obj_t arg) {
     }
 
     // make joined string
-    char *joined_str = m_new(char, required_len + 1);
-    char *s_dest = joined_str;
+    byte *q_ptr;
+    byte *s_dest = qstr_build_start(required_len, &q_ptr);
     for (int i = 0; i < seq_len; i++) {
         if (i > 0) {
             memcpy(s_dest, sep_str, sep_len);
             s_dest += sep_len;
         }
-        const char *s2 = qstr_str(mp_obj_str_get(seq_items[i]));
-        size_t s2_len = strlen(s2);
+        uint s2_len;
+        const byte *s2 = qstr_data(mp_obj_str_get(seq_items[i]), &s2_len);
         memcpy(s_dest, s2, s2_len);
         s_dest += s2_len;
     }
-    *s_dest = '\0';
+    qstr q = qstr_build_end(q_ptr);
 
     // return joined string
-    return mp_obj_new_str(qstr_from_str_take(joined_str, required_len + 1));
+    return mp_obj_new_str(q);
 
 bad_arg:
     nlr_jump(mp_obj_new_exception_msg(MP_QSTR_TypeError, "?str.join expecting a list of str's"));
@@ -246,20 +266,14 @@ mp_obj_t str_strip(uint n_args, const mp_obj_t *args) {
     }
 
     if (first_good_char_pos == 0 && last_good_char_pos == 0) {
-        //string is all whitespace, return '\0'
-        char *empty = m_new(char, 1);
-        empty[0] = '\0';
-        return mp_obj_new_str(qstr_from_str_take(empty, 1));
+        //string is all whitespace, return ''
+        return mp_obj_new_str(MP_QSTR_);
     }
 
     assert(last_good_char_pos >= first_good_char_pos);
     //+1 to accomodate the last character
     size_t stripped_len = last_good_char_pos - first_good_char_pos + 1;
-    //+1 to accomodate '\0'
-    char *stripped_str = m_new(char, stripped_len + 1);
-    memcpy(stripped_str, orig_str + first_good_char_pos, stripped_len);
-    stripped_str[stripped_len] = '\0';
-    return mp_obj_new_str(qstr_from_str_take(stripped_str, stripped_len + 1));
+    return mp_obj_new_str(qstr_from_strn(orig_str + first_good_char_pos, stripped_len));
 }
 
 mp_obj_t str_format(uint n_args, const mp_obj_t *args) {
@@ -288,7 +302,7 @@ mp_obj_t str_format(uint n_args, const mp_obj_t *args) {
         }
     }
 
-    return mp_obj_new_str(qstr_from_str_take(vstr->buf, vstr->alloc));
+    return mp_obj_new_str(qstr_from_strn_take(vstr->buf, vstr->alloc, vstr->len));
 }
 
 static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_find_obj, 2, 4, str_find);
@@ -339,7 +353,7 @@ mp_obj_t str_it_iternext(mp_obj_t self_in) {
     mp_obj_str_it_t *self = self_in;
     const char *str = qstr_str(self->str->qstr);
     if (self->cur < strlen(str)) {
-        mp_obj_t o_out = mp_obj_new_str(qstr_from_strn_copy(str + self->cur, 1));
+        mp_obj_t o_out = mp_obj_new_str(qstr_from_strn(str + self->cur, 1));
         self->cur += 1;
         return o_out;
     } else {

py/objtuple.c

@@ -5,7 +5,7 @@
 #include "nlr.h"
 #include "misc.h"
 #include "mpconfig.h"
-#include "mpqstr.h"
+#include "qstr.h"
 #include "obj.h"
 #include "runtime0.h"
 #include "runtime.h"

py/objtype.c

@@ -6,7 +6,7 @@
 #include "nlr.h"
 #include "misc.h"
 #include "mpconfig.h"
-#include "mpqstr.h"
+#include "qstr.h"
 #include "obj.h"
 #include "map.h"
 #include "runtime0.h"
@@ -166,7 +166,7 @@ static mp_obj_t class_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) {
     if (op_name == NULL) {
         return MP_OBJ_NULL;
     }
-    mp_obj_t member = mp_obj_class_lookup(lhs->base.type, qstr_from_str_static(op_name));
+    mp_obj_t member = mp_obj_class_lookup(lhs->base.type, QSTR_FROM_STR_STATIC(op_name));
     if (member != MP_OBJ_NULL) {
         return rt_call_function_2(member, lhs_in, rhs_in);
     } else {
@@ -219,7 +219,7 @@ static bool class_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t value) {
 
 bool class_store_item(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
     mp_obj_class_t *self = self_in;
-    mp_obj_t member = mp_obj_class_lookup(self->base.type, qstr_from_str_static("__setitem__"));
+    mp_obj_t member = mp_obj_class_lookup(self->base.type, QSTR_FROM_STR_STATIC("__setitem__"));
     if (member != MP_OBJ_NULL) {
         mp_obj_t args[3] = {self_in, index, value};
         rt_call_function_n_kw(member, 3, 0, args);

py/objzip.c

@@ -3,6 +3,7 @@
 
 #include "misc.h"
 #include "mpconfig.h"
+#include "qstr.h"
 #include "obj.h"
 #include "runtime.h"
 

py/parse.c

@@ -8,7 +8,7 @@
 
 #include "misc.h"
 #include "mpconfig.h"
-#include "mpqstr.h"
+#include "qstr.h"
 #include "lexer.h"
 #include "parse.h"
 
@@ -205,7 +205,7 @@ static void push_result_token(parser_t *parser, const mp_lexer_t *lex) {
     const mp_token_t *tok = mp_lexer_cur(lex);
     mp_parse_node_t pn;
     if (tok->kind == MP_TOKEN_NAME) {
-        pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, qstr_from_strn_copy(tok->str, tok->len));
+        pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, qstr_from_strn(tok->str, tok->len));
     } else if (tok->kind == MP_TOKEN_NUMBER) {
         bool dec = false;
         bool small_int = true;
@@ -254,16 +254,16 @@ static void push_result_token(parser_t *parser, const mp_lexer_t *lex) {
             }
         }
         if (dec) {
-            pn = mp_parse_node_new_leaf(MP_PARSE_NODE_DECIMAL, qstr_from_strn_copy(str, len));
+            pn = mp_parse_node_new_leaf(MP_PARSE_NODE_DECIMAL, qstr_from_strn(str, len));
         } else if (small_int && !overflow && MP_FIT_SMALL_INT(int_val)) {
             pn = mp_parse_node_new_leaf(MP_PARSE_NODE_SMALL_INT, int_val);
         } else {
-            pn = mp_parse_node_new_leaf(MP_PARSE_NODE_INTEGER, qstr_from_strn_copy(str, len));
+            pn = mp_parse_node_new_leaf(MP_PARSE_NODE_INTEGER, qstr_from_strn(str, len));
         }
     } else if (tok->kind == MP_TOKEN_STRING) {
-        pn = mp_parse_node_new_leaf(MP_PARSE_NODE_STRING, qstr_from_strn_copy(tok->str, tok->len));
+        pn = mp_parse_node_new_leaf(MP_PARSE_NODE_STRING, qstr_from_strn(tok->str, tok->len));
     } else if (tok->kind == MP_TOKEN_BYTES) {
-        pn = mp_parse_node_new_leaf(MP_PARSE_NODE_BYTES, qstr_from_strn_copy(tok->str, tok->len));
+        pn = mp_parse_node_new_leaf(MP_PARSE_NODE_BYTES, qstr_from_strn(tok->str, tok->len));
     } else {
         pn = mp_parse_node_new_leaf(MP_PARSE_NODE_TOKEN, tok->kind);
     }

py/py.mk

@@ -25,13 +25,8 @@ endif
 
 # default settings; can be overriden in main Makefile
 
-ifndef PY_SRC
-PY_SRC = ../py
-endif
-
-ifndef BUILD
-BUILD = build
-endif
+PY_SRC ?= ../py
+BUILD ?= build
 
 # to create the build directory
 
@@ -42,6 +37,10 @@ $(BUILD):
 
 PY_BUILD = $(BUILD)/py.
 
+# file containing qstr defs for the core Python bit
+
+PY_QSTR_DEFS = $(PY_SRC)/qstrdefs.h
+
 # py object files
 
 PY_O_BASENAME = \
@@ -97,6 +96,7 @@ PY_O_BASENAME = \
 	objstr.o \
 	objtuple.o \
 	objtype.o \
+	objzip.o \
 	stream.o \
 	builtin.o \
 	builtinimport.o \
@@ -105,12 +105,21 @@ PY_O_BASENAME = \
 	vm.o \
 	showbc.o \
 	repl.o \
-	objzip.o \
 
 # prepend the build destination prefix to the py object files
 
 PY_O = $(addprefix $(PY_BUILD), $(PY_O_BASENAME))
 
+# qstr data
+
+$(PY_BUILD)qstr.o: $(PY_BUILD)qstrdefs.generated.h
+
+$(PY_BUILD)qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(PY_SRC)/makeqstrdata.py
+	$(ECHO) "makeqstrdata $(PY_QSTR_DEFS) $(QSTR_DEFS)"
+	$(Q)python $(PY_SRC)/makeqstrdata.py $(PY_QSTR_DEFS) $(QSTR_DEFS) > $@
+
+# emitters
+
 $(PY_BUILD)emitnx64.o: $(PY_SRC)/emitnative.c $(PY_SRC)/emit.h mpconfigport.h
 	$(ECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -DN_X64 -c -o $@ $<
@@ -119,11 +128,13 @@ $(PY_BUILD)emitnthumb.o: $(PY_SRC)/emitnative.c $(PY_SRC)/emit.h mpconfigport.h
 	$(ECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -DN_THUMB -c -o $@ $<
 
+# general source files
+
 $(PY_BUILD)%.o: $(PY_SRC)/%.S
 	$(ECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -c -o $@ $<
 
-$(PY_BUILD)%.o: $(PY_SRC)/%.c mpconfigport.h
+$(PY_BUILD)%.o: $(PY_SRC)/%.c mpconfigport.h $(PY_SRC)/qstr.h $(PY_QSTR_DEFS) $(QSTR_DEFS)
 	$(ECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -c -o $@ $<
 
@@ -141,5 +152,5 @@ $(PY_BUILD)vm.o: $(PY_SRC)/vm.c
 
 $(PY_BUILD)parse.o: $(PY_SRC)/grammar.h
 $(PY_BUILD)compile.o: $(PY_SRC)/grammar.h
-$(PY_BUILD)/emitcpy.o: $(PY_SRC)/emit.h
+$(PY_BUILD)emitcpy.o: $(PY_SRC)/emit.h
 $(PY_BUILD)emitbc.o: $(PY_SRC)/emit.h

py/qstr.c

@@ -2,7 +2,8 @@
 #include <string.h>
 
 #include "misc.h"
-#include "mpqstr.h"
+#include "mpconfig.h"
+#include "qstr.h"
 
 // NOTE: we are using linear arrays to store and search for qstr's (unique strings, interned strings)
 // ultimately we will replace this with a static hash table of some kind
@@ -15,12 +16,33 @@
 #define DEBUG_printf(args...) (void)0
 #endif
 
+// A qstr is an index into the qstr pool.
+// The data for a qstr contains (hash, length, data).
+// For now we use very simple encoding, just to get the framework correct:
+//  - hash is 2 bytes (simply the sum of data bytes)
+//  - length is 2 bytes
+//  - data follows
+//  - \0 terminated (for now, so they can be printed using printf)
+
+#define Q_GET_HASH(q)   ((q)[0] | ((q)[1] << 8))
+#define Q_GET_ALLOC(q)  (4 + Q_GET_LENGTH(q) + 1)
+#define Q_GET_LENGTH(q) ((q)[2] | ((q)[3] << 8))
+#define Q_GET_DATA(q)   ((q) + 4)
+
+static machine_uint_t compute_hash(const byte *data, uint len) {
+    machine_uint_t hash = 0;
+    for (const byte *top = data + len; data < top; data++) {
+        hash += *data;
+    }
+    return hash & 0xffff;
+}
+
 typedef struct _qstr_pool_t {
     struct _qstr_pool_t *prev;
     uint total_prev_len;
     uint alloc;
     uint len;
-    const char *qstrs[];
+    const byte *qstrs[];
 } qstr_pool_t;
 
 const static qstr_pool_t const_pool = {
@@ -29,9 +51,11 @@ const static qstr_pool_t const_pool = {
     10,                 // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below)
     MP_QSTR_number_of,  // corresponds to number of strings in array just below
     {
-        "nil", // must be first, since 0 qstr is nil
-#define Q(id) #id,
-#include "mpqstrraw.h"
+        (const byte*) "\0\0\0\0", // invalid/no qstr has empty data
+        (const byte*) "\0\0\0\0", // empty qstr
+#define Q(id, str) str,
+// TODO having 'build/' here is a bit of a hack, should take config variable from Makefile
+#include "build/py.qstrdefs.generated.h"
 #undef Q
     },
 };
@@ -42,8 +66,20 @@ void qstr_init(void) {
     last_pool = (qstr_pool_t*)&const_pool; // we won't modify the const_pool since it has no allocated room left
 }
 
-static qstr qstr_add(const char *str) {
-    DEBUG_printf("QSTR: add %s\n", str);
+static const byte *find_qstr(qstr q) {
+    // search pool for this qstr
+    for (qstr_pool_t *pool = last_pool; pool != NULL; pool = pool->prev) {
+        if (q >= pool->total_prev_len) {
+            return pool->qstrs[q - pool->total_prev_len];
+        }
+    }
+
+    // not found
+    return 0;
+}
+
+static qstr qstr_add(const byte *q_ptr) {
+    DEBUG_printf("QSTR: add hash=%d len=%d data=%.*s\n", Q_GET_HASH(q_ptr), Q_GET_LENGTH(q_ptr), Q_GET_LENGTH(q_ptr), Q_GET_DATA(q_ptr));
 
     // make sure we have room in the pool for a new qstr
     if (last_pool->len >= last_pool->alloc) {
@@ -57,55 +93,95 @@ static qstr qstr_add(const char *str) {
     }
 
     // add the new qstr
-    last_pool->qstrs[last_pool->len++] = str;
+    last_pool->qstrs[last_pool->len++] = q_ptr;
 
     // return id for the newly-added qstr
     return last_pool->total_prev_len + last_pool->len - 1;
 }
 
-qstr qstr_from_str_static(const char *str) {
+static qstr qstr_find_strn(const byte *str, uint str_len) {
+    // work out hash of str
+    machine_uint_t str_hash = compute_hash((const byte*)str, str_len);
+
+    // search pools for the data
     for (qstr_pool_t *pool = last_pool; pool != NULL; pool = pool->prev) {
-        for (const char **qstr = pool->qstrs, **qstr_top = pool->qstrs + pool->len; qstr < qstr_top; qstr++) {
-            if (strcmp(*qstr, str) == 0) {
-                return pool->total_prev_len + (qstr - pool->qstrs);
+        for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
+            if (Q_GET_HASH(*q) == str_hash && Q_GET_LENGTH(*q) == str_len && strncmp((const char*)Q_GET_DATA(*q), (const char*)str, str_len) == 0) {
+                return pool->total_prev_len + (q - pool->qstrs);
             }
         }
     }
-    return qstr_add(str);
+
+    // not found; return null qstr
+    return 0;
 }
 
-qstr qstr_from_str_take(char *str, int alloc_len) {
-    for (qstr_pool_t *pool = last_pool; pool != NULL; pool = pool->prev) {
-        for (const char **qstr = pool->qstrs, **qstr_top = pool->qstrs + pool->len; qstr < qstr_top; qstr++) {
-            if (strcmp(*qstr, str) == 0) {
-                m_del(char, str, alloc_len);
-                return pool->total_prev_len + (qstr - pool->qstrs);
-            }
-        }
-    }
-    return qstr_add(str);
+qstr qstr_from_str(const char *str) {
+    return qstr_from_strn(str, strlen(str));
 }
 
-qstr qstr_from_strn_copy(const char *str, int len) {
-    for (qstr_pool_t *pool = last_pool; pool != NULL; pool = pool->prev) {
-        for (const char **qstr = pool->qstrs, **qstr_top = pool->qstrs + pool->len; qstr < qstr_top; qstr++) {
-            if (strncmp(*qstr, str, len) == 0 && (*qstr)[len] == '\0') {
-                return pool->total_prev_len + (qstr - pool->qstrs);
+qstr qstr_from_strn(const char *str, uint len) {
+    qstr q = qstr_find_strn((const byte*)str, len);
+    if (q == 0) {
+        machine_uint_t hash = compute_hash((const byte*)str, len);
+        byte *q_ptr = m_new(byte, 4 + len + 1);
+        q_ptr[0] = hash;
+        q_ptr[1] = hash >> 8;
+        q_ptr[2] = len;
+        q_ptr[3] = len >> 8;
+        memcpy(q_ptr + 4, str, len);
+        q_ptr[4 + len] = '\0';
+        q = qstr_add(q_ptr);
     }
-        }
-    }
-    return qstr_add(strndup(str, len));
+    return q;
 }
 
-// convert qstr id to pointer to its string
-const char *qstr_str(qstr qstr) {
-    // search
-    for (qstr_pool_t *pool = last_pool; pool != NULL; pool = pool->prev) {
-        if (qstr >= pool->total_prev_len) {
-            return pool->qstrs[qstr - pool->total_prev_len];
-        }
+qstr qstr_from_strn_take(char *str, uint alloc_len, uint len) {
+    qstr q = qstr_from_strn(str, len);
+    m_del(char, str, alloc_len);
+    return q;
+}
+
+byte *qstr_build_start(uint len, byte **q_ptr) {
+    assert(len <= 65535);
+    *q_ptr = m_new(byte, 4 + len + 1);
+    (*q_ptr)[2] = len;
+    (*q_ptr)[3] = len >> 8;
+    return Q_GET_DATA(*q_ptr);
+}
+
+qstr qstr_build_end(byte *q_ptr) {
+    qstr q = qstr_find_strn(Q_GET_DATA(q_ptr), Q_GET_LENGTH(q_ptr));
+    if (q == 0) {
+        machine_uint_t len = Q_GET_LENGTH(q_ptr);
+        machine_uint_t hash = compute_hash(Q_GET_DATA(q_ptr), len);
+        q_ptr[0] = hash;
+        q_ptr[1] = hash >> 8;
+        q_ptr[4 + len] = '\0';
+        q = qstr_add(q_ptr);
+    } else {
+        m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr));
     }
+    return q;
+}
+
+machine_uint_t qstr_hash(qstr q) {
+    return Q_GET_HASH(find_qstr(q));
+}
+
+uint qstr_len(qstr q) {
+    const byte *qd = find_qstr(q);
+    return Q_GET_LENGTH(qd);
+}
+
+// XXX to remove!
+const char *qstr_str(qstr q) {
+    const byte *qd = find_qstr(q);
+    return (const char*)Q_GET_DATA(qd);
+}
 
-    // not found, return nil
-    return const_pool.qstrs[0];
+const byte *qstr_data(qstr q, uint *len) {
+    const byte *qd = find_qstr(q);
+    *len = Q_GET_LENGTH(qd);
+    return Q_GET_DATA(qd);
 }

py/qstr.h

+// See qstrraw.h for a list of qstr's that are available as constants.
+// Reference them as MP_QSTR_xxxx.
+//
+// Note: it would be possible to define MP_QSTR_xxx as qstr_from_str_static("xxx")
+// for qstrs that are referenced this way, but you don't want to have them in ROM.
+
+enum {
+    MP_QSTR_NULL = 0, // indicates invalid/no qstr
+    MP_QSTR_ = 1, // the empty qstr
+#define Q(id, str) MP_QSTR_##id,
+// TODO having 'build/py.' here is a bit of a hack, should take config variable from Makefile
+#include "build/py.qstrdefs.generated.h"
+#undef Q
+    MP_QSTR_number_of,
+} category_t;
+
+typedef machine_uint_t qstr;
+
+#define QSTR_FROM_STR_STATIC(s) (qstr_from_strn((s), strlen(s)))
+
+void qstr_init(void);
+
+qstr qstr_from_str(const char *str);
+qstr qstr_from_strn(const char *str, uint len);
+//qstr qstr_from_str_static(const char *str);
+qstr qstr_from_strn_take(char *str, uint alloc_len, uint len);
+//qstr qstr_from_strn_copy(const char *str, int len);
+
+byte* qstr_build_start(uint len, byte **q_ptr);
+qstr qstr_build_end(byte *q_ptr);
+
+machine_uint_t qstr_hash(qstr q);
+const char* qstr_str(qstr q);
+uint qstr_len(qstr q);
+const byte* qstr_data(qstr q, uint *len);

py/runtime.c

@@ -11,7 +11,7 @@
 #include "nlr.h"
 #include "misc.h"
 #include "mpconfig.h"
-#include "mpqstr.h"
+#include "qstr.h"
 #include "obj.h"
 #include "runtime0.h"
 #include "runtime.h"
@@ -154,7 +154,7 @@ void rt_init(void) {
 
 #if MICROPY_CPYTHON_COMPAT
     // Precreate sys module, so "import sys" didn't throw exceptions.
-    mp_obj_new_module(qstr_from_str_static("sys"));
+    mp_obj_new_module(QSTR_FROM_STR_STATIC("sys"));