Move lexerstr to main py directory (everyone uses it).

py/lexer.h

@@ -127,6 +127,8 @@ void mp_token_show_error_prefix(const mp_token_t *tok);
 bool mp_token_show_error(const mp_token_t *tok, const char *msg);
 
 mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close);
+mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len);
+
 void mp_lexer_free(mp_lexer_t *lex);
 void mp_lexer_to_next(mp_lexer_t *lex);
 const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex);

stm/lexerstr.c → py/lexerstr.c

@@ -3,9 +3,15 @@
 
 #include "misc.h"
 #include "lexer.h"
-#include "lexerstr.h"
 
-unichar str_buf_next_char(mp_lexer_str_buf_t *sb) {
+typedef struct _mp_lexer_str_buf_t {
+    uint free_len;              // if > 0, src_beg will be freed when done by: m_free(src_beg, free_len)
+    const char *src_beg;        // beginning of source
+    const char *src_cur;        // current location in source
+    const char *src_end;        // end (exclusive) of source
+} mp_lexer_str_buf_t;
+
+static unichar str_buf_next_char(mp_lexer_str_buf_t *sb) {
     if (sb->src_cur < sb->src_end) {
         return *sb->src_cur++;
     } else {
@@ -13,14 +19,15 @@ unichar str_buf_next_char(mp_lexer_str_buf_t *sb) {
     }
 }
 
-void str_buf_free(mp_lexer_str_buf_t *sb) {
-    if (sb->free) {
-        m_del(char, (char*)sb->src_beg, 0 /* don't know allocated size of src */);
+static void str_buf_free(mp_lexer_str_buf_t *sb) {
+    if (sb->free_len > 0) {
+        m_free((char*)sb->src_beg, sb->free_len);
     }
 }
 
-mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, mp_lexer_str_buf_t *sb) {
-    sb->free = free_str;
+mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len) {
+    mp_lexer_str_buf_t *sb = m_new_obj(mp_lexer_str_buf_t);
+    sb->free_len = free_len;
     sb->src_beg = str;
     sb->src_cur = str;
     sb->src_end = str + len;

py/lexerunix.c

@@ -6,42 +6,10 @@
 #include "misc.h"
 #include "mpconfig.h"
 #include "lexer.h"
+#include "lexerunix.h"
 
 #if MICROPY_ENABLE_LEXER_UNIX
 
-typedef struct _str_buf_t {
-    bool free;                  // free src_beg when done
-    const char *src_beg;        // beginning of source
-    const char *src_cur;        // current location in source
-    const char *src_end;        // end (exclusive) of source
-} str_buf_t;
-
-unichar str_buf_next_char(str_buf_t *sb) {
-    if (sb->src_cur < sb->src_end) {
-        return *sb->src_cur++;
-    } else {
-        return MP_LEXER_CHAR_EOF;
-    }
-}
-
-void str_buf_free(str_buf_t *sb) {
-    if (sb) {
-        if (sb->free) {
-            m_del(char, (char*)sb->src_beg, 0 /* unknown size of src_beg */);
-        }
-        m_del_obj(str_buf_t, sb);
-    }
-}
-
-mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str) {
-    str_buf_t *sb = m_new(str_buf_t, 1);
-    sb->free = free_str;
-    sb->src_beg = str;
-    sb->src_cur = str;
-    sb->src_end = str + len;
-    return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_char_t)str_buf_next_char, (mp_lexer_stream_close_t)str_buf_free);
-}
-
 mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
     int fd = open(filename, O_RDONLY);
     if (fd < 0) {
@@ -59,7 +27,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
         return NULL;
     }
 
-    return mp_lexer_new_from_str_len(filename, data, size, true);
+    return mp_lexer_new_from_str_len(filename, data, size, size);
 }
 
 /******************************************************************************/

py/lexerunix.h

-mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str);
 mp_lexer_t *mp_lexer_new_from_file(const char *filename);
 
 void mp_import_set_directory(const char *dir);

py/py.mk

@@ -29,6 +29,7 @@ PY_O_BASENAME = \
 	vstr.o \
 	unicode.o \
 	lexer.o \
+	lexerstr.o \
 	lexerunix.o \
 	parse.o \
 	scope.o \

stm/Makefile

@@ -30,7 +30,6 @@ SRC_C = \
 	string0.c \
 	malloc0.c \
 	systick.c  \
-	lexerstr.c \
 	lexerfatfs.c \
 	led.c \
 	lcd.c \

stm/lexerfatfs.h

-typedef struct _py_lexer_file_buf_t {
+typedef struct _mp_lexer_file_buf_t {
     FIL fp;
     char buf[20];
     uint16_t len;

stm/lexerstr.h

-typedef struct _py_lexer_str_buf_t {
-    bool free;                  // free src_beg when done
-    const char *src_beg;        // beginning of source
-    const char *src_cur;        // current location in source
-    const char *src_end;        // end (exclusive) of source
-} mp_lexer_str_buf_t;
-
-mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, mp_lexer_str_buf_t *sb);

stm/main.c

@@ -19,7 +19,6 @@
 #include "nlr.h"
 #include "misc.h"
 #include "lexer.h"
-#include "lexerstr.h"
 #include "lexerfatfs.h"
 #include "parse.h"
 #include "obj.h"
@@ -432,8 +431,7 @@ void do_repl(void) {
             }
         }
 
-        mp_lexer_str_buf_t sb;
-        mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", vstr_str(&line), vstr_len(&line), false, &sb);
+        mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", vstr_str(&line), vstr_len(&line), 0);
         mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT);
         mp_lexer_free(lex);
 
@@ -1059,8 +1057,7 @@ soft_reset:
             "                pass\n"
             "f()\n";
 
-        mp_lexer_str_buf_t mp_lexer_str_buf;
-        mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), false, &mp_lexer_str_buf);
+        mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), 0);
 
         // nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
         printf("lex; al=%u\n", m_get_total_bytes_allocated());

teensy/Makefile

@@ -29,7 +29,6 @@ SRC_C = \
 	usb.c \
 
 STM_SRC_C = \
-	lexerstr.c \
 	malloc0.c \
 	printf.c \
 	string0.c \
@@ -45,6 +44,7 @@ PY_O = \
 	vstr.o \
 	unicode.o \
 	lexer.o \
+	lexerstr.o \
 	parse.o \
 	scope.o \
 	compile.o \

teensy/main.c

@@ -8,7 +8,6 @@
 #include "mpconfig.h"
 #include "mpqstr.h"
 #include "lexer.h"
-#include "../stm/lexerstr.h"
 #include "parse.h"
 #include "obj.h"
 #include "compile.h"