Commit 9193f892 authored by Damien George's avatar Damien George
Browse files

Move lexerstr to main py directory (everyone uses it).

parent dc141db6
...@@ -127,6 +127,8 @@ void mp_token_show_error_prefix(const mp_token_t *tok); ...@@ -127,6 +127,8 @@ void mp_token_show_error_prefix(const mp_token_t *tok);
bool mp_token_show_error(const mp_token_t *tok, const char *msg); bool mp_token_show_error(const mp_token_t *tok, const char *msg);
mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close); mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close);
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len);
void mp_lexer_free(mp_lexer_t *lex); void mp_lexer_free(mp_lexer_t *lex);
void mp_lexer_to_next(mp_lexer_t *lex); void mp_lexer_to_next(mp_lexer_t *lex);
const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex); const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex);
......
...@@ -3,9 +3,15 @@ ...@@ -3,9 +3,15 @@
#include "misc.h" #include "misc.h"
#include "lexer.h" #include "lexer.h"
#include "lexerstr.h"
unichar str_buf_next_char(mp_lexer_str_buf_t *sb) { typedef struct _mp_lexer_str_buf_t {
uint free_len; // if > 0, src_beg will be freed when done by: m_free(src_beg, free_len)
const char *src_beg; // beginning of source
const char *src_cur; // current location in source
const char *src_end; // end (exclusive) of source
} mp_lexer_str_buf_t;
static unichar str_buf_next_char(mp_lexer_str_buf_t *sb) {
if (sb->src_cur < sb->src_end) { if (sb->src_cur < sb->src_end) {
return *sb->src_cur++; return *sb->src_cur++;
} else { } else {
...@@ -13,14 +19,15 @@ unichar str_buf_next_char(mp_lexer_str_buf_t *sb) { ...@@ -13,14 +19,15 @@ unichar str_buf_next_char(mp_lexer_str_buf_t *sb) {
} }
} }
void str_buf_free(mp_lexer_str_buf_t *sb) { static void str_buf_free(mp_lexer_str_buf_t *sb) {
if (sb->free) { if (sb->free_len > 0) {
m_del(char, (char*)sb->src_beg, 0 /* don't know allocated size of src */); m_free((char*)sb->src_beg, sb->free_len);
} }
} }
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, mp_lexer_str_buf_t *sb) { mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len) {
sb->free = free_str; mp_lexer_str_buf_t *sb = m_new_obj(mp_lexer_str_buf_t);
sb->free_len = free_len;
sb->src_beg = str; sb->src_beg = str;
sb->src_cur = str; sb->src_cur = str;
sb->src_end = str + len; sb->src_end = str + len;
......
...@@ -6,42 +6,10 @@ ...@@ -6,42 +6,10 @@
#include "misc.h" #include "misc.h"
#include "mpconfig.h" #include "mpconfig.h"
#include "lexer.h" #include "lexer.h"
#include "lexerunix.h"
#if MICROPY_ENABLE_LEXER_UNIX #if MICROPY_ENABLE_LEXER_UNIX
typedef struct _str_buf_t {
bool free; // free src_beg when done
const char *src_beg; // beginning of source
const char *src_cur; // current location in source
const char *src_end; // end (exclusive) of source
} str_buf_t;
unichar str_buf_next_char(str_buf_t *sb) {
if (sb->src_cur < sb->src_end) {
return *sb->src_cur++;
} else {
return MP_LEXER_CHAR_EOF;
}
}
void str_buf_free(str_buf_t *sb) {
if (sb) {
if (sb->free) {
m_del(char, (char*)sb->src_beg, 0 /* unknown size of src_beg */);
}
m_del_obj(str_buf_t, sb);
}
}
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str) {
str_buf_t *sb = m_new(str_buf_t, 1);
sb->free = free_str;
sb->src_beg = str;
sb->src_cur = str;
sb->src_end = str + len;
return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_char_t)str_buf_next_char, (mp_lexer_stream_close_t)str_buf_free);
}
mp_lexer_t *mp_lexer_new_from_file(const char *filename) { mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
int fd = open(filename, O_RDONLY); int fd = open(filename, O_RDONLY);
if (fd < 0) { if (fd < 0) {
...@@ -59,7 +27,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) { ...@@ -59,7 +27,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
return NULL; return NULL;
} }
return mp_lexer_new_from_str_len(filename, data, size, true); return mp_lexer_new_from_str_len(filename, data, size, size);
} }
/******************************************************************************/ /******************************************************************************/
......
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str);
mp_lexer_t *mp_lexer_new_from_file(const char *filename); mp_lexer_t *mp_lexer_new_from_file(const char *filename);
void mp_import_set_directory(const char *dir); void mp_import_set_directory(const char *dir);
...@@ -29,6 +29,7 @@ PY_O_BASENAME = \ ...@@ -29,6 +29,7 @@ PY_O_BASENAME = \
vstr.o \ vstr.o \
unicode.o \ unicode.o \
lexer.o \ lexer.o \
lexerstr.o \
lexerunix.o \ lexerunix.o \
parse.o \ parse.o \
scope.o \ scope.o \
......
...@@ -30,7 +30,6 @@ SRC_C = \ ...@@ -30,7 +30,6 @@ SRC_C = \
string0.c \ string0.c \
malloc0.c \ malloc0.c \
systick.c \ systick.c \
lexerstr.c \
lexerfatfs.c \ lexerfatfs.c \
led.c \ led.c \
lcd.c \ lcd.c \
......
typedef struct _py_lexer_file_buf_t { typedef struct _mp_lexer_file_buf_t {
FIL fp; FIL fp;
char buf[20]; char buf[20];
uint16_t len; uint16_t len;
......
typedef struct _py_lexer_str_buf_t {
bool free; // free src_beg when done
const char *src_beg; // beginning of source
const char *src_cur; // current location in source
const char *src_end; // end (exclusive) of source
} mp_lexer_str_buf_t;
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, mp_lexer_str_buf_t *sb);
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
#include "nlr.h" #include "nlr.h"
#include "misc.h" #include "misc.h"
#include "lexer.h" #include "lexer.h"
#include "lexerstr.h"
#include "lexerfatfs.h" #include "lexerfatfs.h"
#include "parse.h" #include "parse.h"
#include "obj.h" #include "obj.h"
...@@ -432,8 +431,7 @@ void do_repl(void) { ...@@ -432,8 +431,7 @@ void do_repl(void) {
} }
} }
mp_lexer_str_buf_t sb; mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", vstr_str(&line), vstr_len(&line), 0);
mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", vstr_str(&line), vstr_len(&line), false, &sb);
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT); mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT);
mp_lexer_free(lex); mp_lexer_free(lex);
...@@ -1059,8 +1057,7 @@ soft_reset: ...@@ -1059,8 +1057,7 @@ soft_reset:
" pass\n" " pass\n"
"f()\n"; "f()\n";
mp_lexer_str_buf_t mp_lexer_str_buf; mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), 0);
mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), false, &mp_lexer_str_buf);
// nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler // nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
printf("lex; al=%u\n", m_get_total_bytes_allocated()); printf("lex; al=%u\n", m_get_total_bytes_allocated());
......
...@@ -29,7 +29,6 @@ SRC_C = \ ...@@ -29,7 +29,6 @@ SRC_C = \
usb.c \ usb.c \
STM_SRC_C = \ STM_SRC_C = \
lexerstr.c \
malloc0.c \ malloc0.c \
printf.c \ printf.c \
string0.c \ string0.c \
...@@ -45,6 +44,7 @@ PY_O = \ ...@@ -45,6 +44,7 @@ PY_O = \
vstr.o \ vstr.o \
unicode.o \ unicode.o \
lexer.o \ lexer.o \
lexerstr.o \
parse.o \ parse.o \
scope.o \ scope.o \
compile.o \ compile.o \
......
...@@ -8,7 +8,6 @@ ...@@ -8,7 +8,6 @@
#include "mpconfig.h" #include "mpconfig.h"
#include "mpqstr.h" #include "mpqstr.h"
#include "lexer.h" #include "lexer.h"
#include "../stm/lexerstr.h"
#include "parse.h" #include "parse.h"
#include "obj.h" #include "obj.h"
#include "compile.h" #include "compile.h"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment