Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
TASTE
uPython-mirror
Commits
5bdf1650
Commit
5bdf1650
authored
Nov 16, 2016
by
Damien George
Browse files
py/lexer: Make lexer use an mp_reader as its source.
parent
66d955c2
Changes
3
Hide whitespace changes
Inline
Side-by-side
esp8266/lexerstr32.c
View file @
5bdf1650
...
...
@@ -35,10 +35,11 @@ typedef struct _mp_lexer_str32_buf_t {
uint8_t
byte_off
;
}
mp_lexer_str32_buf_t
;
STATIC
mp_uint_t
str32_buf_next_byte
(
mp_lexer_str32_buf_t
*
sb
)
{
STATIC
mp_uint_t
str32_buf_next_byte
(
void
*
sb_in
)
{
mp_lexer_str32_buf_t
*
sb
=
(
mp_lexer_str32_buf_t
*
)
sb_in
;
byte
c
=
sb
->
val
&
0xff
;
if
(
c
==
0
)
{
return
MP_
LEX
ER_EOF
;
return
MP_
READ
ER_EOF
;
}
if
(
++
sb
->
byte_off
>
3
)
{
...
...
@@ -51,7 +52,8 @@ STATIC mp_uint_t str32_buf_next_byte(mp_lexer_str32_buf_t *sb) {
return
c
;
}
STATIC
void
str32_buf_free
(
mp_lexer_str32_buf_t
*
sb
)
{
STATIC
void
str32_buf_free
(
void
*
sb_in
)
{
mp_lexer_str32_buf_t
*
sb
=
(
mp_lexer_str32_buf_t
*
)
sb_in
;
m_del_obj
(
mp_lexer_str32_buf_t
,
sb
);
}
...
...
@@ -63,7 +65,8 @@ mp_lexer_t *mp_lexer_new_from_str32(qstr src_name, const char *str, mp_uint_t le
sb
->
byte_off
=
(
uint32_t
)
str
&
3
;
sb
->
src_cur
=
(
uint32_t
*
)(
str
-
sb
->
byte_off
);
sb
->
val
=
*
sb
->
src_cur
++
>>
sb
->
byte_off
*
8
;
return
mp_lexer_new
(
src_name
,
sb
,
(
mp_lexer_stream_next_byte_t
)
str32_buf_next_byte
,
(
mp_lexer_stream_close_t
)
str32_buf_free
);
mp_reader_t
reader
=
{
sb
,
str32_buf_next_byte
,
str32_buf_free
};
return
mp_lexer_new
(
src_name
,
reader
);
}
#endif // MICROPY_ENABLE_COMPILER
py/lexer.c
View file @
5bdf1650
...
...
@@ -52,6 +52,7 @@ STATIC bool str_strn_equal(const char *str, const char *strn, mp_uint_t len) {
return
i
==
len
&&
*
str
==
0
;
}
#define MP_LEXER_EOF ((unichar)MP_READER_EOF)
#define CUR_CHAR(lex) ((lex)->chr0)
STATIC
bool
is_end
(
mp_lexer_t
*
lex
)
{
...
...
@@ -145,7 +146,7 @@ STATIC void next_char(mp_lexer_t *lex) {
lex
->
chr0
=
lex
->
chr1
;
lex
->
chr1
=
lex
->
chr2
;
lex
->
chr2
=
lex
->
st
rea
m_next_
byte
(
lex
->
st
rea
m_
data
);
lex
->
chr2
=
lex
->
rea
der
.
read
byte
(
lex
->
rea
der
.
data
);
if
(
lex
->
chr0
==
'\r'
)
{
// CR is a new line, converted to LF
...
...
@@ -153,7 +154,7 @@ STATIC void next_char(mp_lexer_t *lex) {
if
(
lex
->
chr1
==
'\n'
)
{
// CR LF is a single new line
lex
->
chr1
=
lex
->
chr2
;
lex
->
chr2
=
lex
->
st
rea
m_next_
byte
(
lex
->
st
rea
m_
data
);
lex
->
chr2
=
lex
->
rea
der
.
read
byte
(
lex
->
rea
der
.
data
);
}
}
...
...
@@ -689,21 +690,17 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
}
}
mp_lexer_t
*
mp_lexer_new
(
qstr
src_name
,
void
*
stream_data
,
mp_lexer_stream_next_byte_t
stream_next_byte
,
mp_lexer_stream_close_t
stream_close
)
{
mp_lexer_t
*
mp_lexer_new
(
qstr
src_name
,
mp_reader_t
reader
)
{
mp_lexer_t
*
lex
=
m_new_obj_maybe
(
mp_lexer_t
);
// check for memory allocation error
if
(
lex
==
NULL
)
{
if
(
stream_close
)
{
stream_close
(
stream_data
);
}
reader
.
close
(
reader
.
data
);
return
NULL
;
}
lex
->
source_name
=
src_name
;
lex
->
stream_data
=
stream_data
;
lex
->
stream_next_byte
=
stream_next_byte
;
lex
->
stream_close
=
stream_close
;
lex
->
reader
=
reader
;
lex
->
line
=
1
;
lex
->
column
=
1
;
lex
->
emit_dent
=
0
;
...
...
@@ -724,9 +721,9 @@ mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_
lex
->
indent_level
[
0
]
=
0
;
// preload characters
lex
->
chr0
=
st
rea
m_next_
byte
(
st
rea
m_
data
);
lex
->
chr1
=
st
rea
m_next_
byte
(
st
rea
m_
data
);
lex
->
chr2
=
st
rea
m_next_
byte
(
st
rea
m_
data
);
lex
->
chr0
=
rea
der
.
read
byte
(
rea
der
.
data
);
lex
->
chr1
=
rea
der
.
read
byte
(
rea
der
.
data
);
lex
->
chr2
=
rea
der
.
read
byte
(
rea
der
.
data
);
// if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end
if
(
lex
->
chr0
==
MP_LEXER_EOF
)
{
...
...
@@ -756,7 +753,7 @@ mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, mp_uint_t
if
(
!
mp_reader_new_mem
(
&
reader
,
(
const
byte
*
)
str
,
len
,
free_len
))
{
return
NULL
;
}
return
mp_lexer_new
(
src_name
,
reader
.
data
,
(
mp_lexer_stream_next_byte_t
)
reader
.
readbyte
,
(
mp_lexer_stream_close_t
)
reader
.
close
);
return
mp_lexer_new
(
src_name
,
reader
);
}
#if MICROPY_READER_POSIX || MICROPY_READER_FATFS
...
...
@@ -767,7 +764,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
if
(
ret
!=
0
)
{
return
NULL
;
}
return
mp_lexer_new
(
qstr_from_str
(
filename
),
reader
.
data
,
(
mp_lexer_stream_next_byte_t
)
reader
.
readbyte
,
(
mp_lexer_stream_close_t
)
reader
.
close
);
return
mp_lexer_new
(
qstr_from_str
(
filename
),
reader
);
}
#if MICROPY_HELPER_LEXER_UNIX
...
...
@@ -778,7 +775,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
if
(
ret
!=
0
)
{
return
NULL
;
}
return
mp_lexer_new
(
filename
,
reader
.
data
,
(
mp_lexer_stream_next_byte_t
)
reader
.
readbyte
,
(
mp_lexer_stream_close_t
)
reader
.
close
);
return
mp_lexer_new
(
filename
,
reader
);
}
#endif
...
...
@@ -787,9 +784,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
void
mp_lexer_free
(
mp_lexer_t
*
lex
)
{
if
(
lex
)
{
if
(
lex
->
stream_close
)
{
lex
->
stream_close
(
lex
->
stream_data
);
}
lex
->
reader
.
close
(
lex
->
reader
.
data
);
vstr_clear
(
&
lex
->
vstr
);
m_del
(
uint16_t
,
lex
->
indent_level
,
lex
->
alloc_indent_level
);
m_del_obj
(
mp_lexer_t
,
lex
);
...
...
py/lexer.h
View file @
5bdf1650
...
...
@@ -30,6 +30,7 @@
#include
"py/mpconfig.h"
#include
"py/qstr.h"
#include
"py/reader.h"
/* lexer.h -- simple tokeniser for Micro Python
*
...
...
@@ -142,21 +143,11 @@ typedef enum _mp_token_kind_t {
MP_TOKEN_DEL_MINUS_MORE
,
}
mp_token_kind_t
;
// the next-byte function must return the next byte in the stream
// it must return MP_LEXER_EOF if end of stream
// it can be called again after returning MP_LEXER_EOF, and in that case must return MP_LEXER_EOF
#define MP_LEXER_EOF ((unichar)(-1))
typedef
mp_uint_t
(
*
mp_lexer_stream_next_byte_t
)(
void
*
);
typedef
void
(
*
mp_lexer_stream_close_t
)(
void
*
);
// this data structure is exposed for efficiency
// public members are: source_name, tok_line, tok_column, tok_kind, vstr
typedef
struct
_mp_lexer_t
{
qstr
source_name
;
// name of source
void
*
stream_data
;
// data for stream
mp_lexer_stream_next_byte_t
stream_next_byte
;
// stream callback to get next byte
mp_lexer_stream_close_t
stream_close
;
// stream callback to free
mp_reader_t
reader
;
// stream source
unichar
chr0
,
chr1
,
chr2
;
// current cached characters from source
...
...
@@ -176,7 +167,7 @@ typedef struct _mp_lexer_t {
vstr_t
vstr
;
// token data
}
mp_lexer_t
;
mp_lexer_t
*
mp_lexer_new
(
qstr
src_name
,
void
*
stream_data
,
mp_lexer_stream_next_byte_t
stream_next_byte
,
mp_lexer_stream_close_t
stream_close
);
mp_lexer_t
*
mp_lexer_new
(
qstr
src_name
,
mp_reader_t
reader
);
mp_lexer_t
*
mp_lexer_new_from_str_len
(
qstr
src_name
,
const
char
*
str
,
mp_uint_t
len
,
mp_uint_t
free_len
);
void
mp_lexer_free
(
mp_lexer_t
*
lex
);
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment