Commit 0bfc7638 authored by Damien George's avatar Damien George
Browse files

py: Protect mp_parse and mp_compile with nlr push/pop block.

To enable parsing constants more efficiently, mp_parse should be allowed
to raise an exception, and mp_compile can already raise a MemoryError.
So these functions need to be protected by an nlr push/pop block.

This patch adds that feature in all places.  This allows to simplify how
mp_parse and mp_compile are called: they now raise an exception if they
have an error and so explicit checking is not needed anymore.
parent e1e359ff
......@@ -3,7 +3,6 @@
#include <string.h>
#include "py/nlr.h"
#include "py/parsehelper.h"
#include "py/compile.h"
#include "py/runtime.h"
#include "py/repl.h"
......@@ -15,29 +14,11 @@ void do_str(const char *src) {
return;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
// parse okay
qstr source_name = lex->source_name;
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
if (mp_obj_is_exception_instance(module_fun)) {
// compile error
mp_obj_print_exception(printf_wrapper, NULL, module_fun);
return;
}
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
qstr source_name = lex->source_name;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
mp_call_function_0(module_fun);
nlr_pop();
} else {
......
......@@ -36,7 +36,6 @@
#include "lexer.h"
#include "parse.h"
#include "obj.h"
#include "parsehelper.h"
#include "compile.h"
#include "runtime0.h"
#include "runtime.h"
......
......@@ -28,7 +28,6 @@
#include <string.h>
#include "py/nlr.h"
#include "py/parsehelper.h"
#include "py/compile.h"
#include "py/runtime0.h"
#include "py/runtime.h"
......
......@@ -3,7 +3,6 @@
#include <string.h>
#include "py/nlr.h"
#include "py/parsehelper.h"
#include "py/compile.h"
#include "py/runtime.h"
#include "py/repl.h"
......@@ -15,32 +14,15 @@
void do_str(const char *src) {
mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0);
if (lex == NULL) {
return;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
// parse okay
qstr source_name = lex->source_name;
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
if (mp_obj_is_exception_instance(module_fun)) {
// compile error
mp_obj_print_exception(printf_wrapper, NULL, module_fun);
printf("MemoryError: lexer could not allocate memory\n");
return;
}
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
qstr source_name = lex->source_name;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
mp_call_function_0(module_fun);
nlr_pop();
} else {
......
......@@ -3823,7 +3823,7 @@ mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is
m_del_obj(compiler_t, comp);
if (compile_error != MP_OBJ_NULL) {
return compile_error;
nlr_raise(compile_error);
} else {
#if MICROPY_EMIT_CPYTHON
// can't create code, so just return true
......
......@@ -39,6 +39,7 @@ enum {
MP_EMIT_OPT_ASM_THUMB,
};
// the compiler will raise an exception if an error occurred
// the compiler will free the parse tree (pn) before it returns
mp_obj_t mp_compile(mp_parse_node_t pn, qstr source_file, uint emit_opt, bool is_repl);
......
......@@ -30,6 +30,7 @@
#include <assert.h>
#include <string.h>
#include "py/nlr.h"
#include "py/lexer.h"
#include "py/parse.h"
#include "py/parsenum.h"
......@@ -382,7 +383,7 @@ STATIC void push_result_rule(parser_t *parser, mp_uint_t src_line, const rule_t
push_result_node(parser, (mp_parse_node_t)pn);
}
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) {
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// initialise parser and allocate memory for its stacks
......@@ -717,15 +718,15 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
}
}
mp_parse_node_t result;
mp_obj_t exc = MP_OBJ_NULL;
mp_parse_node_t result = MP_PARSE_NODE_NULL;
// check if we had a memory error
if (parser.had_memory_error) {
memory_error:
*parse_error_kind_out = MP_PARSE_ERROR_MEMORY;
result = MP_PARSE_NODE_NULL;
exc = mp_obj_new_exception_msg(&mp_type_MemoryError,
"parser could not allocate enough memory");
goto finished;
}
// check we are at the end of the token stream
......@@ -747,17 +748,30 @@ finished:
// free the memory that we don't need anymore
m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
// return the result
return result;
// we also free the lexer on behalf of the caller (see below)
if (exc != MP_OBJ_NULL) {
// had an error so raise the exception
// add traceback to give info about file name and location
// we don't have a 'block' name, so just pass the NULL qstr to indicate this
mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL);
mp_lexer_free(lex);
nlr_raise(exc);
} else {
mp_lexer_free(lex);
return result;
}
syntax_error:
if (lex->tok_kind == MP_TOKEN_INDENT) {
*parse_error_kind_out = MP_PARSE_ERROR_UNEXPECTED_INDENT;
exc = mp_obj_new_exception_msg(&mp_type_IndentationError,
"unexpected indent");
} else if (lex->tok_kind == MP_TOKEN_DEDENT_MISMATCH) {
*parse_error_kind_out = MP_PARSE_ERROR_UNMATCHED_UNINDENT;
exc = mp_obj_new_exception_msg(&mp_type_IndentationError,
"unindent does not match any outer indentation level");
} else {
*parse_error_kind_out = MP_PARSE_ERROR_INVALID_SYNTAX;
exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
"invalid syntax");
#ifdef USE_RULE_NAME
// debugging: print the rule name that failed and the token
printf("rule: %s\n", rule->rule_name);
......@@ -766,6 +780,5 @@ syntax_error:
#endif
#endif
}
result = MP_PARSE_NODE_NULL;
goto finished;
}
......@@ -90,14 +90,8 @@ typedef enum {
MP_PARSE_EVAL_INPUT,
} mp_parse_input_kind_t;
typedef enum {
MP_PARSE_ERROR_MEMORY,
MP_PARSE_ERROR_UNEXPECTED_INDENT,
MP_PARSE_ERROR_UNMATCHED_UNINDENT,
MP_PARSE_ERROR_INVALID_SYNTAX,
} mp_parse_error_kind_t;
// returns MP_PARSE_NODE_NULL on error, and then parse_error_kind_out is valid
mp_parse_node_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out);
// the parser will raise an exception if an error occurred
// the parser will free the lexer before it returns
mp_parse_node_t mp_parse(struct _mp_lexer_t *lex, mp_parse_input_kind_t input_kind);
#endif // __MICROPY_INCLUDED_PY_PARSE_H__
/*
* This file is part of the Micro Python project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013, 2014 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
// these functions are separate from parse.c to keep parser independent of mp_obj_t
#include <stdio.h>
#include "py/parsehelper.h"
#define STR_MEMORY "parser could not allocate enough memory"
#define STR_UNEXPECTED_INDENT "unexpected indent"
#define STR_UNMATCHED_UNINDENT "unindent does not match any outer indentation level"
#define STR_INVALID_SYNTAX "invalid syntax"
void mp_parse_show_exception(mp_lexer_t *lex, mp_parse_error_kind_t parse_error_kind) {
printf(" File \"%s\", line " UINT_FMT ", column " UINT_FMT "\n", qstr_str(lex->source_name), lex->tok_line, lex->tok_column);
switch (parse_error_kind) {
case MP_PARSE_ERROR_MEMORY:
printf("MemoryError: %s\n", STR_MEMORY);
break;
case MP_PARSE_ERROR_UNEXPECTED_INDENT:
printf("IndentationError: %s\n", STR_UNEXPECTED_INDENT);
break;
case MP_PARSE_ERROR_UNMATCHED_UNINDENT:
printf("IndentationError: %s\n", STR_UNMATCHED_UNINDENT);
break;
case MP_PARSE_ERROR_INVALID_SYNTAX:
default:
printf("SyntaxError: %s\n", STR_INVALID_SYNTAX);
break;
}
}
mp_obj_t mp_parse_make_exception(mp_lexer_t *lex, mp_parse_error_kind_t parse_error_kind) {
// make exception object
mp_obj_t exc;
switch (parse_error_kind) {
case MP_PARSE_ERROR_MEMORY:
exc = mp_obj_new_exception_msg(&mp_type_MemoryError, STR_MEMORY);
break;
case MP_PARSE_ERROR_UNEXPECTED_INDENT:
exc = mp_obj_new_exception_msg(&mp_type_IndentationError, STR_UNEXPECTED_INDENT);
break;
case MP_PARSE_ERROR_UNMATCHED_UNINDENT:
exc = mp_obj_new_exception_msg(&mp_type_IndentationError, STR_UNMATCHED_UNINDENT);
break;
case MP_PARSE_ERROR_INVALID_SYNTAX:
default:
exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, STR_INVALID_SYNTAX);
break;
}
// add traceback to give info about file name and location
// we don't have a 'block' name, so just pass the NULL qstr to indicate this
mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL);
return exc;
}
/*
* This file is part of the Micro Python project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013, 2014 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef __MICROPY_INCLUDED_PY_PARSEHELPER_H__
#define __MICROPY_INCLUDED_PY_PARSEHELPER_H__
#include "py/lexer.h"
#include "py/parse.h"
#include "py/obj.h"
void mp_parse_show_exception(mp_lexer_t *lex, mp_parse_error_kind_t parse_error_kind);
mp_obj_t mp_parse_make_exception(mp_lexer_t *lex, mp_parse_error_kind_t parse_error_kind);
#endif // __MICROPY_INCLUDED_PY_PARSEHELPER_H__
......@@ -28,7 +28,6 @@ PY_O_BASENAME = \
lexerstr.o \
lexerunix.o \
parse.o \
parsehelper.o \
scope.o \
compile.o \
emitcommon.o \
......
......@@ -30,7 +30,6 @@
#include "py/mpstate.h"
#include "py/nlr.h"
#include "py/parsehelper.h"
#include "py/parsenum.h"
#include "py/compile.h"
#include "py/objtuple.h"
......@@ -1228,47 +1227,30 @@ void mp_import_all(mp_obj_t module) {
// this is implemented in this file so it can optimise access to locals/globals
mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_input_kind, mp_obj_dict_t *globals, mp_obj_dict_t *locals) {
// parse the string
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, parse_input_kind, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error; raise exception
mp_obj_t exc = mp_parse_make_exception(lex, parse_error_kind);
mp_lexer_free(lex);
nlr_raise(exc);
}
qstr source_name = lex->source_name;
mp_lexer_free(lex);
// save context
mp_obj_dict_t *volatile old_globals = mp_globals_get();
mp_obj_dict_t *volatile old_locals = mp_locals_get();
// save context and set new context
mp_obj_dict_t *old_globals = mp_globals_get();
mp_obj_dict_t *old_locals = mp_locals_get();
// set new context
mp_globals_set(globals);
mp_locals_set(locals);
// compile the string
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, false);
// check if there was a compile error
if (mp_obj_is_exception_instance(module_fun)) {
mp_globals_set(old_globals);
mp_locals_set(old_locals);
nlr_raise(module_fun);
}
// for compile only
if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) {
mp_globals_set(old_globals);
mp_locals_set(old_locals);
return module_fun;
}
// complied successfully, execute it
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
mp_obj_t ret = mp_call_function_0(module_fun);
qstr source_name = lex->source_name;
mp_parse_node_t pn = mp_parse(lex, parse_input_kind);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, false);
mp_obj_t ret;
if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) {
// for compile only, return value is the module function
ret = module_fun;
} else {
// execute module function and get return value
ret = mp_call_function_0(module_fun);
}
// finish nlr block, restore context and return value
nlr_pop();
mp_globals_set(old_globals);
mp_locals_set(old_locals);
......
......@@ -5,7 +5,6 @@
#include "py/nlr.h"
#include "py/obj.h"
#include "py/parsehelper.h"
#include "py/compile.h"
#include "py/runtime0.h"
#include "py/runtime.h"
......@@ -20,29 +19,11 @@ void do_str(const char *src) {
return;
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
// parse okay
qstr source_name = lex->source_name;
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
if (mp_obj_is_exception_instance(module_fun)) {
// compile error
mp_obj_print_exception(printf_wrapper, NULL, module_fun);
return;
}
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
qstr source_name = lex->source_name;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
mp_call_function_0(module_fun);
nlr_pop();
} else {
......
......@@ -5,7 +5,6 @@
#include "py/nlr.h"
#include "py/obj.h"
#include "py/parsehelper.h"
#include "py/compile.h"
#include "py/runtime0.h"
#include "py/runtime.h"
......@@ -24,27 +23,11 @@ inline void do_str(const char *src) {
tt_abort_msg("Lexer initialization error");
}
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
tt_abort_msg("Parser error");
}
// parse okay
qstr source_name = lex->source_name;
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
if (mp_obj_is_exception_instance(module_fun)) {
mp_obj_print_exception(printf_wrapper, NULL, module_fun);
tt_abort_msg("Compile error");
}
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
qstr source_name = lex->source_name;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, true);
mp_call_function_0(module_fun);
nlr_pop();
} else {
......
......@@ -29,7 +29,6 @@
#include <stdint.h>
#include "py/nlr.h"
#include "py/parsehelper.h"
#include "py/compile.h"
#include "py/runtime.h"
#include "py/repl.h"
......@@ -57,39 +56,18 @@ STATIC bool repl_display_debugging_info = 0;
// EXEC_FLAG_IS_REPL is used for REPL inputs (flag passed on to mp_compile)
STATIC int parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, int exec_flags) {
int ret = 0;
uint32_t start = 0;
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, input_kind, &parse_error_kind);
qstr source_name = lex->source_name;
// check for parse error
if (pn == MP_PARSE_NODE_NULL) {
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
goto finish;
}
mp_lexer_free(lex);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
// check for compile error
if (mp_obj_is_exception_instance(module_fun)) {
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
mp_obj_print_exception(printf_wrapper, NULL, module_fun);
goto finish;
}
// execute code
nlr_buf_t nlr;
uint32_t start = HAL_GetTick();
if (nlr_push(&nlr) == 0) {
// parse and compile the script
qstr source_name = lex->source_name;
mp_parse_node_t pn = mp_parse(lex, input_kind);
mp_obj_t module_fun = mp_compile(pn, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
// execute code
mp_hal_set_interrupt_char(CHAR_CTRL_C); // allow ctrl-C to interrupt us
start = HAL_GetTick();
mp_call_function_0(module_fun);
mp_hal_set_interrupt_char(-1); // disable interrupt
nlr_pop();
......@@ -131,7 +109,6 @@ STATIC int parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t input_ki
gc_dump_info();
}
finish:
if (exec_flags & EXEC_FLAG_PRINT_EOF) {
stdout_tx_strn("\x04", 1);
}
......
......@@ -30,7 +30,6 @@
#include <string.h>
#include "py/nlr.h"
#include "py/parsehelper.h"
#include "py/compile.h"
#include "py/runtime.h"
#include "py/pfenv.h"
......@@ -51,17 +50,7 @@ void do_file(const char *file) {
} else {
// parse
mp_parse_error_kind_t parse_error_kind;
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT, &parse_error_kind);
if (pn == MP_PARSE_NODE_NULL) {
// parse error
mp_parse_show_exception(lex, parse_error_kind);
mp_lexer_free(lex);
return;
}
mp_lexer_free(lex);
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT);
if (pn != MP_PARSE_NODE_NULL) {
//printf("----------------\n");
......@@ -72,10 +61,6 @@ void do_file(const char *file) {
mp_obj_t module_fun = mp_compile(pn, 0, MP_EMIT_OPT_NONE, false);
//printf("----------------\n");
if (mp_obj_is_exception_instance(module_fun)) {
mp_obj_print_exception(printf_wrapper, NULL, module_fun);
}
}
}
}
......
......@@ -38,7 +38,6 @@
#include "py/mpstate.h"
#include "py/nlr.h"
#include "py/compile.h"
#include "py/parsehelper.h"
#include "py/runtime.h"
#include "py/builtin.h"
#include "py/repl.h"
......@@ -101,55 +100,10 @@ STATIC int handle_uncaught_exception(mp_obj_t exc) {
// value of the exit is in the lower 8 bits of the return value
STATIC int execute_from_lexer(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, bool is_repl) {
if (lex == NULL) {
printf("MemoryError: lexer could not allocate memory\n");
return 1;
}
if (0) {
// just tokenise
while (lex->tok_kind != MP_TOKEN_END) {
mp_lexer_show_token(lex);
mp_lexer_to_next(lex);