compile.c 131 KB
Newer Older
1
2
3
4
5
/*
 * This file is part of the Micro Python project, http://micropython.org/
 *
 * The MIT License (MIT)
 *
6
 * Copyright (c) 2013-2015 Damien P. George
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

xbe's avatar
xbe committed
27
#include <stdbool.h>
Damien's avatar
Damien committed
28
29
30
31
32
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>

33
34
35
36
#include "py/scope.h"
#include "py/emit.h"
#include "py/compile.h"
#include "py/runtime.h"
Damien's avatar
Damien committed
37
38
39
40

// TODO need to mangle __attr names

typedef enum {
41
#define DEF_RULE(rule, comp, kind, ...) PN_##rule,
42
#include "py/grammar.h"
Damien's avatar
Damien committed
43
44
#undef DEF_RULE
    PN_maximum_number_of,
45
    PN_string, // special node for non-interned string
46
    PN_bytes, // special node for non-interned bytes
47
    PN_const_object, // special node for a constant, generic Python object
Damien's avatar
Damien committed
48
49
} pn_kind_t;

50
#define NEED_METHOD_TABLE MICROPY_EMIT_NATIVE
51
52
53
54

#if NEED_METHOD_TABLE

// we need a method table to do the lookup for the emitter functions
55
56
#define EMIT(fun) (comp->emit_method_table->fun(comp->emit))
#define EMIT_ARG(fun, ...) (comp->emit_method_table->fun(comp->emit, __VA_ARGS__))
57
58
#define EMIT_LOAD_FAST(qst, local_num) (comp->emit_method_table->load_id.fast(comp->emit, qst, local_num))
#define EMIT_LOAD_GLOBAL(qst) (comp->emit_method_table->load_id.global(comp->emit, qst))
Damien's avatar
Damien committed
59

60
61
62
63
64
65
66
67
68
69
#else

// if we only have the bytecode emitter enabled then we can do a direct call to the functions
#define EMIT(fun) (mp_emit_bc_##fun(comp->emit))
#define EMIT_ARG(fun, ...) (mp_emit_bc_##fun(comp->emit, __VA_ARGS__))
#define EMIT_LOAD_FAST(qst, local_num) (mp_emit_bc_load_fast(comp->emit, qst, local_num))
#define EMIT_LOAD_GLOBAL(qst) (mp_emit_bc_load_global(comp->emit, qst))

#endif

70
71
72
#define EMIT_INLINE_ASM(fun) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm))
#define EMIT_INLINE_ASM_ARG(fun, ...) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm, __VA_ARGS__))

73
// elements in this struct are ordered to make it compact
Damien's avatar
Damien committed
74
typedef struct _compiler_t {
75
    qstr source_file;
76

77
78
79
    uint8_t is_repl;
    uint8_t pass; // holds enum type pass_kind_t
    uint8_t func_arg_is_super; // used to compile special case of super() function call
80
81
82
    uint8_t have_star;

    // try to keep compiler clean from nlr
83
84
    mp_obj_t compile_error; // set to an exception object if there's an error
    mp_uint_t compile_error_line; // set to best guess of line of error
Damien's avatar
Damien committed
85

86
    uint next_label;
87

88
89
90
    uint16_t num_dict_params;
    uint16_t num_default_params;

91
92
    uint16_t break_label; // highest bit set indicates we are breaking out of a for loop
    uint16_t continue_label;
93
    uint16_t cur_except_level; // increased for SETUP_EXCEPT, SETUP_FINALLY; decreased for POP_BLOCK, POP_EXCEPT
94
    uint16_t break_continue_except_level;
Damien's avatar
Damien committed
95
96
97
98

    scope_t *scope_head;
    scope_t *scope_cur;

99
    emit_t *emit;                                   // current emitter
100
    #if NEED_METHOD_TABLE
101
    const emit_method_table_t *emit_method_table;   // current emit method table
102
    #endif
103

104
    #if MICROPY_EMIT_INLINE_THUMB
105
106
    emit_inline_asm_t *emit_inline_asm;                                   // current emitter for inline asm
    const emit_inline_asm_method_table_t *emit_inline_asm_method_table;   // current emit method table for inline asm
107
    #endif
Damien's avatar
Damien committed
108
109
} compiler_t;

110
111
112
113
STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) {
    // if the line of the error is unknown then try to update it from the pn
    if (comp->compile_error_line == 0 && MP_PARSE_NODE_IS_STRUCT(pn)) {
        comp->compile_error_line = (mp_uint_t)((mp_parse_node_struct_t*)pn)->source_line;
114
    }
115
116
117
}

STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const char *msg) {
118
119
120
121
122
    // only register the error if there has been no other error
    if (comp->compile_error == MP_OBJ_NULL) {
        comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg);
        compile_error_set_line(comp, pn);
    }
123
124
}

125
STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_arglist, bool is_method_call, int n_positional_extra);
126
STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, scope_kind_t kind);
127
STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn);
Damien's avatar
Damien committed
128

129
STATIC uint comp_next_label(compiler_t *comp) {
130
131
132
    return comp->next_label++;
}

133
134
135
136
137
138
139
140
141
142
143
144
STATIC void compile_increase_except_level(compiler_t *comp) {
    comp->cur_except_level += 1;
    if (comp->cur_except_level > comp->scope_cur->exc_stack_size) {
        comp->scope_cur->exc_stack_size = comp->cur_except_level;
    }
}

STATIC void compile_decrease_except_level(compiler_t *comp) {
    assert(comp->cur_except_level > 0);
    comp->cur_except_level -= 1;
}

145
STATIC scope_t *scope_new_and_link(compiler_t *comp, scope_kind_t kind, mp_parse_node_t pn, uint emit_options) {
146
    scope_t *scope = scope_new(kind, pn, comp->source_file, emit_options);
Damien's avatar
Damien committed
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    scope->parent = comp->scope_cur;
    scope->next = NULL;
    if (comp->scope_head == NULL) {
        comp->scope_head = scope;
    } else {
        scope_t *s = comp->scope_head;
        while (s->next != NULL) {
            s = s->next;
        }
        s->next = scope;
    }
    return scope;
}

161
162
163
typedef void (*apply_list_fun_t)(compiler_t *comp, mp_parse_node_t pn);

STATIC void apply_to_single_or_list(compiler_t *comp, mp_parse_node_t pn, pn_kind_t pn_list_kind, apply_list_fun_t f) {
164
    if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, pn_list_kind)) {
165
166
        mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
        int num_nodes = MP_PARSE_NODE_STRUCT_NUM_NODES(pns);
Damien's avatar
Damien committed
167
168
169
        for (int i = 0; i < num_nodes; i++) {
            f(comp, pns->nodes[i]);
        }
170
    } else if (!MP_PARSE_NODE_IS_NULL(pn)) {
Damien's avatar
Damien committed
171
172
173
174
        f(comp, pn);
    }
}

175
STATIC void compile_generic_all_nodes(compiler_t *comp, mp_parse_node_struct_t *pns) {
176
    int num_nodes = MP_PARSE_NODE_STRUCT_NUM_NODES(pns);
Damien's avatar
Damien committed
177
178
    for (int i = 0; i < num_nodes; i++) {
        compile_node(comp, pns->nodes[i]);
179
180
181
182
183
        if (comp->compile_error != MP_OBJ_NULL) {
            // add line info for the error in case it didn't have a line number
            compile_error_set_line(comp, pns->nodes[i]);
            return;
        }
Damien's avatar
Damien committed
184
185
186
    }
}

187
188
189
190
STATIC void compile_load_id(compiler_t *comp, qstr qst) {
    if (comp->pass == MP_PASS_SCOPE) {
        mp_emit_common_get_id_for_load(comp->scope_cur, qst);
    } else {
191
        #if NEED_METHOD_TABLE
192
        mp_emit_common_id_op(comp->emit, &comp->emit_method_table->load_id, comp->scope_cur, qst);
193
194
195
        #else
        mp_emit_common_id_op(comp->emit, &mp_emit_bc_method_table_load_id_ops, comp->scope_cur, qst);
        #endif
196
197
198
199
200
201
202
    }
}

STATIC void compile_store_id(compiler_t *comp, qstr qst) {
    if (comp->pass == MP_PASS_SCOPE) {
        mp_emit_common_get_id_for_modification(comp->scope_cur, qst);
    } else {
203
        #if NEED_METHOD_TABLE
204
        mp_emit_common_id_op(comp->emit, &comp->emit_method_table->store_id, comp->scope_cur, qst);
205
206
207
        #else
        mp_emit_common_id_op(comp->emit, &mp_emit_bc_method_table_store_id_ops, comp->scope_cur, qst);
        #endif
208
209
210
211
212
213
214
    }
}

STATIC void compile_delete_id(compiler_t *comp, qstr qst) {
    if (comp->pass == MP_PASS_SCOPE) {
        mp_emit_common_get_id_for_modification(comp->scope_cur, qst);
    } else {
215
        #if NEED_METHOD_TABLE
216
        mp_emit_common_id_op(comp->emit, &comp->emit_method_table->delete_id, comp->scope_cur, qst);
217
218
219
        #else
        mp_emit_common_id_op(comp->emit, &mp_emit_bc_method_table_delete_id_ops, comp->scope_cur, qst);
        #endif
220
221
222
    }
}

223
STATIC void c_tuple(compiler_t *comp, mp_parse_node_t pn, mp_parse_node_struct_t *pns_list) {
224
    int total = 0;
225
    if (!MP_PARSE_NODE_IS_NULL(pn)) {
226
227
228
229
        compile_node(comp, pn);
        total += 1;
    }
    if (pns_list != NULL) {
230
        int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns_list);
231
232
233
234
235
        for (int i = 0; i < n; i++) {
            compile_node(comp, pns_list->nodes[i]);
        }
        total += n;
    }
236
    EMIT_ARG(build_tuple, total);
237
}
Damien's avatar
Damien committed
238

239
STATIC void compile_generic_tuple(compiler_t *comp, mp_parse_node_struct_t *pns) {
Damien's avatar
Damien committed
240
    // a simple tuple expression
241
    c_tuple(comp, MP_PARSE_NODE_NULL, pns);
Damien's avatar
Damien committed
242
243
}

244
STATIC bool node_is_const_false(mp_parse_node_t pn) {
245
246
    return MP_PARSE_NODE_IS_TOKEN_KIND(pn, MP_TOKEN_KW_FALSE)
        || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_SMALL_INT(pn) == 0);
Damien's avatar
Damien committed
247
248
}

249
STATIC bool node_is_const_true(mp_parse_node_t pn) {
250
251
    return MP_PARSE_NODE_IS_TOKEN_KIND(pn, MP_TOKEN_KW_TRUE)
        || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_SMALL_INT(pn) != 0);
Damien's avatar
Damien committed
252
253
}

254
STATIC void c_if_cond(compiler_t *comp, mp_parse_node_t pn, bool jump_if, int label) {
255
256
    if (node_is_const_false(pn)) {
        if (jump_if == false) {
257
            EMIT_ARG(jump, label);
258
259
260
261
        }
        return;
    } else if (node_is_const_true(pn)) {
        if (jump_if == true) {
262
            EMIT_ARG(jump, label);
263
264
        }
        return;
265
266
267
268
    } else if (MP_PARSE_NODE_IS_STRUCT(pn)) {
        mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
        int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns);
        if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_or_test) {
269
            if (jump_if == false) {
270
            and_or_logic1:;
271
                uint label2 = comp_next_label(comp);
272
                for (int i = 0; i < n - 1; i++) {
273
                    c_if_cond(comp, pns->nodes[i], !jump_if, label2);
274
                }
275
                c_if_cond(comp, pns->nodes[n - 1], jump_if, label);
276
                EMIT_ARG(label_assign, label2);
277
            } else {
278
            and_or_logic2:
279
                for (int i = 0; i < n; i++) {
280
                    c_if_cond(comp, pns->nodes[i], jump_if, label);
281
282
283
                }
            }
            return;
284
        } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_and_test) {
285
            if (jump_if == false) {
286
                goto and_or_logic2;
287
            } else {
288
                goto and_or_logic1;
289
            }
290
        } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_not_test_2) {
291
292
            c_if_cond(comp, pns->nodes[0], !jump_if, label);
            return;
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
        } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_atom_paren) {
            // cond is something in parenthesis
            if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) {
                // empty tuple, acts as false for the condition
                if (jump_if == false) {
                    EMIT_ARG(jump, label);
                }
            } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)) {
                // non-empty tuple, acts as true for the condition
                if (jump_if == true) {
                    EMIT_ARG(jump, label);
                }
            } else {
                // parenthesis around 1 item, is just that item
                c_if_cond(comp, pns->nodes[0], jump_if, label);
            }
            return;
310
311
312
313
314
        }
    }

    // nothing special, fall back to default compiling for node and jump
    compile_node(comp, pn);
315
    EMIT_ARG(pop_jump_if, jump_if, label);
Damien's avatar
Damien committed
316
317
318
}

typedef enum { ASSIGN_STORE, ASSIGN_AUG_LOAD, ASSIGN_AUG_STORE } assign_kind_t;
319
STATIC void c_assign(compiler_t *comp, mp_parse_node_t pn, assign_kind_t kind);
Damien's avatar
Damien committed
320

321
STATIC void c_assign_power(compiler_t *comp, mp_parse_node_struct_t *pns, assign_kind_t assign_kind) {
Damien's avatar
Damien committed
322
323
324
325
    if (assign_kind != ASSIGN_AUG_STORE) {
        compile_node(comp, pns->nodes[0]);
    }

326
327
328
329
    if (MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])) {
        mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t*)pns->nodes[1];
        if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_power_trailers) {
            int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns1);
Damien's avatar
Damien committed
330
331
332
333
334
            if (assign_kind != ASSIGN_AUG_STORE) {
                for (int i = 0; i < n - 1; i++) {
                    compile_node(comp, pns1->nodes[i]);
                }
            }
335
336
            assert(MP_PARSE_NODE_IS_STRUCT(pns1->nodes[n - 1]));
            pns1 = (mp_parse_node_struct_t*)pns1->nodes[n - 1];
Damien's avatar
Damien committed
337
        }
338
        if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_bracket) {
Damien's avatar
Damien committed
339
340
341
342
343
344
345
            if (assign_kind == ASSIGN_AUG_STORE) {
                EMIT(rot_three);
                EMIT(store_subscr);
            } else {
                compile_node(comp, pns1->nodes[0]);
                if (assign_kind == ASSIGN_AUG_LOAD) {
                    EMIT(dup_top_two);
346
                    EMIT(load_subscr);
Damien's avatar
Damien committed
347
348
349
350
                } else {
                    EMIT(store_subscr);
                }
            }
351
352
        } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_period) {
            assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0]));
Damien's avatar
Damien committed
353
354
            if (assign_kind == ASSIGN_AUG_LOAD) {
                EMIT(dup_top);
355
                EMIT_ARG(load_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]));
Damien's avatar
Damien committed
356
357
358
359
            } else {
                if (assign_kind == ASSIGN_AUG_STORE) {
                    EMIT(rot_two);
                }
360
                EMIT_ARG(store_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]));
Damien's avatar
Damien committed
361
362
            }
        } else {
363
            goto cannot_assign;
Damien's avatar
Damien committed
364
365
        }
    } else {
366
        goto cannot_assign;
Damien's avatar
Damien committed
367
368
    }

369
    if (!MP_PARSE_NODE_IS_NULL(pns->nodes[2])) {
370
        goto cannot_assign;
Damien's avatar
Damien committed
371
    }
372
373
374
375
376

    return;

cannot_assign:
    compile_syntax_error(comp, (mp_parse_node_t)pns, "can't assign to expression");
Damien's avatar
Damien committed
377
378
}

379
// we need to allow for a caller passing in 1 initial node (node_head) followed by an array of nodes (nodes_tail)
380
STATIC void c_assign_tuple(compiler_t *comp, mp_parse_node_t node_head, uint num_tail, mp_parse_node_t *nodes_tail) {
381
382
383
    uint num_head = (node_head == MP_PARSE_NODE_NULL) ? 0 : 1;

    // look for star expression
384
    uint have_star_index = -1;
385
386
387
388
    if (num_head != 0 && MP_PARSE_NODE_IS_STRUCT_KIND(node_head, PN_star_expr)) {
        EMIT_ARG(unpack_ex, 0, num_tail);
        have_star_index = 0;
    }
389
    for (uint i = 0; i < num_tail; i++) {
390
        if (MP_PARSE_NODE_IS_STRUCT_KIND(nodes_tail[i], PN_star_expr)) {
391
            if (have_star_index == (uint)-1) {
392
393
                EMIT_ARG(unpack_ex, num_head + i, num_tail - i - 1);
                have_star_index = num_head + i;
Damien's avatar
Damien committed
394
            } else {
395
                compile_syntax_error(comp, nodes_tail[i], "multiple *x in assignment");
Damien's avatar
Damien committed
396
397
398
399
                return;
            }
        }
    }
400
    if (have_star_index == (uint)-1) {
401
        EMIT_ARG(unpack_sequence, num_head + num_tail);
Damien's avatar
Damien committed
402
    }
403
404
405
    if (num_head != 0) {
        if (0 == have_star_index) {
            c_assign(comp, ((mp_parse_node_struct_t*)node_head)->nodes[0], ASSIGN_STORE);
Damien's avatar
Damien committed
406
        } else {
407
408
409
            c_assign(comp, node_head, ASSIGN_STORE);
        }
    }
410
    for (uint i = 0; i < num_tail; i++) {
411
412
413
414
        if (num_head + i == have_star_index) {
            c_assign(comp, ((mp_parse_node_struct_t*)nodes_tail[i])->nodes[0], ASSIGN_STORE);
        } else {
            c_assign(comp, nodes_tail[i], ASSIGN_STORE);
Damien's avatar
Damien committed
415
416
417
418
419
        }
    }
}

// assigns top of stack to pn
420
STATIC void c_assign(compiler_t *comp, mp_parse_node_t pn, assign_kind_t assign_kind) {
Damien's avatar
Damien committed
421
    tail_recursion:
422
423
    assert(!MP_PARSE_NODE_IS_NULL(pn));
    if (MP_PARSE_NODE_IS_LEAF(pn)) {
424
        if (MP_PARSE_NODE_IS_ID(pn)) {
425
            qstr arg = MP_PARSE_NODE_LEAF_ARG(pn);
Damien's avatar
Damien committed
426
427
428
            switch (assign_kind) {
                case ASSIGN_STORE:
                case ASSIGN_AUG_STORE:
429
                    compile_store_id(comp, arg);
Damien's avatar
Damien committed
430
431
                    break;
                case ASSIGN_AUG_LOAD:
432
                default:
433
                    compile_load_id(comp, arg);
Damien's avatar
Damien committed
434
435
436
                    break;
            }
        } else {
437
            compile_syntax_error(comp, pn, "can't assign to literal");
Damien's avatar
Damien committed
438
439
440
            return;
        }
    } else {
441
        // pn must be a struct
442
443
        mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
        switch (MP_PARSE_NODE_STRUCT_KIND(pns)) {
Damien's avatar
Damien committed
444
445
446
447
448
449
450
451
452
453
454
            case PN_power:
                // lhs is an index or attribute
                c_assign_power(comp, pns, assign_kind);
                break;

            case PN_testlist_star_expr:
            case PN_exprlist:
                // lhs is a tuple
                if (assign_kind != ASSIGN_STORE) {
                    goto bad_aug;
                }
455
                c_assign_tuple(comp, MP_PARSE_NODE_NULL, MP_PARSE_NODE_STRUCT_NUM_NODES(pns), pns->nodes);
Damien's avatar
Damien committed
456
457
458
459
                break;

            case PN_atom_paren:
                // lhs is something in parenthesis
460
                if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) {
Damien's avatar
Damien committed
461
                    // empty tuple
462
                    goto cannot_assign;
463
                } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)) {
464
465
466
                    if (assign_kind != ASSIGN_STORE) {
                        goto bad_aug;
                    }
467
                    pns = (mp_parse_node_struct_t*)pns->nodes[0];
Damien's avatar
Damien committed
468
469
470
471
472
473
474
475
476
477
478
479
480
                    goto testlist_comp;
                } else {
                    // parenthesis around 1 item, is just that item
                    pn = pns->nodes[0];
                    goto tail_recursion;
                }
                break;

            case PN_atom_bracket:
                // lhs is something in brackets
                if (assign_kind != ASSIGN_STORE) {
                    goto bad_aug;
                }
481
                if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) {
Damien's avatar
Damien committed
482
                    // empty list, assignment allowed
483
                    c_assign_tuple(comp, MP_PARSE_NODE_NULL, 0, NULL);
484
485
                } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)) {
                    pns = (mp_parse_node_struct_t*)pns->nodes[0];
Damien's avatar
Damien committed
486
487
488
                    goto testlist_comp;
                } else {
                    // brackets around 1 item
489
                    c_assign_tuple(comp, pns->nodes[0], 0, NULL);
Damien's avatar
Damien committed
490
491
492
493
                }
                break;

            default:
494
                goto cannot_assign;
Damien's avatar
Damien committed
495
496
497
498
499
        }
        return;

        testlist_comp:
        // lhs is a sequence
500
501
502
        if (MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])) {
            mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t*)pns->nodes[1];
            if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_testlist_comp_3b) {
Damien's avatar
Damien committed
503
                // sequence of one item, with trailing comma
504
                assert(MP_PARSE_NODE_IS_NULL(pns2->nodes[0]));
505
                c_assign_tuple(comp, pns->nodes[0], 0, NULL);
506
            } else if (MP_PARSE_NODE_STRUCT_KIND(pns2) == PN_testlist_comp_3c) {
Damien's avatar
Damien committed
507
                // sequence of many items
508
509
                uint n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns2);
                c_assign_tuple(comp, pns->nodes[0], n, pns2->nodes);
510
            } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_comp_for) {
511
                // TODO can we ever get here? can it be compiled?
512
                goto cannot_assign;
Damien's avatar
Damien committed
513
514
515
516
517
518
519
            } else {
                // sequence with 2 items
                goto sequence_with_2_items;
            }
        } else {
            // sequence with 2 items
            sequence_with_2_items:
520
            c_assign_tuple(comp, MP_PARSE_NODE_NULL, 2, pns->nodes);
Damien's avatar
Damien committed
521
522
523
524
525
        }
        return;
    }
    return;

526
527
528
529
    cannot_assign:
    compile_syntax_error(comp, pn, "can't assign to expression");
    return;

Damien's avatar
Damien committed
530
    bad_aug:
531
    compile_syntax_error(comp, pn, "illegal expression for augmented assignment");
Damien's avatar
Damien committed
532
533
}

534
// stuff for lambda and comprehensions and generators:
535
536
537
//  if n_pos_defaults > 0 then there is a tuple on the stack with the positional defaults
//  if n_kw_defaults > 0 then there is a dictionary on the stack with the keyword defaults
//  if both exist, the tuple is above the dictionary (ie the first pop gets the tuple)
538
STATIC void close_over_variables_etc(compiler_t *comp, scope_t *this_scope, int n_pos_defaults, int n_kw_defaults) {
539
540
541
    assert(n_pos_defaults >= 0);
    assert(n_kw_defaults >= 0);

542
543
544
545
546
547
    // set flags
    if (n_kw_defaults > 0) {
        this_scope->scope_flags |= MP_SCOPE_FLAG_DEFKWARGS;
    }
    this_scope->num_def_pos_args = n_pos_defaults;

Damien's avatar
Damien committed
548
    // make closed over variables, if any
549
    // ensure they are closed over in the order defined in the outer scope (mainly to agree with CPython)
Damien's avatar
Damien committed
550
551
    int nfree = 0;
    if (comp->scope_cur->kind != SCOPE_MODULE) {
552
553
554
555
556
        for (int i = 0; i < comp->scope_cur->id_info_len; i++) {
            id_info_t *id = &comp->scope_cur->id_info[i];
            if (id->kind == ID_INFO_KIND_CELL || id->kind == ID_INFO_KIND_FREE) {
                for (int j = 0; j < this_scope->id_info_len; j++) {
                    id_info_t *id2 = &this_scope->id_info[j];
557
                    if (id2->kind == ID_INFO_KIND_FREE && id->qst == id2->qst) {
Damien George's avatar
Damien George committed
558
                        // in Micro Python we load closures using LOAD_FAST
559
                        EMIT_LOAD_FAST(id->qst, id->local_num);
560
561
562
                        nfree += 1;
                    }
                }
Damien's avatar
Damien committed
563
564
565
566
567
568
            }
        }
    }

    // make the function/closure
    if (nfree == 0) {
569
        EMIT_ARG(make_function, this_scope, n_pos_defaults, n_kw_defaults);
Damien's avatar
Damien committed
570
    } else {
571
        EMIT_ARG(make_closure, this_scope, nfree, n_pos_defaults, n_kw_defaults);
Damien's avatar
Damien committed
572
573
574
    }
}

575
576
577
STATIC void compile_funcdef_lambdef_param(compiler_t *comp, mp_parse_node_t pn) {
    if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_typedargslist_star)
        || MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_varargslist_star)) {
578
579
        comp->have_star = true;
        /* don't need to distinguish bare from named star
580
        mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
581
582
        if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) {
            // bare star
583
584
        } else {
            // named star
585
        }
586
        */
587

588
589
    } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_typedargslist_dbl_star)
        || MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_varargslist_dbl_star)) {
590
        // named double star
591
592
593
594
595
596
597
598
599
600
601
602
603
        // TODO do we need to do anything with this?

    } else {
        mp_parse_node_t pn_id;
        mp_parse_node_t pn_colon;
        mp_parse_node_t pn_equal;
        if (MP_PARSE_NODE_IS_ID(pn)) {
            // this parameter is just an id

            pn_id = pn;
            pn_colon = MP_PARSE_NODE_NULL;
            pn_equal = MP_PARSE_NODE_NULL;

604
        } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_typedargslist_name)) {
605
606
607
608
609
610
            // this parameter has a colon and/or equal specifier

            mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
            pn_id = pns->nodes[0];
            pn_colon = pns->nodes[1];
            pn_equal = pns->nodes[2];
611
612
613
614
615
616
617
618

        } else {
            assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_varargslist_name)); // should be
            // this parameter has an equal specifier

            mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
            pn_id = pns->nodes[0];
            pn_equal = pns->nodes[1];
619
620
621
622
623
624
        }

        if (MP_PARSE_NODE_IS_NULL(pn_equal)) {
            // this parameter does not have a default value

            // check for non-default parameters given after default parameters (allowed by parser, but not syntactically valid)
625
            if (!comp->have_star && comp->num_default_params != 0) {
626
                compile_syntax_error(comp, pn, "non-default argument follows default argument");
627
628
629
630
                return;
            }

        } else {
Damien's avatar
Damien committed
631
632
            // this parameter has a default value
            // in CPython, None (and True, False?) as default parameters are loaded with LOAD_NAME; don't understandy why
633

634
            if (comp->have_star) {
635
636
637
                comp->num_dict_params += 1;
                // in Micro Python we put the default dict parameters into a dictionary using the bytecode
                if (comp->num_dict_params == 1) {
638
639
640
641
                    // in Micro Python we put the default positional parameters into a tuple using the bytecode
                    // we need to do this here before we start building the map for the default keywords
                    if (comp->num_default_params > 0) {
                        EMIT_ARG(build_tuple, comp->num_default_params);
642
643
                    } else {
                        EMIT(load_null); // sentinel indicating empty default positional args
644
                    }
645
646
647
                    // first default dict param, so make the map
                    EMIT_ARG(build_map, 0);
                }
648
649

                // compile value then key, then store it to the dict
650
                compile_node(comp, pn_equal);
651
                EMIT_ARG(load_const_str, MP_PARSE_NODE_LEAF_ARG(pn_id));
652
                EMIT(store_map);
Damien's avatar
Damien committed
653
            } else {
654
655
                comp->num_default_params += 1;
                compile_node(comp, pn_equal);
Damien's avatar
Damien committed
656
657
            }
        }
658
659
660

        // TODO pn_colon not implemented
        (void)pn_colon;
Damien's avatar
Damien committed
661
662
663
    }
}

664
STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, mp_parse_node_t pn_params, pn_kind_t pn_list_kind) {
Damien's avatar
Damien committed
665
    // compile default parameters
666
    comp->have_star = false;
667
668
    comp->num_dict_params = 0;
    comp->num_default_params = 0;
669
    apply_to_single_or_list(comp, pn_params, pn_list_kind, compile_funcdef_lambdef_param);
670

671
    if (comp->compile_error != MP_OBJ_NULL) {
672
        return;
673
674
    }

675
    // in Micro Python we put the default positional parameters into a tuple using the bytecode
676
677
    // the default keywords args may have already made the tuple; if not, do it now
    if (comp->num_default_params > 0 && comp->num_dict_params == 0) {
678
        EMIT_ARG(build_tuple, comp->num_default_params);
679
        EMIT(load_null); // sentinel indicating empty default keyword args
680
681
    }

682
683
684
685
686
687
688
689
690
691
692
693
694
695
    // make the function
    close_over_variables_etc(comp, scope, comp->num_default_params, comp->num_dict_params);
}

// leaves function object on stack
// returns function name
STATIC qstr compile_funcdef_helper(compiler_t *comp, mp_parse_node_struct_t *pns, uint emit_options) {
    if (comp->pass == MP_PASS_SCOPE) {
        // create a new scope for this function
        scope_t *s = scope_new_and_link(comp, SCOPE_FUNCTION, (mp_parse_node_t)pns, emit_options);
        // store the function scope so the compiling function can use it at each pass
        pns->nodes[4] = (mp_parse_node_t)s;
    }

Damien's avatar
Damien committed
696
697
698
    // get the scope for this function
    scope_t *fscope = (scope_t*)pns->nodes[4];

699
700
    // compile the function definition
    compile_funcdef_lambdef(comp, fscope, pns->nodes[1], PN_typedargslist);
Damien's avatar
Damien committed
701
702
703
704
705
706
707

    // return its name (the 'f' in "def f(...):")
    return fscope->simple_name;
}

// leaves class object on stack
// returns class name
708
STATIC qstr compile_classdef_helper(compiler_t *comp, mp_parse_node_struct_t *pns, uint emit_options) {
709
    if (comp->pass == MP_PASS_SCOPE) {
Damien's avatar
Damien committed
710
        // create a new scope for this class
711
        scope_t *s = scope_new_and_link(comp, SCOPE_CLASS, (mp_parse_node_t)pns, emit_options);
Damien's avatar
Damien committed
712
        // store the class scope so the compiling function can use it at each pass
713
        pns->nodes[3] = (mp_parse_node_t)s;
Damien's avatar
Damien committed
714
715
716
717
718
719
720
721
722
723
724
    }

    EMIT(load_build_class);

    // scope for this class
    scope_t *cscope = (scope_t*)pns->nodes[3];

    // compile the class
    close_over_variables_etc(comp, cscope, 0, 0);

    // get its name
725
    EMIT_ARG(load_const_str, cscope->simple_name);
Damien's avatar
Damien committed
726
727

    // nodes[1] has parent classes, if any
728
729
730
731
732
    // empty parenthesis (eg class C():) gets here as an empty PN_classdef_2 and needs special handling
    mp_parse_node_t parents = pns->nodes[1];
    if (MP_PARSE_NODE_IS_STRUCT_KIND(parents, PN_classdef_2)) {
        parents = MP_PARSE_NODE_NULL;
    }
733
    comp->func_arg_is_super = false;
734
    compile_trailer_paren_helper(comp, parents, false, 2);
Damien's avatar
Damien committed
735
736
737
738
739

    // return its name (the 'C' in class C(...):")
    return cscope->simple_name;
}

740
// returns true if it was a built-in decorator (even if the built-in had an error)
741
STATIC bool compile_built_in_decorator(compiler_t *comp, int name_len, mp_parse_node_t *name_nodes, uint *emit_options) {
742
    if (MP_PARSE_NODE_LEAF_ARG(name_nodes[0]) != MP_QSTR_micropython) {
743
744
745
746
        return false;
    }

    if (name_len != 2) {
747
        compile_syntax_error(comp, name_nodes[0], "invalid micropython decorator");
748
749
750
        return true;
    }

751
    qstr attr = MP_PARSE_NODE_LEAF_ARG(name_nodes[1]);
752
    if (attr == MP_QSTR_bytecode) {
753
        *emit_options = MP_EMIT_OPT_BYTECODE;
754
#if MICROPY_EMIT_NATIVE
755
    } else if (attr == MP_QSTR_native) {
756
        *emit_options = MP_EMIT_OPT_NATIVE_PYTHON;
757
    } else if (attr == MP_QSTR_viper) {
758
        *emit_options = MP_EMIT_OPT_VIPER;
759
#endif
760
#if MICROPY_EMIT_INLINE_THUMB
761
    } else if (attr == MP_QSTR_asm_thumb) {
762
        *emit_options = MP_EMIT_OPT_ASM_THUMB;
763
#endif
764
    } else {
765
        compile_syntax_error(comp, name_nodes[1], "invalid micropython decorator");
766
767
768
769
770
    }

    return true;
}

771
STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) {
Damien's avatar
Damien committed
772
    // get the list of decorators
773
    mp_parse_node_t *nodes;
774
    int n = mp_parse_node_extract_list(&pns->nodes[0], PN_decorators, &nodes);
Damien's avatar
Damien committed
775

776
777
778
779
780
    // inherit emit options for this function/class definition
    uint emit_options = comp->scope_cur->emit_options;

    // compile each decorator
    int num_built_in_decorators = 0;
Damien's avatar
Damien committed
781
    for (int i = 0; i < n; i++) {
782
783
        assert(MP_PARSE_NODE_IS_STRUCT_KIND(nodes[i], PN_decorator)); // should be
        mp_parse_node_struct_t *pns_decorator = (mp_parse_node_struct_t*)nodes[i];
784
785

        // nodes[0] contains the decorator function, which is a dotted name
786
        mp_parse_node_t *name_nodes;
787
        int name_len = mp_parse_node_extract_list(&pns_decorator->nodes[0], PN_dotted_name, &name_nodes);
788
789
790
791
792
793
794
795
796
797
798

        // check for built-in decorators
        if (compile_built_in_decorator(comp, name_len, name_nodes, &emit_options)) {
            // this was a built-in
            num_built_in_decorators += 1;

        } else {
            // not a built-in, compile normally

            // compile the decorator function
            compile_node(comp, name_nodes[0]);
799
800
801
            for (int j = 1; j < name_len; j++) {
                assert(MP_PARSE_NODE_IS_ID(name_nodes[j])); // should be
                EMIT_ARG(load_attr, MP_PARSE_NODE_LEAF_ARG(name_nodes[j]));
802
803
804
            }

            // nodes[1] contains arguments to the decorator function, if any
805
            if (!MP_PARSE_NODE_IS_NULL(pns_decorator->nodes[1])) {
806
                // call the decorator function with the arguments in nodes[1]
Damien George's avatar
Damien George committed
807
                comp->func_arg_is_super = false;
808
809
                compile_node(comp, pns_decorator->nodes[1]);
            }
Damien's avatar
Damien committed
810
811
812
813
        }
    }

    // compile the body (funcdef or classdef) and get its name
814
    mp_parse_node_struct_t *pns_body = (mp_parse_node_struct_t*)pns->nodes[1];
Damien's avatar
Damien committed
815
    qstr body_name = 0;
816
    if (MP_PARSE_NODE_STRUCT_KIND(pns_body) == PN_funcdef) {
817
        body_name = compile_funcdef_helper(comp, pns_body, emit_options);
Damien's avatar
Damien committed
818
    } else {
819
820
        assert(MP_PARSE_NODE_STRUCT_KIND(pns_body) == PN_classdef); // should be
        body_name = compile_classdef_helper(comp, pns_body, emit_options);
Damien's avatar
Damien committed
821
822
823
    }

    // call each decorator
824
    for (int i = 0; i < n - num_built_in_decorators; i++) {
825
        EMIT_ARG(call_function, 1, 0, 0);
Damien's avatar
Damien committed
826
827
828
    }

    // store func/class object into name
829
    compile_store_id(comp, body_name);
Damien's avatar
Damien committed
830
831
}

832
STATIC void compile_funcdef(compiler_t *comp, mp_parse_node_struct_t *pns) {
833
    qstr fname = compile_funcdef_helper(comp, pns, comp->scope_cur->emit_options);
Damien's avatar
Damien committed
834
    // store function object into function name
835
    compile_store_id(comp, fname);
Damien's avatar
Damien committed
836
837
}

838
STATIC void c_del_stmt(compiler_t *comp, mp_parse_node_t pn) {
839
    if (MP_PARSE_NODE_IS_ID(pn)) {
840
        compile_delete_id(comp, MP_PARSE_NODE_LEAF_ARG(pn));
841
842
    } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_power)) {
        mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
Damien's avatar
Damien committed
843
844
845

        compile_node(comp, pns->nodes[0]); // base of the power node

846
847
848
849
        if (MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])) {
            mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t*)pns->nodes[1];
            if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_power_trailers) {
                int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns1);
Damien's avatar
Damien committed
850
851
852
                for (int i = 0; i < n - 1; i++) {
                    compile_node(comp, pns1->nodes[i]);
                }
853
854
                assert(MP_PARSE_NODE_IS_STRUCT(pns1->nodes[n - 1]));
                pns1 = (mp_parse_node_struct_t*)pns1->nodes[n - 1];
Damien's avatar
Damien committed
855
            }
856
            if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_bracket) {
Damien's avatar
Damien committed
857
858
                compile_node(comp, pns1->nodes[0]);
                EMIT(delete_subscr);
859
860
            } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_period) {
                assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0]));
861
                EMIT_ARG(delete_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]));
Damien's avatar
Damien committed
862
            } else {
863
                goto cannot_delete;
Damien's avatar
Damien committed
864
865
            }
        } else {
866
            goto cannot_delete;
Damien's avatar
Damien committed
867
868
        }

869
        if (!MP_PARSE_NODE_IS_NULL(pns->nodes[2])) {
870
            goto cannot_delete;
Damien's avatar
Damien committed
871
        }
872
873
874
875
    } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_atom_paren)) {
        pn = ((mp_parse_node_struct_t*)pn)->nodes[0];
        if (MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_testlist_comp)) {
            mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn;
Damien's avatar
Damien committed
876
877
            // TODO perhaps factorise testlist_comp code with other uses of PN_testlist_comp

878
879
880
            if (MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])) {
                mp_parse_node_struct_t *pns1 = (mp_parse_node_struct_t*)pns->nodes[1];
                if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_testlist_comp_3b) {
Damien's avatar
Damien committed
881
                    // sequence of one item, with trailing comma
882
                    assert(MP_PARSE_NODE_IS_NULL(pns1->nodes[0]));
Damien's avatar
Damien committed
883
                    c_del_stmt(comp, pns->nodes[0]);
884
                } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_testlist_comp_3c) {
Damien's avatar
Damien committed
885
                    // sequence of many items
886
                    int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns1);
Damien's avatar
Damien committed
887
888
889
890
                    c_del_stmt(comp, pns->nodes[0]);
                    for (int i = 0; i < n; i++) {
                        c_del_stmt(comp, pns1->nodes[i]);
                    }
891
                } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_comp_for) {
892
                    // TODO not implemented; can't del comprehension? can we get here?
893
                    goto cannot_delete;
Damien's avatar
Damien committed
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
                } else {
                    // sequence with 2 items
                    goto sequence_with_2_items;
                }
            } else {
                // sequence with 2 items
                sequence_with_2_items:
                c_del_stmt(comp, pns->nodes[0]);
                c_del_stmt(comp, pns->nodes[1]);
            }
        } else {
            // tuple with 1 element
            c_del_stmt(comp, pn);
        }
    } else {
909
910
        // TODO is there anything else to implement?
        goto cannot_delete;
Damien's avatar
Damien committed
911
    }
912
913
914
915
916

    return;

cannot_delete:
    compile_syntax_error(comp, (mp_parse_node_t)pn, "can't delete expression");
Damien's avatar
Damien committed
917
918
}

919
STATIC void compile_del_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
Damien's avatar
Damien committed
920
921
922
    apply_to_single_or_list(comp, pns->nodes[0], PN_exprlist, c_del_stmt);
}

923
STATIC void compile_break_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
Damien's avatar
Damien committed
924
    if (comp->break_label == 0) {
925
        compile_syntax_error(comp, (mp_parse_node_t)pns, "'break' outside loop");
Damien's avatar
Damien committed
926
    }
927
    assert(comp->cur_except_level >= comp->break_continue_except_level);
928
    EMIT_ARG(break_loop, comp->break_label, comp->cur_except_level - comp->break_continue_except_level);
Damien's avatar
Damien committed
929
930
}

931
STATIC void compile_continue_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
Damien's avatar
Damien committed
932
    if (comp->continue_label == 0) {
933
        compile_syntax_error(comp, (mp_parse_node_t)pns, "'continue' outside loop");
Damien's avatar
Damien committed
934
    }
935
    assert(comp->cur_except_level >= comp->break_continue_except_level);
936
    EMIT_ARG(continue_loop, comp->continue_label, comp->cur_except_level - comp->break_continue_except_level);
Damien's avatar
Damien committed
937
938
}

939
STATIC void compile_return_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
Damien's avatar
Damien committed
940
    if (comp->scope_cur->kind != SCOPE_FUNCTION) {
941
        compile_syntax_error(comp, (mp_parse_node_t)pns, "'return' outside function");
Damien's avatar
Damien committed
942
943
        return;
    }
944
    if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) {
Damien's avatar
Damien committed
945
        // no argument to 'return', so return None
946
        EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
947
    } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_test_if_expr)) {
Damien's avatar
Damien committed
948
        // special case when returning an if-expression; to match CPython optimisation
949
950
        mp_parse_node_struct_t *pns_test_if_expr = (mp_parse_node_struct_t*)pns->nodes[0];
        mp_parse_node_struct_t *pns_test_if_else = (mp_parse_node_struct_t*)pns_test_if_expr->nodes[1];
Damien's avatar
Damien committed
951

952
        uint l_fail = comp_next_label(comp