gc.c 28.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/*
 * This file is part of the Micro Python project, http://micropython.org/
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2013, 2014 Damien P. George
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

27
#include <assert.h>
Damien's avatar
Damien committed
28
29
30
#include <stdio.h>
#include <string.h>

31
#include "py/mpstate.h"
32
33
34
#include "py/gc.h"
#include "py/obj.h"
#include "py/runtime.h"
mux's avatar
mux committed
35

36
37
#if MICROPY_ENABLE_GC

stijn's avatar
stijn committed
38
#if 0 // print debugging info
39
#define DEBUG_PRINT (1)
40
#define DEBUG_printf DEBUG_printf
41
#else // don't print debugging info
42
#define DEBUG_PRINT (0)
43
#define DEBUG_printf(...) (void)0
44
45
#endif

46
47
48
// make this 1 to dump the heap each time it changes
#define EXTENSIVE_HEAP_PROFILING (0)

Damien's avatar
Damien committed
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#define WORDS_PER_BLOCK (4)
#define BYTES_PER_BLOCK (WORDS_PER_BLOCK * BYTES_PER_WORD)

// ATB = allocation table byte
// 0b00 = FREE -- free block
// 0b01 = HEAD -- head of a chain of blocks
// 0b10 = TAIL -- in the tail of a chain of blocks
// 0b11 = MARK -- marked head block

#define AT_FREE (0)
#define AT_HEAD (1)
#define AT_TAIL (2)
#define AT_MARK (3)

#define BLOCKS_PER_ATB (4)
#define ATB_MASK_0 (0x03)
#define ATB_MASK_1 (0x0c)
#define ATB_MASK_2 (0x30)
#define ATB_MASK_3 (0xc0)

#define ATB_0_IS_FREE(a) (((a) & ATB_MASK_0) == 0)
#define ATB_1_IS_FREE(a) (((a) & ATB_MASK_1) == 0)
#define ATB_2_IS_FREE(a) (((a) & ATB_MASK_2) == 0)
#define ATB_3_IS_FREE(a) (((a) & ATB_MASK_3) == 0)

#define BLOCK_SHIFT(block) (2 * ((block) & (BLOCKS_PER_ATB - 1)))
75
76
77
78
79
80
81
82
83
#define ATB_GET_KIND(block) ((MP_STATE_MEM(gc_alloc_table_start)[(block) / BLOCKS_PER_ATB] >> BLOCK_SHIFT(block)) & 3)
#define ATB_ANY_TO_FREE(block) do { MP_STATE_MEM(gc_alloc_table_start)[(block) / BLOCKS_PER_ATB] &= (~(AT_MARK << BLOCK_SHIFT(block))); } while (0)
#define ATB_FREE_TO_HEAD(block) do { MP_STATE_MEM(gc_alloc_table_start)[(block) / BLOCKS_PER_ATB] |= (AT_HEAD << BLOCK_SHIFT(block)); } while (0)
#define ATB_FREE_TO_TAIL(block) do { MP_STATE_MEM(gc_alloc_table_start)[(block) / BLOCKS_PER_ATB] |= (AT_TAIL << BLOCK_SHIFT(block)); } while (0)
#define ATB_HEAD_TO_MARK(block) do { MP_STATE_MEM(gc_alloc_table_start)[(block) / BLOCKS_PER_ATB] |= (AT_MARK << BLOCK_SHIFT(block)); } while (0)
#define ATB_MARK_TO_HEAD(block) do { MP_STATE_MEM(gc_alloc_table_start)[(block) / BLOCKS_PER_ATB] &= (~(AT_TAIL << BLOCK_SHIFT(block))); } while (0)

#define BLOCK_FROM_PTR(ptr) (((ptr) - (mp_uint_t)MP_STATE_MEM(gc_pool_start)) / BYTES_PER_BLOCK)
#define PTR_FROM_BLOCK(block) (((block) * BYTES_PER_BLOCK + (mp_uint_t)MP_STATE_MEM(gc_pool_start)))
Damien's avatar
Damien committed
84
85
#define ATB_FROM_BLOCK(bl) ((bl) / BLOCKS_PER_ATB)

86
87
88
89
90
91
#if MICROPY_ENABLE_FINALISER
// FTB = finaliser table byte
// if set, then the corresponding block may have a finaliser

#define BLOCKS_PER_FTB (8)

92
93
94
#define FTB_GET(block) ((MP_STATE_MEM(gc_finaliser_table_start)[(block) / BLOCKS_PER_FTB] >> ((block) & 7)) & 1)
#define FTB_SET(block) do { MP_STATE_MEM(gc_finaliser_table_start)[(block) / BLOCKS_PER_FTB] |= (1 << ((block) & 7)); } while (0)
#define FTB_CLEAR(block) do { MP_STATE_MEM(gc_finaliser_table_start)[(block) / BLOCKS_PER_FTB] &= (~(1 << ((block) & 7))); } while (0)
95
96
#endif

97
98
99
// TODO waste less memory; currently requires that all entries in alloc_table have a corresponding block in pool
void gc_init(void *start, void *end) {
    // align end pointer on block boundary
100
    end = (void*)((mp_uint_t)end & (~(BYTES_PER_BLOCK - 1)));
101
    DEBUG_printf("Initializing GC heap: %p..%p = " UINT_FMT " bytes\n", start, end, (byte*)end - (byte*)start);
102
103
104
105
106
107

    // calculate parameters for GC (T=total, A=alloc table, F=finaliser table, P=pool; all in bytes):
    // T = A + F + P
    //     F = A * BLOCKS_PER_ATB / BLOCKS_PER_FTB
    //     P = A * BLOCKS_PER_ATB * BYTES_PER_BLOCK
    // => T = A * (1 + BLOCKS_PER_ATB / BLOCKS_PER_FTB + BLOCKS_PER_ATB * BYTES_PER_BLOCK)
108
    mp_uint_t total_byte_len = (byte*)end - (byte*)start;
109
#if MICROPY_ENABLE_FINALISER
110
    MP_STATE_MEM(gc_alloc_table_byte_len) = total_byte_len * BITS_PER_BYTE / (BITS_PER_BYTE + BITS_PER_BYTE * BLOCKS_PER_ATB / BLOCKS_PER_FTB + BITS_PER_BYTE * BLOCKS_PER_ATB * BYTES_PER_BLOCK);
111
#else
112
    MP_STATE_MEM(gc_alloc_table_byte_len) = total_byte_len / (1 + BITS_PER_BYTE / 2 * BYTES_PER_BLOCK);
113
114
#endif

115
    MP_STATE_MEM(gc_alloc_table_start) = (byte*)start;
mux's avatar
mux committed
116

117
#if MICROPY_ENABLE_FINALISER
118
119
    mp_uint_t gc_finaliser_table_byte_len = (MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB + BLOCKS_PER_FTB - 1) / BLOCKS_PER_FTB;
    MP_STATE_MEM(gc_finaliser_table_start) = MP_STATE_MEM(gc_alloc_table_start) + MP_STATE_MEM(gc_alloc_table_byte_len);
120
#endif
mux's avatar
mux committed
121

122
123
124
    mp_uint_t gc_pool_block_len = MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB;
    MP_STATE_MEM(gc_pool_start) = (mp_uint_t*)((byte*)end - gc_pool_block_len * BYTES_PER_BLOCK);
    MP_STATE_MEM(gc_pool_end) = (mp_uint_t*)end;
125

126
#if MICROPY_ENABLE_FINALISER
127
    assert((byte*)MP_STATE_MEM(gc_pool_start) >= MP_STATE_MEM(gc_finaliser_table_start) + gc_finaliser_table_byte_len);
128
129
#endif

130
    // clear ATBs
131
    memset(MP_STATE_MEM(gc_alloc_table_start), 0, MP_STATE_MEM(gc_alloc_table_byte_len));
132

133
134
#if MICROPY_ENABLE_FINALISER
    // clear FTBs
135
    memset(MP_STATE_MEM(gc_finaliser_table_start), 0, gc_finaliser_table_byte_len);
136
#endif
mux's avatar
mux committed
137

Damien George's avatar
Damien George committed
138
    // set last free ATB index to start of heap
139
    MP_STATE_MEM(gc_last_free_atb_index) = 0;
Damien George's avatar
Damien George committed
140

141
    // unlock the GC
142
    MP_STATE_MEM(gc_lock_depth) = 0;
143

144
    // allow auto collection
145
    MP_STATE_MEM(gc_auto_collect_enabled) = 1;
146

147
    DEBUG_printf("GC layout:\n");
148
    DEBUG_printf("  alloc table at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(gc_alloc_table_start), MP_STATE_MEM(gc_alloc_table_byte_len), MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB);
149
#if MICROPY_ENABLE_FINALISER
150
    DEBUG_printf("  finaliser table at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(gc_finaliser_table_start), gc_finaliser_table_byte_len, gc_finaliser_table_byte_len * BLOCKS_PER_FTB);
151
#endif
152
    DEBUG_printf("  pool at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(gc_pool_start), gc_pool_block_len * BYTES_PER_BLOCK, gc_pool_block_len);
153
154
}

155
void gc_lock(void) {
156
    MP_STATE_MEM(gc_lock_depth)++;
157
158
159
}

void gc_unlock(void) {
160
    MP_STATE_MEM(gc_lock_depth)--;
161
162
}

163
bool gc_is_locked(void) {
164
    return MP_STATE_MEM(gc_lock_depth) != 0;
165
166
}

167
168
#define VERIFY_PTR(ptr) ( \
        (ptr & (BYTES_PER_BLOCK - 1)) == 0          /* must be aligned on a block */ \
169
170
        && ptr >= (mp_uint_t)MP_STATE_MEM(gc_pool_start)     /* must be above start of pool */ \
        && ptr < (mp_uint_t)MP_STATE_MEM(gc_pool_end)        /* must be below end of pool */ \
171
172
    )

Damien's avatar
Damien committed
173
174
#define VERIFY_MARK_AND_PUSH(ptr) \
    do { \
175
        if (VERIFY_PTR(ptr)) { \
176
            mp_uint_t _block = BLOCK_FROM_PTR(ptr); \
Damien's avatar
Damien committed
177
178
179
            if (ATB_GET_KIND(_block) == AT_HEAD) { \
                /* an unmarked head, mark it, and push it on gc stack */ \
                ATB_HEAD_TO_MARK(_block); \
180
181
                if (MP_STATE_MEM(gc_sp) < &MP_STATE_MEM(gc_stack)[MICROPY_ALLOC_GC_STACK_SIZE]) { \
                    *MP_STATE_MEM(gc_sp)++ = _block; \
Damien's avatar
Damien committed
182
                } else { \
183
                    MP_STATE_MEM(gc_stack_overflow) = 1; \
Damien's avatar
Damien committed
184
185
186
187
188
                } \
            } \
        } \
    } while (0)

189
STATIC void gc_drain_stack(void) {
190
    while (MP_STATE_MEM(gc_sp) > MP_STATE_MEM(gc_stack)) {
Damien's avatar
Damien committed
191
        // pop the next block off the stack
192
        mp_uint_t block = *--MP_STATE_MEM(gc_sp);
Damien's avatar
Damien committed
193

194
        // work out number of consecutive blocks in the chain starting with this one
195
        mp_uint_t n_blocks = 0;
Damien's avatar
Damien committed
196
197
198
199
200
        do {
            n_blocks += 1;
        } while (ATB_GET_KIND(block + n_blocks) == AT_TAIL);

        // check this block's children
201
202
203
        mp_uint_t *scan = (mp_uint_t*)PTR_FROM_BLOCK(block);
        for (mp_uint_t i = n_blocks * WORDS_PER_BLOCK; i > 0; i--, scan++) {
            mp_uint_t ptr2 = *scan;
Damien's avatar
Damien committed
204
205
206
207
208
            VERIFY_MARK_AND_PUSH(ptr2);
        }
    }
}

209
STATIC void gc_deal_with_stack_overflow(void) {
210
211
212
    while (MP_STATE_MEM(gc_stack_overflow)) {
        MP_STATE_MEM(gc_stack_overflow) = 0;
        MP_STATE_MEM(gc_sp) = MP_STATE_MEM(gc_stack);
Damien's avatar
Damien committed
213
214

        // scan entire memory looking for blocks which have been marked but not their children
215
        for (mp_uint_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) {
Damien's avatar
Damien committed
216
217
            // trace (again) if mark bit set
            if (ATB_GET_KIND(block) == AT_MARK) {
218
                *MP_STATE_MEM(gc_sp)++ = block;
Damien's avatar
Damien committed
219
220
221
222
223
224
                gc_drain_stack();
            }
        }
    }
}

225
226
227
228
#if MICROPY_PY_GC_COLLECT_RETVAL
uint gc_collected;
#endif

229
STATIC void gc_sweep(void) {
230
231
232
    #if MICROPY_PY_GC_COLLECT_RETVAL
    gc_collected = 0;
    #endif
Damien's avatar
Damien committed
233
234
    // free unmarked heads and their tails
    int free_tail = 0;
235
    for (mp_uint_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) {
Damien's avatar
Damien committed
236
237
        switch (ATB_GET_KIND(block)) {
            case AT_HEAD:
238
239
240
241
242
243
244
245
246
247
248
#if MICROPY_ENABLE_FINALISER
                if (FTB_GET(block)) {
                    mp_obj_t obj = (mp_obj_t)PTR_FROM_BLOCK(block);
                    if (((mp_obj_base_t*)obj)->type != MP_OBJ_NULL) {
                        // if the object has a type then see if it has a __del__ method
                        mp_obj_t dest[2];
                        mp_load_method_maybe(obj, MP_QSTR___del__, dest);
                        if (dest[0] != MP_OBJ_NULL) {
                            // load_method returned a method
                            mp_call_method_n_kw(0, 0, dest);
                        }
mux's avatar
mux committed
249
                    }
250
251
                    // clear finaliser flag
                    FTB_CLEAR(block);
mux's avatar
mux committed
252
                }
253
#endif
Damien's avatar
Damien committed
254
                free_tail = 1;
255
256
257
                #if MICROPY_PY_GC_COLLECT_RETVAL
                gc_collected++;
                #endif
Damien's avatar
Damien committed
258
259
260
261
                // fall through to free the head

            case AT_TAIL:
                if (free_tail) {
stijn's avatar
stijn committed
262
                    DEBUG_printf("gc_sweep(%p)\n",PTR_FROM_BLOCK(block));
Damien's avatar
Damien committed
263
264
265
266
267
268
269
270
271
272
273
274
                    ATB_ANY_TO_FREE(block);
                }
                break;

            case AT_MARK:
                ATB_MARK_TO_HEAD(block);
                free_tail = 0;
                break;
        }
    }
}

275
void gc_collect_start(void) {
276
    gc_lock();
277
278
279
280
281
282
283
    MP_STATE_MEM(gc_stack_overflow) = 0;
    MP_STATE_MEM(gc_sp) = MP_STATE_MEM(gc_stack);
    // Trace root pointers.  This relies on the root pointers being organised
    // correctly in the mp_state_ctx structure.  We scan nlr_top, dict_locals,
    // dict_globals, then the root pointer section of mp_state_vm.
    void **ptrs = (void**)(void*)&mp_state_ctx;
    gc_collect_root(ptrs, offsetof(mp_state_ctx_t, vm.stack_top) / sizeof(mp_uint_t));
Damien's avatar
Damien committed
284
285
}

286
287
288
void gc_collect_root(void **ptrs, mp_uint_t len) {
    for (mp_uint_t i = 0; i < len; i++) {
        mp_uint_t ptr = (mp_uint_t)ptrs[i];
Damien's avatar
Damien committed
289
290
291
292
293
        VERIFY_MARK_AND_PUSH(ptr);
        gc_drain_stack();
    }
}

294
void gc_collect_end(void) {
Damien's avatar
Damien committed
295
296
    gc_deal_with_stack_overflow();
    gc_sweep();
297
    MP_STATE_MEM(gc_last_free_atb_index) = 0;
298
    gc_unlock();
299
}
Damien's avatar
Damien committed
300

301
void gc_info(gc_info_t *info) {
302
    info->total = (MP_STATE_MEM(gc_pool_end) - MP_STATE_MEM(gc_pool_start)) * sizeof(mp_uint_t);
303
304
305
306
307
    info->used = 0;
    info->free = 0;
    info->num_1block = 0;
    info->num_2block = 0;
    info->max_block = 0;
308
    for (mp_uint_t block = 0, len = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) {
309
        mp_uint_t kind = ATB_GET_KIND(block);
310
311
312
313
314
315
316
317
318
319
320
        if (kind == AT_FREE || kind == AT_HEAD) {
            if (len == 1) {
                info->num_1block += 1;
            } else if (len == 2) {
                info->num_2block += 1;
            }
            if (len > info->max_block) {
                info->max_block = len;
            }
        }
        switch (kind) {
Damien's avatar
Damien committed
321
            case AT_FREE:
322
323
                info->free += 1;
                len = 0;
Damien's avatar
Damien committed
324
325
326
                break;

            case AT_HEAD:
327
328
329
330
                info->used += 1;
                len = 1;
                break;

Damien's avatar
Damien committed
331
            case AT_TAIL:
332
333
                info->used += 1;
                len += 1;
Damien's avatar
Damien committed
334
335
336
                break;

            case AT_MARK:
337
                // shouldn't happen
Damien's avatar
Damien committed
338
339
340
341
                break;
        }
    }

342
343
    info->used *= BYTES_PER_BLOCK;
    info->free *= BYTES_PER_BLOCK;
Damien's avatar
Damien committed
344
345
}

346
347
void *gc_alloc(mp_uint_t n_bytes, bool has_finaliser) {
    mp_uint_t n_blocks = ((n_bytes + BYTES_PER_BLOCK - 1) & (~(BYTES_PER_BLOCK - 1))) / BYTES_PER_BLOCK;
348
    DEBUG_printf("gc_alloc(" UINT_FMT " bytes -> " UINT_FMT " blocks)\n", n_bytes, n_blocks);
Damien's avatar
Damien committed
349

350
    // check if GC is locked
351
    if (MP_STATE_MEM(gc_lock_depth) > 0) {
352
        return NULL;
353
354
    }

Damien's avatar
Damien committed
355
356
357
358
359
    // check for 0 allocation
    if (n_blocks == 0) {
        return NULL;
    }

360
361
362
363
    mp_uint_t i;
    mp_uint_t end_block;
    mp_uint_t start_block;
    mp_uint_t n_free = 0;
364
    int collected = !MP_STATE_MEM(gc_auto_collect_enabled);
Damien's avatar
Damien committed
365
366
367
    for (;;) {

        // look for a run of n_blocks available blocks
368
369
        for (i = MP_STATE_MEM(gc_last_free_atb_index); i < MP_STATE_MEM(gc_alloc_table_byte_len); i++) {
            byte a = MP_STATE_MEM(gc_alloc_table_start)[i];
Damien George's avatar
Damien George committed
370
371
372
373
374
            if (ATB_0_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 0; goto found; } } else { n_free = 0; }
            if (ATB_1_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 1; goto found; } } else { n_free = 0; }
            if (ATB_2_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 2; goto found; } } else { n_free = 0; }
            if (ATB_3_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 3; goto found; } } else { n_free = 0; }
        }
Damien's avatar
Damien committed
375
376
377
378
379

        // nothing found!
        if (collected) {
            return NULL;
        }
380
        DEBUG_printf("gc_alloc(" UINT_FMT "): no free mem, triggering GC\n", n_bytes);
Damien's avatar
Damien committed
381
382
383
384
385
386
387
388
389
390
        gc_collect();
        collected = 1;
    }

    // found, ending at block i inclusive
found:
    // get starting and end blocks, both inclusive
    end_block = i;
    start_block = i - n_free + 1;

391
392
393
    // Set last free ATB index to block after last block we found, for start of
    // next scan.  To reduce fragmentation, we only do this if we were looking
    // for a single free block, which guarantees that there are no free blocks
394
395
    // before this one.  Also, whenever we free or shink a block we must check
    // if this index needs adjusting (see gc_realloc and gc_free).
396
    if (n_free == 1) {
397
        MP_STATE_MEM(gc_last_free_atb_index) = (i + 1) / BLOCKS_PER_ATB;
398
    }
Damien George's avatar
Damien George committed
399

Damien's avatar
Damien committed
400
401
402
403
404
    // mark first block as used head
    ATB_FREE_TO_HEAD(start_block);

    // mark rest of blocks as used tail
    // TODO for a run of many blocks can make this more efficient
405
    for (mp_uint_t bl = start_block + 1; bl <= end_block; bl++) {
Damien's avatar
Damien committed
406
407
408
        ATB_FREE_TO_TAIL(bl);
    }

409
    // get pointer to first block
410
    void *ret_ptr = (void*)(MP_STATE_MEM(gc_pool_start) + start_block * WORDS_PER_BLOCK);
411
    DEBUG_printf("gc_alloc(%p)\n", ret_ptr);
412

413
    // zero out the additional bytes of the newly allocated blocks
414
415
416
417
    // This is needed because the blocks may have previously held pointers
    // to the heap and will not be set to something else if the caller
    // doesn't actually use the entire block.  As such they will continue
    // to point to the heap and may prevent other blocks from being reclaimed.
418
    memset((byte*)ret_ptr + n_bytes, 0, (end_block - start_block + 1) * BYTES_PER_BLOCK - n_bytes);
419

420
421
#if MICROPY_ENABLE_FINALISER
    if (has_finaliser) {
422
423
        // clear type pointer in case it is never set
        ((mp_obj_base_t*)ret_ptr)->type = MP_OBJ_NULL;
424
425
        // set mp_obj flag only if it has a finaliser
        FTB_SET(start_block);
mux's avatar
mux committed
426
    }
427
#endif
mux's avatar
mux committed
428

429
430
431
432
    #if EXTENSIVE_HEAP_PROFILING
    gc_dump_alloc_table();
    #endif

433
    return ret_ptr;
Damien's avatar
Damien committed
434
435
}

436
/*
437
void *gc_alloc(mp_uint_t n_bytes) {
mux's avatar
mux committed
438
439
440
    return _gc_alloc(n_bytes, false);
}

441
void *gc_alloc_with_finaliser(mp_uint_t n_bytes) {
mux's avatar
mux committed
442
443
    return _gc_alloc(n_bytes, true);
}
444
*/
mux's avatar
mux committed
445

446
447
// force the freeing of a piece of memory
void gc_free(void *ptr_in) {
448
    if (MP_STATE_MEM(gc_lock_depth) > 0) {
449
450
        // TODO how to deal with this error?
        return;
451
452
    }

453
    mp_uint_t ptr = (mp_uint_t)ptr_in;
stijn's avatar
stijn committed
454
    DEBUG_printf("gc_free(%p)\n", ptr);
455
456

    if (VERIFY_PTR(ptr)) {
457
        mp_uint_t block = BLOCK_FROM_PTR(ptr);
458
        if (ATB_GET_KIND(block) == AT_HEAD) {
459
            // set the last_free pointer to this block if it's earlier in the heap
460
461
            if (block / BLOCKS_PER_ATB < MP_STATE_MEM(gc_last_free_atb_index)) {
                MP_STATE_MEM(gc_last_free_atb_index) = block / BLOCKS_PER_ATB;
462
463
            }

464
465
466
467
468
            // free head and all of its tail blocks
            do {
                ATB_ANY_TO_FREE(block);
                block += 1;
            } while (ATB_GET_KIND(block) == AT_TAIL);
469
470
471
472

            #if EXTENSIVE_HEAP_PROFILING
            gc_dump_alloc_table();
            #endif
473
474
        } else {
            assert(!"bad free");
475
        }
476
477
    } else if (ptr_in != NULL) {
        assert(!"bad free");
478
479
480
    }
}

481
mp_uint_t gc_nbytes(const void *ptr_in) {
482
    mp_uint_t ptr = (mp_uint_t)ptr_in;
Damien's avatar
Damien committed
483

484
    if (VERIFY_PTR(ptr)) {
485
        mp_uint_t block = BLOCK_FROM_PTR(ptr);
Damien's avatar
Damien committed
486
487
        if (ATB_GET_KIND(block) == AT_HEAD) {
            // work out number of consecutive blocks in the chain starting with this on
488
            mp_uint_t n_blocks = 0;
Damien's avatar
Damien committed
489
490
491
492
493
494
495
496
497
498
499
            do {
                n_blocks += 1;
            } while (ATB_GET_KIND(block + n_blocks) == AT_TAIL);
            return n_blocks * BYTES_PER_BLOCK;
        }
    }

    // invalid pointer
    return 0;
}

mux's avatar
mux committed
500
#if 0
501
// old, simple realloc that didn't expand memory in place
502
503
void *gc_realloc(void *ptr, mp_uint_t n_bytes) {
    mp_uint_t n_existing = gc_nbytes(ptr);
504
505
506
    if (n_bytes <= n_existing) {
        return ptr;
    } else {
507
508
509
510
        bool has_finaliser;
        if (ptr == NULL) {
            has_finaliser = false;
        } else {
511
#if MICROPY_ENABLE_FINALISER
512
            has_finaliser = FTB_GET(BLOCK_FROM_PTR((mp_uint_t)ptr));
513
#else
514
            has_finaliser = false;
515
#endif
516
517
        }
        void *ptr2 = gc_alloc(n_bytes, has_finaliser);
518
519
520
521
522
523
524
525
        if (ptr2 == NULL) {
            return ptr2;
        }
        memcpy(ptr2, ptr, n_existing);
        gc_free(ptr);
        return ptr2;
    }
}
526
527

#else // Alternative gc_realloc impl
528

529
void *gc_realloc(void *ptr_in, mp_uint_t n_bytes) {
530
    if (MP_STATE_MEM(gc_lock_depth) > 0) {
531
        return NULL;
532
533
    }

534
    // check for pure allocation
mux's avatar
mux committed
535
    if (ptr_in == NULL) {
536
        return gc_alloc(n_bytes, false);
mux's avatar
mux committed
537
538
    }

539
540
541
542
543
544
    // check for pure free
    if (n_bytes == 0) {
        gc_free(ptr_in);
        return NULL;
    }

545
    mp_uint_t ptr = (mp_uint_t)ptr_in;
546
547
548
549
550
551

    // sanity check the ptr
    if (!VERIFY_PTR(ptr)) {
        return NULL;
    }

552
    // get first block
553
    mp_uint_t block = BLOCK_FROM_PTR(ptr);
554

555
556
557
558
    // sanity check the ptr is pointing to the head of a block
    if (ATB_GET_KIND(block) != AT_HEAD) {
        return NULL;
    }
mux's avatar
mux committed
559

560
    // compute number of new blocks that are requested
561
    mp_uint_t new_blocks = (n_bytes + BYTES_PER_BLOCK - 1) / BYTES_PER_BLOCK;
mux's avatar
mux committed
562

563
564
565
566
567
568
    // Get the total number of consecutive blocks that are already allocated to
    // this chunk of memory, and then count the number of free blocks following
    // it.  Stop if we reach the end of the heap, or if we find enough extra
    // free blocks to satisfy the realloc.  Note that we need to compute the
    // total size of the existing memory chunk so we can correctly and
    // efficiently shrink it (see below for shrinking code).
569
570
    mp_uint_t n_free   = 0;
    mp_uint_t n_blocks = 1; // counting HEAD block
571
    mp_uint_t max_block = MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB;
572
573
574
575
576
    for (mp_uint_t bl = block + n_blocks; bl < max_block; bl++) {
        byte block_type = ATB_GET_KIND(bl);
        if (block_type == AT_TAIL) {
            n_blocks++;
            continue;
mux's avatar
mux committed
577
        }
578
579
580
581
582
583
584
        if (block_type == AT_FREE) {
            n_free++;
            if (n_blocks + n_free >= new_blocks) {
                // stop as soon as we find enough blocks for n_bytes
                break;
            }
            continue;
585
586
587
588
589
590
591
592
        }
        break;
    }

    // return original ptr if it already has the requested number of blocks
    if (new_blocks == n_blocks) {
        return ptr_in;
    }
mux's avatar
mux committed
593

594
595
596
    // check if we can shrink the allocated area
    if (new_blocks < n_blocks) {
        // free unneeded tail blocks
597
        for (mp_uint_t bl = block + new_blocks, count = n_blocks - new_blocks; count > 0; bl++, count--) {
598
            ATB_ANY_TO_FREE(bl);
599
        }
600
601

        // set the last_free pointer to end of this block if it's earlier in the heap
602
603
        if ((block + new_blocks) / BLOCKS_PER_ATB < MP_STATE_MEM(gc_last_free_atb_index)) {
            MP_STATE_MEM(gc_last_free_atb_index) = (block + new_blocks) / BLOCKS_PER_ATB;
604
605
606
607
608
609
        }

        #if EXTENSIVE_HEAP_PROFILING
        gc_dump_alloc_table();
        #endif

610
611
        return ptr_in;
    }
612

613
614
615
    // check if we can expand in place
    if (new_blocks <= n_blocks + n_free) {
        // mark few more blocks as used tail
616
        for (mp_uint_t bl = block + n_blocks; bl < block + new_blocks; bl++) {
617
618
619
            assert(ATB_GET_KIND(bl) == AT_FREE);
            ATB_FREE_TO_TAIL(bl);
        }
620

621
        // zero out the additional bytes of the newly allocated blocks (see comment above in gc_alloc)
622
        memset((byte*)ptr_in + n_bytes, 0, new_blocks * BYTES_PER_BLOCK - n_bytes);
623

624
625
626
627
        #if EXTENSIVE_HEAP_PROFILING
        gc_dump_alloc_table();
        #endif

628
629
        return ptr_in;
    }
mux's avatar
mux committed
630

631
632
    // can't resize inplace; try to find a new contiguous chain
    void *ptr_out = gc_alloc(n_bytes,
633
#if MICROPY_ENABLE_FINALISER
634
        FTB_GET(block)
635
#else
636
        false
637
#endif
638
639
640
641
642
    );

    // check that the alloc succeeded
    if (ptr_out == NULL) {
        return NULL;
Damien's avatar
Damien committed
643
    }
mux's avatar
mux committed
644

stijn's avatar
stijn committed
645
    DEBUG_printf("gc_realloc(%p -> %p)\n", ptr_in, ptr_out);
646
647
648
    memcpy(ptr_out, ptr_in, n_blocks * BYTES_PER_BLOCK);
    gc_free(ptr_in);
    return ptr_out;
Damien's avatar
Damien committed
649
}
650
#endif // Alternative gc_realloc impl
mux's avatar
mux committed
651

652
void gc_dump_info(void) {
653
654
655
656
657
658
659
    gc_info_t info;
    gc_info(&info);
    printf("GC: total: " UINT_FMT ", used: " UINT_FMT ", free: " UINT_FMT "\n", info.total, info.used, info.free);
    printf(" No. of 1-blocks: " UINT_FMT ", 2-blocks: " UINT_FMT ", max blk sz: " UINT_FMT "\n",
           info.num_1block, info.num_2block, info.max_block);
}

660
void gc_dump_alloc_table(void) {
661
662
663
664
    static const mp_uint_t DUMP_BYTES_PER_LINE = 64;
    #if !EXTENSIVE_HEAP_PROFILING
    // When comparing heap output we don't want to print the starting
    // pointer of the heap because it changes from run to run.
665
    printf("GC memory layout; from %p:", MP_STATE_MEM(gc_pool_start));
666
    #endif
667
    for (mp_uint_t bl = 0; bl < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; bl++) {
668
669
670
671
        if (bl % DUMP_BYTES_PER_LINE == 0) {
            // a new line of blocks
            {
                // check if this line contains only free blocks
672
                mp_uint_t bl2 = bl;
673
                while (bl2 < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB && ATB_GET_KIND(bl2) == AT_FREE) {
674
675
676
677
678
679
                    bl2++;
                }
                if (bl2 - bl >= 2 * DUMP_BYTES_PER_LINE) {
                    // there are at least 2 lines containing only free blocks, so abbreviate their printing
                    printf("\n       (" UINT_FMT " lines all free)", (bl2 - bl) / DUMP_BYTES_PER_LINE);
                    bl = bl2 & (~(DUMP_BYTES_PER_LINE - 1));
680
                    if (bl >= MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB) {
681
                        // got to end of heap
682
683
684
685
686
                        break;
                    }
                }
            }
            // print header for new line of blocks
687
688
689
690
691
            #if EXTENSIVE_HEAP_PROFILING
            printf("\n%05x: ", (uint)(bl * BYTES_PER_BLOCK) & 0xfffff);
            #else
            printf("\n%05x: ", (uint)PTR_FROM_BLOCK(bl) & 0xfffff);
            #endif
692
693
        }
        int c = ' ';
694
        switch (ATB_GET_KIND(bl)) {
695
            case AT_FREE: c = '.'; break;
696
697
698
            /* this prints out if the object is reachable from BSS or STACK (for unix only)
            case AT_HEAD: {
                c = 'h';
699
700
                void **ptrs = (void**)(void*)&mp_state_ctx;
                mp_uint_t len = offsetof(mp_state_ctx_t, vm.stack_top) / sizeof(mp_uint_t);
701
702
703
704
705
706
707
708
709
                for (mp_uint_t i = 0; i < len; i++) {
                    mp_uint_t ptr = (mp_uint_t)ptrs[i];
                    if (VERIFY_PTR(ptr) && BLOCK_FROM_PTR(ptr) == bl) {
                        c = 'B';
                        break;
                    }
                }
                if (c == 'h') {
                    ptrs = (void**)&c;
710
                    len = ((mp_uint_t)MP_STATE_VM(stack_top) - (mp_uint_t)&c) / sizeof(mp_uint_t);
711
712
713
714
715
716
717
718
719
720
721
                    for (mp_uint_t i = 0; i < len; i++) {
                        mp_uint_t ptr = (mp_uint_t)ptrs[i];
                        if (VERIFY_PTR(ptr) && BLOCK_FROM_PTR(ptr) == bl) {
                            c = 'S';
                            break;
                        }
                    }
                }
                break;
            }
            */
722
            /* this prints the uPy object type of the head block */
723
            case AT_HEAD: {
724
                mp_uint_t *ptr = MP_STATE_MEM(gc_pool_start) + bl * WORDS_PER_BLOCK;
725
726
727
                if (*ptr == (mp_uint_t)&mp_type_tuple) { c = 'T'; }
                else if (*ptr == (mp_uint_t)&mp_type_list) { c = 'L'; }
                else if (*ptr == (mp_uint_t)&mp_type_dict) { c = 'D'; }
728
                #if MICROPY_PY_BUILTINS_FLOAT
729
                else if (*ptr == (mp_uint_t)&mp_type_float) { c = 'F'; }
730
                #endif
731
                else if (*ptr == (mp_uint_t)&mp_type_fun_bc) { c = 'B'; }
732
                else if (*ptr == (mp_uint_t)&mp_type_module) { c = 'M'; }
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
                else {
                    c = 'h';
                    #if 0
                    // This code prints "Q" for qstr-pool data, and "q" for qstr-str
                    // data.  It can be useful to see how qstrs are being allocated,
                    // but is disabled by default because it is very slow.
                    for (qstr_pool_t *pool = MP_STATE_VM(last_pool); c == 'h' && pool != NULL; pool = pool->prev) {
                        if ((qstr_pool_t*)ptr == pool) {
                            c = 'Q';
                            break;
                        }
                        for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
                            if ((const byte*)ptr == *q) {
                                c = 'q';
                                break;
                            }
                        }
                    }
                    #endif
                }
753
754
                break;
            }
755
756
            case AT_TAIL: c = 't'; break;
            case AT_MARK: c = 'm'; break;
757
        }
758
        printf("%c", c);
759
    }
760
    printf("\n");
761
762
}

763
#if DEBUG_PRINT
764
void gc_test(void) {
765
766
767
    mp_uint_t len = 500;
    mp_uint_t *heap = malloc(len);
    gc_init(heap, heap + len / sizeof(mp_uint_t));
Damien's avatar
Damien committed
768
769
    void *ptrs[100];
    {
770
        mp_uint_t **p = gc_alloc(16, false);
771
772
773
774
        p[0] = gc_alloc(64, false);
        p[1] = gc_alloc(1, false);
        p[2] = gc_alloc(1, false);
        p[3] = gc_alloc(1, false);
775
        mp_uint_t ***p2 = gc_alloc(16, false);
Damien's avatar
Damien committed
776
777
778
779
        p2[0] = p;
        p2[1] = p;
        ptrs[0] = p2;
    }
780
    for (int i = 0; i < 25; i+=2) {
781
        mp_uint_t *p = gc_alloc(i, false);
Damien's avatar
Damien committed
782
783
784
785
786
787
        printf("p=%p\n", p);
        if (i & 3) {
            //ptrs[i] = p;
        }
    }

788
    printf("Before GC:\n");
789
    gc_dump_alloc_table();
790
791
792
793
794
    printf("Starting GC...\n");
    gc_collect_start();
    gc_collect_root(ptrs, sizeof(ptrs) / sizeof(void*));
    gc_collect_end();
    printf("After GC:\n");
795
    gc_dump_alloc_table();
Damien's avatar
Damien committed
796
}
797
#endif
798
799

#endif // MICROPY_ENABLE_GC