gc.c 15.9 KB
Newer Older
Damien's avatar
Damien committed
1
2
3
#include <stdio.h>
#include <string.h>

4
#include "mpconfig.h"
Damien's avatar
Damien committed
5
6
#include "gc.h"

7
8
#if MICROPY_ENABLE_GC

9
10
#if 0 // print debugging info
#define DEBUG_PRINT (1)
11
#define DEBUG_printf DEBUG_printf
12
#else // don't print debugging info
13
#define DEBUG_printf(...) (void)0
14
15
#endif

Damien's avatar
Damien committed
16
17
18
19
20
21
typedef unsigned char byte;

#define WORDS_PER_BLOCK (4)
#define BYTES_PER_BLOCK (WORDS_PER_BLOCK * BYTES_PER_WORD)
#define STACK_SIZE (64) // tunable; minimum is 1

22
23
24
25
STATIC byte *gc_alloc_table_start;
STATIC machine_uint_t gc_alloc_table_byte_len;
STATIC machine_uint_t *gc_pool_start;
STATIC machine_uint_t *gc_pool_end;
Damien's avatar
Damien committed
26

27
28
29
STATIC int gc_stack_overflow;
STATIC machine_uint_t gc_stack[STACK_SIZE];
STATIC machine_uint_t *gc_sp;
Damien's avatar
Damien committed
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

// ATB = allocation table byte
// 0b00 = FREE -- free block
// 0b01 = HEAD -- head of a chain of blocks
// 0b10 = TAIL -- in the tail of a chain of blocks
// 0b11 = MARK -- marked head block

#define AT_FREE (0)
#define AT_HEAD (1)
#define AT_TAIL (2)
#define AT_MARK (3)

#define BLOCKS_PER_ATB (4)
#define ATB_MASK_0 (0x03)
#define ATB_MASK_1 (0x0c)
#define ATB_MASK_2 (0x30)
#define ATB_MASK_3 (0xc0)

#define ATB_0_IS_FREE(a) (((a) & ATB_MASK_0) == 0)
#define ATB_1_IS_FREE(a) (((a) & ATB_MASK_1) == 0)
#define ATB_2_IS_FREE(a) (((a) & ATB_MASK_2) == 0)
#define ATB_3_IS_FREE(a) (((a) & ATB_MASK_3) == 0)

#define BLOCK_SHIFT(block) (2 * ((block) & (BLOCKS_PER_ATB - 1)))
#define ATB_GET_KIND(block) ((gc_alloc_table_start[(block) / BLOCKS_PER_ATB] >> BLOCK_SHIFT(block)) & 3)
#define ATB_ANY_TO_FREE(block) do { gc_alloc_table_start[(block) / BLOCKS_PER_ATB] &= (~(AT_MARK << BLOCK_SHIFT(block))); } while (0)
#define ATB_FREE_TO_HEAD(block) do { gc_alloc_table_start[(block) / BLOCKS_PER_ATB] |= (AT_HEAD << BLOCK_SHIFT(block)); } while (0)
#define ATB_FREE_TO_TAIL(block) do { gc_alloc_table_start[(block) / BLOCKS_PER_ATB] |= (AT_TAIL << BLOCK_SHIFT(block)); } while (0)
#define ATB_HEAD_TO_MARK(block) do { gc_alloc_table_start[(block) / BLOCKS_PER_ATB] |= (AT_MARK << BLOCK_SHIFT(block)); } while (0)
#define ATB_MARK_TO_HEAD(block) do { gc_alloc_table_start[(block) / BLOCKS_PER_ATB] &= (~(AT_TAIL << BLOCK_SHIFT(block))); } while (0)

#define BLOCK_FROM_PTR(ptr) (((ptr) - (machine_uint_t)gc_pool_start) / BYTES_PER_BLOCK)
#define PTR_FROM_BLOCK(block) (((block) * BYTES_PER_BLOCK + (machine_uint_t)gc_pool_start))
#define ATB_FROM_BLOCK(bl) ((bl) / BLOCKS_PER_ATB)

65
66
67
68
// TODO waste less memory; currently requires that all entries in alloc_table have a corresponding block in pool
void gc_init(void *start, void *end) {
    // align end pointer on block boundary
    end = (void*)((machine_uint_t)end & (~(BYTES_PER_BLOCK - 1)));
69
    DEBUG_printf("Initializing GC heap: %p-%p\n", start, end);
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

    // calculate parameters for GC
    machine_uint_t total_word_len = (machine_uint_t*)end - (machine_uint_t*)start;
    gc_alloc_table_byte_len = total_word_len * BYTES_PER_WORD / (1 + BITS_PER_BYTE / 2 * BYTES_PER_BLOCK);
    gc_alloc_table_start = (byte*)start;
    machine_uint_t gc_pool_block_len = gc_alloc_table_byte_len * BITS_PER_BYTE / 2;
    machine_uint_t gc_pool_word_len = gc_pool_block_len * WORDS_PER_BLOCK;
    gc_pool_start = (machine_uint_t*)end - gc_pool_word_len;
    gc_pool_end = end;

    // clear ATBs
    memset(gc_alloc_table_start, 0, gc_alloc_table_byte_len);

    // allocate first block because gc_pool_start points there and it will never
    // be freed, so allocating 1 block with null pointers will minimise memory loss
    ATB_FREE_TO_HEAD(0);
    for (int i = 0; i < WORDS_PER_BLOCK; i++) {
        gc_pool_start[i] = 0;
    }

90
    DEBUG_printf("GC layout:\n");
91
92
    DEBUG_printf("  alloc table at %p, length " UINT_FMT " bytes\n", gc_alloc_table_start, gc_alloc_table_byte_len);
    DEBUG_printf("  pool at %p, length " UINT_FMT " blocks = " UINT_FMT " words = " UINT_FMT " bytes\n", gc_pool_start, gc_pool_block_len, gc_pool_word_len, gc_pool_word_len * BYTES_PER_WORD);
93
94
}

95
96
97
98
99
100
#define VERIFY_PTR(ptr) ( \
        (ptr & (BYTES_PER_BLOCK - 1)) == 0          /* must be aligned on a block */ \
        && ptr >= (machine_uint_t)gc_pool_start     /* must be above start of pool */ \
        && ptr < (machine_uint_t)gc_pool_end        /* must be below end of pool */ \
    )

Damien's avatar
Damien committed
101
102
#define VERIFY_MARK_AND_PUSH(ptr) \
    do { \
103
        if (VERIFY_PTR(ptr)) { \
Damien's avatar
Damien committed
104
105
106
107
108
109
110
111
112
113
114
115
116
            machine_uint_t _block = BLOCK_FROM_PTR(ptr); \
            if (ATB_GET_KIND(_block) == AT_HEAD) { \
                /* an unmarked head, mark it, and push it on gc stack */ \
                ATB_HEAD_TO_MARK(_block); \
                if (gc_sp < &gc_stack[STACK_SIZE]) { \
                    *gc_sp++ = _block; \
                } else { \
                    gc_stack_overflow = 1; \
                } \
            } \
        } \
    } while (0)

117
STATIC void gc_drain_stack(void) {
Damien's avatar
Damien committed
118
119
120
121
    while (gc_sp > gc_stack) {
        // pop the next block off the stack
        machine_uint_t block = *--gc_sp;

122
        // work out number of consecutive blocks in the chain starting with this one
Damien's avatar
Damien committed
123
124
125
126
127
128
129
130
131
132
133
134
135
136
        machine_uint_t n_blocks = 0;
        do {
            n_blocks += 1;
        } while (ATB_GET_KIND(block + n_blocks) == AT_TAIL);

        // check this block's children
        machine_uint_t *scan = (machine_uint_t*)PTR_FROM_BLOCK(block);
        for (machine_uint_t i = n_blocks * WORDS_PER_BLOCK; i > 0; i--, scan++) {
            machine_uint_t ptr2 = *scan;
            VERIFY_MARK_AND_PUSH(ptr2);
        }
    }
}

137
STATIC void gc_deal_with_stack_overflow(void) {
Damien's avatar
Damien committed
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    while (gc_stack_overflow) {
        gc_stack_overflow = 0;
        gc_sp = gc_stack;

        // scan entire memory looking for blocks which have been marked but not their children
        for (machine_uint_t block = 0; block < gc_alloc_table_byte_len * BLOCKS_PER_ATB; block++) {
            // trace (again) if mark bit set
            if (ATB_GET_KIND(block) == AT_MARK) {
                *gc_sp++ = block;
                gc_drain_stack();
            }
        }
    }
}

153
STATIC void gc_sweep(void) {
Damien's avatar
Damien committed
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    // free unmarked heads and their tails
    int free_tail = 0;
    for (machine_uint_t block = 0; block < gc_alloc_table_byte_len * BLOCKS_PER_ATB; block++) {
        switch (ATB_GET_KIND(block)) {
            case AT_HEAD:
                free_tail = 1;
                // fall through to free the head

            case AT_TAIL:
                if (free_tail) {
                    ATB_ANY_TO_FREE(block);
                }
                break;

            case AT_MARK:
                ATB_MARK_TO_HEAD(block);
                free_tail = 0;
                break;
        }
    }
}

176
void gc_collect_start(void) {
Damien's avatar
Damien committed
177
178
179
180
181
182
183
184
185
186
187
188
    gc_stack_overflow = 0;
    gc_sp = gc_stack;
}

void gc_collect_root(void **ptrs, machine_uint_t len) {
    for (machine_uint_t i = 0; i < len; i++) {
        machine_uint_t ptr = (machine_uint_t)ptrs[i];
        VERIFY_MARK_AND_PUSH(ptr);
        gc_drain_stack();
    }
}

189
void gc_collect_end(void) {
Damien's avatar
Damien committed
190
191
    gc_deal_with_stack_overflow();
    gc_sweep();
192
}
Damien's avatar
Damien committed
193

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
void gc_info(gc_info_t *info) {
    info->total = (gc_pool_end - gc_pool_start) * sizeof(machine_uint_t);
    info->used = 0;
    info->free = 0;
    info->num_1block = 0;
    info->num_2block = 0;
    info->max_block = 0;
    for (machine_uint_t block = 0, len = 0; block < gc_alloc_table_byte_len * BLOCKS_PER_ATB; block++) {
        machine_uint_t kind = ATB_GET_KIND(block);
        if (kind == AT_FREE || kind == AT_HEAD) {
            if (len == 1) {
                info->num_1block += 1;
            } else if (len == 2) {
                info->num_2block += 1;
            }
            if (len > info->max_block) {
                info->max_block = len;
            }
        }
        switch (kind) {
Damien's avatar
Damien committed
214
            case AT_FREE:
215
216
                info->free += 1;
                len = 0;
Damien's avatar
Damien committed
217
218
219
                break;

            case AT_HEAD:
220
221
222
223
                info->used += 1;
                len = 1;
                break;

Damien's avatar
Damien committed
224
            case AT_TAIL:
225
226
                info->used += 1;
                len += 1;
Damien's avatar
Damien committed
227
228
229
                break;

            case AT_MARK:
230
                // shouldn't happen
Damien's avatar
Damien committed
231
232
233
234
                break;
        }
    }

235
236
    info->used *= BYTES_PER_BLOCK;
    info->free *= BYTES_PER_BLOCK;
Damien's avatar
Damien committed
237
238
239
240
}

void *gc_alloc(machine_uint_t n_bytes) {
    machine_uint_t n_blocks = ((n_bytes + BYTES_PER_BLOCK - 1) & (~(BYTES_PER_BLOCK - 1))) / BYTES_PER_BLOCK;
241
    DEBUG_printf("gc_alloc(" UINT_FMT " bytes -> " UINT_FMT " blocks)\n", n_bytes, n_blocks);
Damien's avatar
Damien committed
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267

    // check for 0 allocation
    if (n_blocks == 0) {
        return NULL;
    }

    machine_uint_t i;
    machine_uint_t end_block;
    machine_uint_t start_block;
    machine_uint_t n_free = 0;
    int collected = 0;
    for (;;) {

        // look for a run of n_blocks available blocks
        for (i = 0; i < gc_alloc_table_byte_len; i++) {
            byte a = gc_alloc_table_start[i];
            if (ATB_0_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 0; goto found; } } else { n_free = 0; }
            if (ATB_1_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 1; goto found; } } else { n_free = 0; }
            if (ATB_2_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 2; goto found; } } else { n_free = 0; }
            if (ATB_3_IS_FREE(a)) { if (++n_free >= n_blocks) { i = i * BLOCKS_PER_ATB + 3; goto found; } } else { n_free = 0; }
        }

        // nothing found!
        if (collected) {
            return NULL;
        }
268
        DEBUG_printf("gc_alloc(" UINT_FMT "): no free mem, triggering GC\n", n_bytes);
Damien's avatar
Damien committed
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
        gc_collect();
        collected = 1;
    }

    // found, ending at block i inclusive
found:
    // get starting and end blocks, both inclusive
    end_block = i;
    start_block = i - n_free + 1;

    // mark first block as used head
    ATB_FREE_TO_HEAD(start_block);

    // mark rest of blocks as used tail
    // TODO for a run of many blocks can make this more efficient
    for (machine_uint_t bl = start_block + 1; bl <= end_block; bl++) {
        ATB_FREE_TO_TAIL(bl);
    }

    // return pointer to first block
    return (void*)(gc_pool_start + start_block * WORDS_PER_BLOCK);
}

292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
// force the freeing of a piece of memory
void gc_free(void *ptr_in) {
    machine_uint_t ptr = (machine_uint_t)ptr_in;

    if (VERIFY_PTR(ptr)) {
        machine_uint_t block = BLOCK_FROM_PTR(ptr);
        if (ATB_GET_KIND(block) == AT_HEAD) {
            // free head and all of its tail blocks
            do {
                ATB_ANY_TO_FREE(block);
                block += 1;
            } while (ATB_GET_KIND(block) == AT_TAIL);
        }
    }
}

Damien's avatar
Damien committed
308
309
310
machine_uint_t gc_nbytes(void *ptr_in) {
    machine_uint_t ptr = (machine_uint_t)ptr_in;

311
    if (VERIFY_PTR(ptr)) {
Damien's avatar
Damien committed
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
        machine_uint_t block = BLOCK_FROM_PTR(ptr);
        if (ATB_GET_KIND(block) == AT_HEAD) {
            // work out number of consecutive blocks in the chain starting with this on
            machine_uint_t n_blocks = 0;
            do {
                n_blocks += 1;
            } while (ATB_GET_KIND(block + n_blocks) == AT_TAIL);
            return n_blocks * BYTES_PER_BLOCK;
        }
    }

    // invalid pointer
    return 0;
}

mux's avatar
mux committed
327
#if 0
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
// use this realloc for now, one below is broken
void *gc_realloc(void *ptr, machine_uint_t n_bytes) {
    machine_uint_t n_existing = gc_nbytes(ptr);
    if (n_bytes <= n_existing) {
        return ptr;
    } else {
        // TODO check if we can grow inplace
        void *ptr2 = gc_alloc(n_bytes);
        if (ptr2 == NULL) {
            return ptr2;
        }
        memcpy(ptr2, ptr, n_existing);
        gc_free(ptr);
        return ptr2;
    }
}
mux's avatar
mux committed
344
#else
mux's avatar
mux committed
345
346
void *gc_realloc(void *ptr_in, machine_uint_t n_bytes) {
    void *ptr_out = NULL;
347
    machine_uint_t block = 0;
mux's avatar
mux committed
348
349
350
351
352
353
    machine_uint_t ptr = (machine_uint_t)ptr_in;

    if (ptr_in == NULL) {
        return gc_alloc(n_bytes);
    }

Damien George's avatar
Damien George committed
354
355
356
    if (VERIFY_PTR(ptr)                         // verify pointer
        && (block = BLOCK_FROM_PTR(ptr))        // get first block
        && ATB_GET_KIND(block) == AT_HEAD) {    // make sure it's a HEAD block
mux's avatar
mux committed
357
358
359

        byte block_type;
        machine_uint_t n_free   = 0;
Damien George's avatar
Damien George committed
360
        machine_uint_t n_blocks = 1; // counting HEAD block
mux's avatar
mux committed
361
        machine_uint_t max_block = gc_alloc_table_byte_len * BLOCKS_PER_ATB;
mux's avatar
mux committed
362

Damien George's avatar
Damien George committed
363
364
365
        // get the number of consecutive tail blocks and
        // the number of free blocks after last tail block
        // stop if we reach (or are at) end of heap
mux's avatar
mux committed
366
        while ((block + n_blocks + n_free) < max_block
Damien George's avatar
Damien George committed
367
                // stop as soon as we find enough blocks for n_bytes
mux's avatar
mux committed
368
                && (n_bytes > ((n_blocks+n_free) * BYTES_PER_BLOCK))
Damien George's avatar
Damien George committed
369
                // stop if block is HEAD
mux's avatar
mux committed
370
                && (block_type = ATB_GET_KIND(block + n_blocks + n_free)) != AT_HEAD) {
mux's avatar
mux committed
371
372
373
374
375
            switch (block_type) {
                case AT_FREE: n_free++; break;
                case AT_TAIL: n_blocks++; break;
                default: break;
            }
mux's avatar
mux committed
376
        }
Damien George's avatar
Damien George committed
377
        // number of allocated bytes
mux's avatar
mux committed
378
379
        machine_uint_t n_existing = n_blocks * BYTES_PER_BLOCK;

Damien George's avatar
Damien George committed
380
        // check if realloc'ing to a smaller size
mux's avatar
mux committed
381
382
        if (n_bytes <= n_existing) {
            ptr_out = ptr_in;
Damien George's avatar
Damien George committed
383
            // free unneeded tail blocks
384
385
            for (machine_uint_t bl = block + n_blocks; ATB_GET_KIND(bl) == AT_TAIL; bl++) {
                ATB_ANY_TO_FREE(bl);
mux's avatar
mux committed
386
387
            }

Damien George's avatar
Damien George committed
388
        // check if we can expand in place
mux's avatar
mux committed
389
        } else if (n_bytes <= (n_existing + (n_free * BYTES_PER_BLOCK))) {
Damien George's avatar
Damien George committed
390
            // number of blocks needed to expand +1 if there's a remainder
mux's avatar
mux committed
391
392
393
394
395
396
            machine_uint_t n_diff = ( n_bytes - n_existing)/BYTES_PER_BLOCK+
                                    ((n_bytes - n_existing)%BYTES_PER_BLOCK!=0);

            DEBUG_printf("gc_realloc: expanding " UINT_FMT " blocks (" UINT_FMT " bytes) to " UINT_FMT " blocks (" UINT_FMT " bytes)\n",
                    n_existing/BYTES_PER_BLOCK, n_existing, n_existing/BYTES_PER_BLOCK+n_diff, n_existing + n_diff*BYTES_PER_BLOCK);

Damien George's avatar
Damien George committed
397
            // mark rest of blocks as used tail
398
            for (machine_uint_t bl = block + n_blocks; bl < (block + n_blocks + n_diff); bl++) {
mux's avatar
mux committed
399
400
401
402
                ATB_FREE_TO_TAIL(bl);
            }
            ptr_out = ptr_in;

Damien George's avatar
Damien George committed
403
        // try to find a new contiguous chain
mux's avatar
mux committed
404
        } else if ((ptr_out = gc_alloc(n_bytes)) != NULL) {
Damien George's avatar
Damien George committed
405
            DEBUG_printf("gc_realloc: allocating new block\n");
mux's avatar
mux committed
406
407
            memcpy(ptr_out, ptr_in, n_existing);
            gc_free(ptr_in);
408
        }
Damien's avatar
Damien committed
409
    }
mux's avatar
mux committed
410
411

    return ptr_out;
Damien's avatar
Damien committed
412
}
mux's avatar
mux committed
413

414
#endif
Damien's avatar
Damien committed
415

416
417
418
419
420
421
422
423
void gc_dump_info() {
    gc_info_t info;
    gc_info(&info);
    printf("GC: total: " UINT_FMT ", used: " UINT_FMT ", free: " UINT_FMT "\n", info.total, info.used, info.free);
    printf(" No. of 1-blocks: " UINT_FMT ", 2-blocks: " UINT_FMT ", max blk sz: " UINT_FMT "\n",
           info.num_1block, info.num_2block, info.max_block);
}

424
425
void gc_dump_alloc_table(void) {
    printf("GC memory layout:");
426
    for (machine_uint_t bl = 0; bl < gc_alloc_table_byte_len * BLOCKS_PER_ATB; bl++) {
427
428
429
430
        if (bl % 64 == 0) {
            printf("\n%04x: ", (uint)bl);
        }
        int c = ' ';
431
        switch (ATB_GET_KIND(bl)) {
432
433
434
435
            case AT_FREE: c = '.'; break;
            case AT_HEAD: c = 'h'; break;
            case AT_TAIL: c = 't'; break;
            case AT_MARK: c = 'm'; break;
436
        }
437
        printf("%c", c);
438
    }
439
    printf("\n");
440
441
}

442
#if DEBUG_PRINT
443
444
void gc_test(void) {
    machine_uint_t len = 500;
Damien's avatar
Damien committed
445
446
447
448
    machine_uint_t *heap = malloc(len);
    gc_init(heap, heap + len / sizeof(machine_uint_t));
    void *ptrs[100];
    {
449
        machine_uint_t **p = gc_alloc(16);
Damien's avatar
Damien committed
450
451
452
453
        p[0] = gc_alloc(64);
        p[1] = gc_alloc(1);
        p[2] = gc_alloc(1);
        p[3] = gc_alloc(1);
454
        machine_uint_t ***p2 = gc_alloc(16);
Damien's avatar
Damien committed
455
456
457
458
        p2[0] = p;
        p2[1] = p;
        ptrs[0] = p2;
    }
459
    for (int i = 0; i < 25; i+=2) {
Damien's avatar
Damien committed
460
461
462
463
464
465
466
        machine_uint_t *p = gc_alloc(i);
        printf("p=%p\n", p);
        if (i & 3) {
            //ptrs[i] = p;
        }
    }

467
    printf("Before GC:\n");
468
    gc_dump_alloc_table();
469
470
471
472
473
    printf("Starting GC...\n");
    gc_collect_start();
    gc_collect_root(ptrs, sizeof(ptrs) / sizeof(void*));
    gc_collect_end();
    printf("After GC:\n");
474
    gc_dump_alloc_table();
Damien's avatar
Damien committed
475
}
476
#endif
477
478

#endif // MICROPY_ENABLE_GC