| File: | ports/unix/../../py/gc.c |
| Warning: | line 978, column 36 Array access (from variable 'current_reference_block') results in a null pointer dereference |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* | |||
| 2 | * This file is part of the MicroPython project, http://micropython.org/ | |||
| 3 | * | |||
| 4 | * The MIT License (MIT) | |||
| 5 | * | |||
| 6 | * Copyright (c) 2013, 2014 Damien P. George | |||
| 7 | * | |||
| 8 | * Permission is hereby granted, free of charge, to any person obtaining a copy | |||
| 9 | * of this software and associated documentation files (the "Software"), to deal | |||
| 10 | * in the Software without restriction, including without limitation the rights | |||
| 11 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |||
| 12 | * copies of the Software, and to permit persons to whom the Software is | |||
| 13 | * furnished to do so, subject to the following conditions: | |||
| 14 | * | |||
| 15 | * The above copyright notice and this permission notice shall be included in | |||
| 16 | * all copies or substantial portions of the Software. | |||
| 17 | * | |||
| 18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||
| 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||
| 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |||
| 21 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |||
| 22 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |||
| 23 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | |||
| 24 | * THE SOFTWARE. | |||
| 25 | */ | |||
| 26 | ||||
| 27 | #include <assert.h> | |||
| 28 | #include <stdio.h> | |||
| 29 | #include <string.h> | |||
| 30 | ||||
| 31 | #include "py/gc.h" | |||
| 32 | #include "py/runtime.h" | |||
| 33 | ||||
| 34 | #include "supervisor/shared/safe_mode.h" | |||
| 35 | ||||
| 36 | #if MICROPY_ENABLE_GC(1) | |||
| 37 | ||||
| 38 | #if MICROPY_DEBUG_VERBOSE(0) // print debugging info | |||
| 39 | #define DEBUG_PRINT(0) (1) | |||
| 40 | #define DEBUG_printf DEBUG_printf | |||
| 41 | #else // don't print debugging info | |||
| 42 | #define DEBUG_PRINT(0) (0) | |||
| 43 | #define DEBUG_printf(...)(void)0 (void)0 | |||
| 44 | #endif | |||
| 45 | ||||
| 46 | // Uncomment this if you want to use a debugger to capture state at every allocation and free. | |||
| 47 | // #define LOG_HEAP_ACTIVITY 1 | |||
| 48 | ||||
| 49 | // make this 1 to dump the heap each time it changes | |||
| 50 | #define EXTENSIVE_HEAP_PROFILING(0) (0) | |||
| 51 | ||||
| 52 | // make this 1 to zero out swept memory to more eagerly | |||
| 53 | // detect untraced object still in use | |||
| 54 | #define CLEAR_ON_SWEEP(0) (0) | |||
| 55 | ||||
| 56 | #define WORDS_PER_BLOCK(((4 * (sizeof(mp_uint_t)))) / (sizeof(mp_uint_t))) ((MICROPY_BYTES_PER_GC_BLOCK(4 * (sizeof(mp_uint_t)))) / BYTES_PER_WORD(sizeof(mp_uint_t))) | |||
| 57 | #define BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) (MICROPY_BYTES_PER_GC_BLOCK(4 * (sizeof(mp_uint_t)))) | |||
| 58 | ||||
| 59 | // ATB = allocation table byte | |||
| 60 | // 0b00 = FREE -- free block | |||
| 61 | // 0b01 = HEAD -- head of a chain of blocks | |||
| 62 | // 0b10 = TAIL -- in the tail of a chain of blocks | |||
| 63 | // 0b11 = MARK -- marked head block | |||
| 64 | ||||
| 65 | #define AT_FREE(0) (0) | |||
| 66 | #define AT_HEAD(1) (1) | |||
| 67 | #define AT_TAIL(2) (2) | |||
| 68 | #define AT_MARK(3) (3) | |||
| 69 | ||||
| 70 | #define BLOCKS_PER_ATB(4) (4) | |||
| 71 | ||||
| 72 | #define BLOCK_SHIFT(block)(2 * ((block) & ((4) - 1))) (2 * ((block) & (BLOCKS_PER_ATB(4) - 1))) | |||
| 73 | #define ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3) ((MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start)[(block) / BLOCKS_PER_ATB(4)] >> BLOCK_SHIFT(block)(2 * ((block) & ((4) - 1)))) & 3) | |||
| 74 | #define ATB_ANY_TO_FREE(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] &= (~((3) << (2 * ((block) & ((4) - 1))))); } while ( 0) do { MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start)[(block) / BLOCKS_PER_ATB(4)] &= (~(AT_MARK(3) << BLOCK_SHIFT(block)(2 * ((block) & ((4) - 1))))); } while (0) | |||
| 75 | #define ATB_FREE_TO_HEAD(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] |= ((1) << (2 * ((block) & ((4) - 1)))); } while (0) do { MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start)[(block) / BLOCKS_PER_ATB(4)] |= (AT_HEAD(1) << BLOCK_SHIFT(block)(2 * ((block) & ((4) - 1)))); } while (0) | |||
| 76 | #define ATB_FREE_TO_TAIL(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] |= ((2) << (2 * ((block) & ((4) - 1)))); } while (0) do { MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start)[(block) / BLOCKS_PER_ATB(4)] |= (AT_TAIL(2) << BLOCK_SHIFT(block)(2 * ((block) & ((4) - 1)))); } while (0) | |||
| 77 | #define ATB_HEAD_TO_MARK(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] |= ((3) << (2 * ((block) & ((4) - 1)))); } while (0) do { MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start)[(block) / BLOCKS_PER_ATB(4)] |= (AT_MARK(3) << BLOCK_SHIFT(block)(2 * ((block) & ((4) - 1)))); } while (0) | |||
| 78 | #define ATB_MARK_TO_HEAD(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] &= (~((2) << (2 * ((block) & ((4) - 1))))); } while ( 0) do { MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start)[(block) / BLOCKS_PER_ATB(4)] &= (~(AT_TAIL(2) << BLOCK_SHIFT(block)(2 * ((block) & ((4) - 1))))); } while (0) | |||
| 79 | ||||
| 80 | #define BLOCK_FROM_PTR(ptr)(((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof (mp_uint_t))))) (((byte*)(ptr) - MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start)) / BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))) | |||
| 81 | #define PTR_FROM_BLOCK(block)(((block) * ((4 * (sizeof(mp_uint_t)))) + (uintptr_t)(mp_state_ctx .mem.gc_pool_start))) (((block) * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) + (uintptr_t)MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start))) | |||
| 82 | #define ATB_FROM_BLOCK(bl)((bl) / (4)) ((bl) / BLOCKS_PER_ATB(4)) | |||
| 83 | ||||
| 84 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 85 | // FTB = finaliser table byte | |||
| 86 | // if set, then the corresponding block may have a finaliser | |||
| 87 | ||||
| 88 | #define BLOCKS_PER_FTB(8) (8) | |||
| 89 | ||||
| 90 | #define FTB_GET(block)(((mp_state_ctx.mem.gc_finaliser_table_start)[(block) / (8)] >> ((block) & 7)) & 1) ((MP_STATE_MEM(gc_finaliser_table_start)(mp_state_ctx.mem.gc_finaliser_table_start)[(block) / BLOCKS_PER_FTB(8)] >> ((block) & 7)) & 1) | |||
| 91 | #define FTB_SET(block)do { (mp_state_ctx.mem.gc_finaliser_table_start)[(block) / (8 )] |= (1 << ((block) & 7)); } while (0) do { MP_STATE_MEM(gc_finaliser_table_start)(mp_state_ctx.mem.gc_finaliser_table_start)[(block) / BLOCKS_PER_FTB(8)] |= (1 << ((block) & 7)); } while (0) | |||
| 92 | #define FTB_CLEAR(block)do { (mp_state_ctx.mem.gc_finaliser_table_start)[(block) / (8 )] &= (~(1 << ((block) & 7))); } while (0) do { MP_STATE_MEM(gc_finaliser_table_start)(mp_state_ctx.mem.gc_finaliser_table_start)[(block) / BLOCKS_PER_FTB(8)] &= (~(1 << ((block) & 7))); } while (0) | |||
| 93 | #endif | |||
| 94 | ||||
| 95 | #if MICROPY_PY_THREAD1 && !MICROPY_PY_THREAD_GIL0 | |||
| 96 | #define GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1) mp_thread_mutex_lock(&MP_STATE_MEM(gc_mutex)(mp_state_ctx.mem.gc_mutex), 1) | |||
| 97 | #define GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)) mp_thread_mutex_unlock(&MP_STATE_MEM(gc_mutex)(mp_state_ctx.mem.gc_mutex)) | |||
| 98 | #else | |||
| 99 | #define GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1) | |||
| 100 | #define GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)) | |||
| 101 | #endif | |||
| 102 | ||||
| 103 | #ifdef LOG_HEAP_ACTIVITY | |||
| 104 | volatile uint32_t change_me; | |||
| 105 | #pragma GCC push_options | |||
| 106 | #pragma GCC optimize ("O0") | |||
| 107 | void __attribute__ ((noinline)) gc_log_change(uint32_t start_block, uint32_t length) { | |||
| 108 | change_me += start_block; | |||
| 109 | change_me += length; // Break on this line. | |||
| 110 | } | |||
| 111 | #pragma GCC pop_options | |||
| 112 | #endif | |||
| 113 | ||||
| 114 | // TODO waste less memory; currently requires that all entries in alloc_table have a corresponding block in pool | |||
| 115 | void gc_init(void *start, void *end) { | |||
| 116 | // align end pointer on block boundary | |||
| 117 | end = (void*)((uintptr_t)end & (~(BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) - 1))); | |||
| 118 | DEBUG_printf("Initializing GC heap: %p..%p = " UINT_FMT " bytes\n", start, end, (byte*)end - (byte*)start)(void)0; | |||
| 119 | ||||
| 120 | // calculate parameters for GC (T=total, A=alloc table, F=finaliser table, P=pool; all in bytes): | |||
| 121 | // T = A + F + P | |||
| 122 | // F = A * BLOCKS_PER_ATB / BLOCKS_PER_FTB | |||
| 123 | // P = A * BLOCKS_PER_ATB * BYTES_PER_BLOCK | |||
| 124 | // => T = A * (1 + BLOCKS_PER_ATB / BLOCKS_PER_FTB + BLOCKS_PER_ATB * BYTES_PER_BLOCK) | |||
| 125 | size_t total_byte_len = (byte*)end - (byte*)start; | |||
| 126 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 127 | MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) = total_byte_len * BITS_PER_BYTE(8) / (BITS_PER_BYTE(8) + BITS_PER_BYTE(8) * BLOCKS_PER_ATB(4) / BLOCKS_PER_FTB(8) + BITS_PER_BYTE(8) * BLOCKS_PER_ATB(4) * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 128 | #else | |||
| 129 | MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) = total_byte_len / (1 + BITS_PER_BYTE(8) / 2 * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 130 | #endif | |||
| 131 | ||||
| 132 | MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start) = (byte*)start; | |||
| 133 | ||||
| 134 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 135 | size_t gc_finaliser_table_byte_len = (MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4) + BLOCKS_PER_FTB(8) - 1) / BLOCKS_PER_FTB(8); | |||
| 136 | MP_STATE_MEM(gc_finaliser_table_start)(mp_state_ctx.mem.gc_finaliser_table_start) = MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start) + MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len); | |||
| 137 | #endif | |||
| 138 | ||||
| 139 | size_t gc_pool_block_len = MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4); | |||
| 140 | MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start) = (byte*)end - gc_pool_block_len * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))); | |||
| 141 | MP_STATE_MEM(gc_pool_end)(mp_state_ctx.mem.gc_pool_end) = end; | |||
| 142 | ||||
| 143 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 144 | assert(MP_STATE_MEM(gc_pool_start) >= MP_STATE_MEM(gc_finaliser_table_start) + gc_finaliser_table_byte_len)((void) (0)); | |||
| 145 | #endif | |||
| 146 | ||||
| 147 | // clear ATBs | |||
| 148 | memset(MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start), 0, MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len)); | |||
| 149 | ||||
| 150 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 151 | // clear FTBs | |||
| 152 | memset(MP_STATE_MEM(gc_finaliser_table_start)(mp_state_ctx.mem.gc_finaliser_table_start), 0, gc_finaliser_table_byte_len); | |||
| 153 | #endif | |||
| 154 | ||||
| 155 | // Set first free ATB index to the start of the heap. | |||
| 156 | MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index) = 0; | |||
| 157 | // Set last free ATB index to the end of the heap. | |||
| 158 | MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index) = MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) - 1; | |||
| 159 | // Set the lowest long lived ptr to the end of the heap to start. This will be lowered as long | |||
| 160 | // lived objects are allocated. | |||
| 161 | MP_STATE_MEM(gc_lowest_long_lived_ptr)(mp_state_ctx.mem.gc_lowest_long_lived_ptr) = (void*) PTR_FROM_BLOCK(MP_STATE_MEM(gc_alloc_table_byte_len * BLOCKS_PER_ATB))((((mp_state_ctx.mem.gc_alloc_table_byte_len * (4))) * ((4 * ( sizeof(mp_uint_t)))) + (uintptr_t)(mp_state_ctx.mem.gc_pool_start ))); | |||
| 162 | ||||
| 163 | // unlock the GC | |||
| 164 | MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth) = 0; | |||
| 165 | ||||
| 166 | // allow auto collection | |||
| 167 | MP_STATE_MEM(gc_auto_collect_enabled)(mp_state_ctx.mem.gc_auto_collect_enabled) = true1; | |||
| 168 | ||||
| 169 | #if MICROPY_GC_ALLOC_THRESHOLD(1) | |||
| 170 | // by default, maxuint for gc threshold, effectively turning gc-by-threshold off | |||
| 171 | MP_STATE_MEM(gc_alloc_threshold)(mp_state_ctx.mem.gc_alloc_threshold) = (size_t)-1; | |||
| 172 | MP_STATE_MEM(gc_alloc_amount)(mp_state_ctx.mem.gc_alloc_amount) = 0; | |||
| 173 | #endif | |||
| 174 | ||||
| 175 | #if MICROPY_PY_THREAD1 | |||
| 176 | mp_thread_mutex_init(&MP_STATE_MEM(gc_mutex)(mp_state_ctx.mem.gc_mutex)); | |||
| 177 | #endif | |||
| 178 | ||||
| 179 | MP_STATE_MEM(permanent_pointers)(mp_state_ctx.mem.permanent_pointers) = NULL((void*)0); | |||
| 180 | ||||
| 181 | DEBUG_printf("GC layout:\n")(void)0; | |||
| 182 | DEBUG_printf(" alloc table at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(gc_alloc_table_start), MP_STATE_MEM(gc_alloc_table_byte_len), MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB)(void)0; | |||
| 183 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 184 | DEBUG_printf(" finaliser table at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(gc_finaliser_table_start), gc_finaliser_table_byte_len, gc_finaliser_table_byte_len * BLOCKS_PER_FTB)(void)0; | |||
| 185 | #endif | |||
| 186 | DEBUG_printf(" pool at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", MP_STATE_MEM(gc_pool_start), gc_pool_block_len * BYTES_PER_BLOCK, gc_pool_block_len)(void)0; | |||
| 187 | } | |||
| 188 | ||||
| 189 | void gc_deinit(void) { | |||
| 190 | // Run any finalizers before we stop using the heap. | |||
| 191 | gc_sweep_all(); | |||
| 192 | ||||
| 193 | MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start) = 0; | |||
| 194 | } | |||
| 195 | ||||
| 196 | void gc_lock(void) { | |||
| 197 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 198 | MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth)++; | |||
| 199 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 200 | } | |||
| 201 | ||||
| 202 | void gc_unlock(void) { | |||
| 203 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 204 | MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth)--; | |||
| 205 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 206 | } | |||
| 207 | ||||
| 208 | bool_Bool gc_is_locked(void) { | |||
| 209 | return MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth) != 0; | |||
| 210 | } | |||
| 211 | ||||
| 212 | // ptr should be of type void* | |||
| 213 | #define VERIFY_PTR(ptr)( ((uintptr_t)(ptr) & (((4 * (sizeof(mp_uint_t)))) - 1)) == 0 && ptr >= (void*)(mp_state_ctx.mem.gc_pool_start ) && ptr < (void*)(mp_state_ctx.mem.gc_pool_end) ) ( \ | |||
| 214 | ((uintptr_t)(ptr) & (BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) - 1)) == 0 /* must be aligned on a block */ \ | |||
| 215 | && ptr >= (void*)MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start) /* must be above start of pool */ \ | |||
| 216 | && ptr < (void*)MP_STATE_MEM(gc_pool_end)(mp_state_ctx.mem.gc_pool_end) /* must be below end of pool */ \ | |||
| 217 | ) | |||
| 218 | ||||
| 219 | #ifndef TRACE_MARK | |||
| 220 | #if DEBUG_PRINT(0) | |||
| 221 | #define TRACE_MARK(block, ptr) DEBUG_printf("gc_mark(%p)\n", ptr)(void)0 | |||
| 222 | #else | |||
| 223 | #define TRACE_MARK(block, ptr) | |||
| 224 | #endif | |||
| 225 | #endif | |||
| 226 | ||||
| 227 | // Take the given block as the topmost block on the stack. Check all it's | |||
| 228 | // children: mark the unmarked child blocks and put those newly marked | |||
| 229 | // blocks on the stack. When all children have been checked, pop off the | |||
| 230 | // topmost block on the stack and repeat with that one. | |||
| 231 | STATICstatic void gc_mark_subtree(size_t block) { | |||
| 232 | // Start with the block passed in the argument. | |||
| 233 | size_t sp = 0; | |||
| 234 | for (;;) { | |||
| 235 | // work out number of consecutive blocks in the chain starting with this one | |||
| 236 | size_t n_blocks = 0; | |||
| 237 | do { | |||
| 238 | n_blocks += 1; | |||
| 239 | } while (ATB_GET_KIND(block + n_blocks)(((mp_state_ctx.mem.gc_alloc_table_start)[(block + n_blocks) / (4)] >> (2 * ((block + n_blocks) & ((4) - 1)))) & 3) == AT_TAIL(2)); | |||
| 240 | ||||
| 241 | // check this block's children | |||
| 242 | void **ptrs = (void**)PTR_FROM_BLOCK(block)(((block) * ((4 * (sizeof(mp_uint_t)))) + (uintptr_t)(mp_state_ctx .mem.gc_pool_start))); | |||
| 243 | for (size_t i = n_blocks * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) / sizeof(void*); i > 0; i--, ptrs++) { | |||
| 244 | void *ptr = *ptrs; | |||
| 245 | if (VERIFY_PTR(ptr)( ((uintptr_t)(ptr) & (((4 * (sizeof(mp_uint_t)))) - 1)) == 0 && ptr >= (void*)(mp_state_ctx.mem.gc_pool_start ) && ptr < (void*)(mp_state_ctx.mem.gc_pool_end) )) { | |||
| 246 | // Mark and push this pointer | |||
| 247 | size_t childblock = BLOCK_FROM_PTR(ptr)(((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof (mp_uint_t))))); | |||
| 248 | if (ATB_GET_KIND(childblock)(((mp_state_ctx.mem.gc_alloc_table_start)[(childblock) / (4)] >> (2 * ((childblock) & ((4) - 1)))) & 3) == AT_HEAD(1)) { | |||
| 249 | // an unmarked head, mark it, and push it on gc stack | |||
| 250 | TRACE_MARK(childblock, ptr); | |||
| 251 | ATB_HEAD_TO_MARK(childblock)do { (mp_state_ctx.mem.gc_alloc_table_start)[(childblock) / ( 4)] |= ((3) << (2 * ((childblock) & ((4) - 1)))); } while (0); | |||
| 252 | if (sp < MICROPY_ALLOC_GC_STACK_SIZE(64)) { | |||
| 253 | MP_STATE_MEM(gc_stack)(mp_state_ctx.mem.gc_stack)[sp++] = childblock; | |||
| 254 | } else { | |||
| 255 | MP_STATE_MEM(gc_stack_overflow)(mp_state_ctx.mem.gc_stack_overflow) = 1; | |||
| 256 | } | |||
| 257 | } | |||
| 258 | } | |||
| 259 | } | |||
| 260 | ||||
| 261 | // Are there any blocks on the stack? | |||
| 262 | if (sp == 0) { | |||
| 263 | break; // No, stack is empty, we're done. | |||
| 264 | } | |||
| 265 | ||||
| 266 | // pop the next block off the stack | |||
| 267 | block = MP_STATE_MEM(gc_stack)(mp_state_ctx.mem.gc_stack)[--sp]; | |||
| 268 | } | |||
| 269 | } | |||
| 270 | ||||
| 271 | STATICstatic void gc_deal_with_stack_overflow(void) { | |||
| 272 | while (MP_STATE_MEM(gc_stack_overflow)(mp_state_ctx.mem.gc_stack_overflow)) { | |||
| 273 | MP_STATE_MEM(gc_stack_overflow)(mp_state_ctx.mem.gc_stack_overflow) = 0; | |||
| 274 | ||||
| 275 | // scan entire memory looking for blocks which have been marked but not their children | |||
| 276 | for (size_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4); block++) { | |||
| 277 | // trace (again) if mark bit set | |||
| 278 | if (ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3) == AT_MARK(3)) { | |||
| 279 | gc_mark_subtree(block); | |||
| 280 | } | |||
| 281 | } | |||
| 282 | } | |||
| 283 | } | |||
| 284 | ||||
| 285 | STATICstatic void gc_sweep(void) { | |||
| 286 | #if MICROPY_PY_GC_COLLECT_RETVAL(1) | |||
| 287 | MP_STATE_MEM(gc_collected)(mp_state_ctx.mem.gc_collected) = 0; | |||
| 288 | #endif | |||
| 289 | // free unmarked heads and their tails | |||
| 290 | int free_tail = 0; | |||
| 291 | for (size_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4); block++) { | |||
| 292 | switch (ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3)) { | |||
| 293 | case AT_HEAD(1): | |||
| 294 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 295 | if (FTB_GET(block)(((mp_state_ctx.mem.gc_finaliser_table_start)[(block) / (8)] >> ((block) & 7)) & 1)) { | |||
| 296 | mp_obj_base_t *obj = (mp_obj_base_t*)PTR_FROM_BLOCK(block)(((block) * ((4 * (sizeof(mp_uint_t)))) + (uintptr_t)(mp_state_ctx .mem.gc_pool_start))); | |||
| 297 | if (obj->type != NULL((void*)0)) { | |||
| 298 | // if the object has a type then see if it has a __del__ method | |||
| 299 | mp_obj_t dest[2]; | |||
| 300 | mp_load_method_maybe(MP_OBJ_FROM_PTR(obj)((mp_obj_t)obj), MP_QSTR___del__, dest); | |||
| 301 | if (dest[0] != MP_OBJ_NULL(((mp_obj_t)(void*)0))) { | |||
| 302 | // load_method returned a method, execute it in a protected environment | |||
| 303 | #if MICROPY_ENABLE_SCHEDULER(0) | |||
| 304 | mp_sched_lock(); | |||
| 305 | #endif | |||
| 306 | mp_call_function_1_protected(dest[0], dest[1]); | |||
| 307 | #if MICROPY_ENABLE_SCHEDULER(0) | |||
| 308 | mp_sched_unlock(); | |||
| 309 | #endif | |||
| 310 | } | |||
| 311 | } | |||
| 312 | // clear finaliser flag | |||
| 313 | FTB_CLEAR(block)do { (mp_state_ctx.mem.gc_finaliser_table_start)[(block) / (8 )] &= (~(1 << ((block) & 7))); } while (0); | |||
| 314 | } | |||
| 315 | #endif | |||
| 316 | free_tail = 1; | |||
| 317 | ATB_ANY_TO_FREE(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] &= (~((3) << (2 * ((block) & ((4) - 1))))); } while ( 0); | |||
| 318 | #if CLEAR_ON_SWEEP(0) | |||
| 319 | memset((void*)PTR_FROM_BLOCK(block)(((block) * ((4 * (sizeof(mp_uint_t)))) + (uintptr_t)(mp_state_ctx .mem.gc_pool_start))), 0, BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 320 | #endif | |||
| 321 | DEBUG_printf("gc_sweep(%x)\n", PTR_FROM_BLOCK(block))(void)0; | |||
| 322 | ||||
| 323 | #ifdef LOG_HEAP_ACTIVITY | |||
| 324 | gc_log_change(block, 0); | |||
| 325 | #endif | |||
| 326 | #if MICROPY_PY_GC_COLLECT_RETVAL(1) | |||
| 327 | MP_STATE_MEM(gc_collected)(mp_state_ctx.mem.gc_collected)++; | |||
| 328 | #endif | |||
| 329 | break; | |||
| 330 | ||||
| 331 | case AT_TAIL(2): | |||
| 332 | if (free_tail) { | |||
| 333 | ATB_ANY_TO_FREE(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] &= (~((3) << (2 * ((block) & ((4) - 1))))); } while ( 0); | |||
| 334 | #if CLEAR_ON_SWEEP(0) | |||
| 335 | memset((void*)PTR_FROM_BLOCK(block)(((block) * ((4 * (sizeof(mp_uint_t)))) + (uintptr_t)(mp_state_ctx .mem.gc_pool_start))), 0, BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 336 | #endif | |||
| 337 | } | |||
| 338 | break; | |||
| 339 | ||||
| 340 | case AT_MARK(3): | |||
| 341 | ATB_MARK_TO_HEAD(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] &= (~((2) << (2 * ((block) & ((4) - 1))))); } while ( 0); | |||
| 342 | free_tail = 0; | |||
| 343 | break; | |||
| 344 | } | |||
| 345 | } | |||
| 346 | } | |||
| 347 | ||||
| 348 | // Mark can handle NULL pointers because it verifies the pointer is within the heap bounds. | |||
| 349 | STATICstatic void gc_mark(void* ptr) { | |||
| 350 | if (VERIFY_PTR(ptr)( ((uintptr_t)(ptr) & (((4 * (sizeof(mp_uint_t)))) - 1)) == 0 && ptr >= (void*)(mp_state_ctx.mem.gc_pool_start ) && ptr < (void*)(mp_state_ctx.mem.gc_pool_end) )) { | |||
| 351 | size_t block = BLOCK_FROM_PTR(ptr)(((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof (mp_uint_t))))); | |||
| 352 | if (ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3) == AT_HEAD(1)) { | |||
| 353 | // An unmarked head: mark it, and mark all its children | |||
| 354 | TRACE_MARK(block, ptr); | |||
| 355 | ATB_HEAD_TO_MARK(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] |= ((3) << (2 * ((block) & ((4) - 1)))); } while (0); | |||
| 356 | gc_mark_subtree(block); | |||
| 357 | } | |||
| 358 | } | |||
| 359 | } | |||
| 360 | ||||
| 361 | void gc_collect_start(void) { | |||
| 362 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 363 | MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth)++; | |||
| 364 | #if MICROPY_GC_ALLOC_THRESHOLD(1) | |||
| 365 | MP_STATE_MEM(gc_alloc_amount)(mp_state_ctx.mem.gc_alloc_amount) = 0; | |||
| 366 | #endif | |||
| 367 | MP_STATE_MEM(gc_stack_overflow)(mp_state_ctx.mem.gc_stack_overflow) = 0; | |||
| 368 | ||||
| 369 | // Trace root pointers. This relies on the root pointers being organised | |||
| 370 | // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, | |||
| 371 | // dict_globals, then the root pointer section of mp_state_vm. | |||
| 372 | void **ptrs = (void**)(void*)&mp_state_ctx; | |||
| 373 | size_t root_start = offsetof(mp_state_ctx_t, thread.dict_locals)__builtin_offsetof(mp_state_ctx_t, thread.dict_locals); | |||
| 374 | size_t root_end = offsetof(mp_state_ctx_t, vm.qstr_last_chunk)__builtin_offsetof(mp_state_ctx_t, vm.qstr_last_chunk); | |||
| 375 | gc_collect_root(ptrs + root_start / sizeof(void*), (root_end - root_start) / sizeof(void*)); | |||
| 376 | ||||
| 377 | gc_mark(MP_STATE_MEM(permanent_pointers)(mp_state_ctx.mem.permanent_pointers)); | |||
| 378 | ||||
| 379 | #if MICROPY_ENABLE_PYSTACK(0) | |||
| 380 | // Trace root pointers from the Python stack. | |||
| 381 | ptrs = (void**)(void*)MP_STATE_THREAD(pystack_start)(mp_thread_get_state()->pystack_start); | |||
| 382 | gc_collect_root(ptrs, (MP_STATE_THREAD(pystack_cur)(mp_thread_get_state()->pystack_cur) - MP_STATE_THREAD(pystack_start)(mp_thread_get_state()->pystack_start)) / sizeof(void*)); | |||
| 383 | #endif | |||
| 384 | } | |||
| 385 | ||||
| 386 | void gc_collect_ptr(void *ptr) { | |||
| 387 | gc_mark(ptr); | |||
| 388 | } | |||
| 389 | ||||
| 390 | void gc_collect_root(void **ptrs, size_t len) { | |||
| 391 | for (size_t i = 0; i < len; i++) { | |||
| 392 | void *ptr = ptrs[i]; | |||
| 393 | gc_mark(ptr); | |||
| 394 | } | |||
| 395 | } | |||
| 396 | ||||
| 397 | void gc_collect_end(void) { | |||
| 398 | gc_deal_with_stack_overflow(); | |||
| 399 | gc_sweep(); | |||
| 400 | MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index) = 0; | |||
| 401 | MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index) = MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) - 1; | |||
| 402 | MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth)--; | |||
| 403 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 404 | } | |||
| 405 | ||||
| 406 | void gc_sweep_all(void) { | |||
| 407 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 408 | MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth)++; | |||
| 409 | MP_STATE_MEM(gc_stack_overflow)(mp_state_ctx.mem.gc_stack_overflow) = 0; | |||
| 410 | gc_collect_end(); | |||
| 411 | } | |||
| 412 | ||||
| 413 | void gc_info(gc_info_t *info) { | |||
| 414 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 415 | info->total = MP_STATE_MEM(gc_pool_end)(mp_state_ctx.mem.gc_pool_end) - MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start); | |||
| 416 | info->used = 0; | |||
| 417 | info->free = 0; | |||
| 418 | info->max_free = 0; | |||
| 419 | info->num_1block = 0; | |||
| 420 | info->num_2block = 0; | |||
| 421 | info->max_block = 0; | |||
| 422 | bool_Bool finish = false0; | |||
| 423 | for (size_t block = 0, len = 0, len_free = 0; !finish;) { | |||
| 424 | size_t kind = ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3); | |||
| 425 | switch (kind) { | |||
| 426 | case AT_FREE(0): | |||
| 427 | info->free += 1; | |||
| 428 | len_free += 1; | |||
| 429 | len = 0; | |||
| 430 | break; | |||
| 431 | ||||
| 432 | case AT_HEAD(1): | |||
| 433 | info->used += 1; | |||
| 434 | len = 1; | |||
| 435 | break; | |||
| 436 | ||||
| 437 | case AT_TAIL(2): | |||
| 438 | info->used += 1; | |||
| 439 | len += 1; | |||
| 440 | break; | |||
| 441 | ||||
| 442 | case AT_MARK(3): | |||
| 443 | // shouldn't happen | |||
| 444 | break; | |||
| 445 | } | |||
| 446 | ||||
| 447 | block++; | |||
| 448 | finish = (block == MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4)); | |||
| 449 | // Get next block type if possible | |||
| 450 | if (!finish) { | |||
| 451 | kind = ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3); | |||
| 452 | } | |||
| 453 | ||||
| 454 | if (finish || kind == AT_FREE(0) || kind == AT_HEAD(1)) { | |||
| 455 | if (len == 1) { | |||
| 456 | info->num_1block += 1; | |||
| 457 | } else if (len == 2) { | |||
| 458 | info->num_2block += 1; | |||
| 459 | } | |||
| 460 | if (len > info->max_block) { | |||
| 461 | info->max_block = len; | |||
| 462 | } | |||
| 463 | if (finish || kind == AT_HEAD(1)) { | |||
| 464 | if (len_free > info->max_free) { | |||
| 465 | info->max_free = len_free; | |||
| 466 | } | |||
| 467 | len_free = 0; | |||
| 468 | } | |||
| 469 | } | |||
| 470 | } | |||
| 471 | ||||
| 472 | info->used *= BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))); | |||
| 473 | info->free *= BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))); | |||
| 474 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 475 | } | |||
| 476 | ||||
| 477 | // We place long lived objects at the end of the heap rather than the start. This reduces | |||
| 478 | // fragmentation by localizing the heap churn to one portion of memory (the start of the heap.) | |||
| 479 | void *gc_alloc(size_t n_bytes, bool_Bool has_finaliser, bool_Bool long_lived) { | |||
| 480 | size_t n_blocks = ((n_bytes + BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) - 1) & (~(BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) - 1))) / BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))); | |||
| 481 | DEBUG_printf("gc_alloc(" UINT_FMT " bytes -> " UINT_FMT " blocks)\n", n_bytes, n_blocks)(void)0; | |||
| 482 | ||||
| 483 | // check for 0 allocation | |||
| 484 | if (n_blocks == 0) { | |||
| 485 | return NULL((void*)0); | |||
| 486 | } | |||
| 487 | ||||
| 488 | if (MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start) == 0) { | |||
| 489 | reset_into_safe_mode(GC_ALLOC_OUTSIDE_VM); | |||
| 490 | } | |||
| 491 | ||||
| 492 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 493 | ||||
| 494 | // check if GC is locked | |||
| 495 | if (MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth) > 0) { | |||
| 496 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 497 | return NULL((void*)0); | |||
| 498 | } | |||
| 499 | ||||
| 500 | size_t found_block = 0xffffffff; | |||
| 501 | size_t end_block; | |||
| 502 | size_t start_block; | |||
| 503 | size_t n_free; | |||
| 504 | bool_Bool collected = !MP_STATE_MEM(gc_auto_collect_enabled)(mp_state_ctx.mem.gc_auto_collect_enabled); | |||
| 505 | ||||
| 506 | #if MICROPY_GC_ALLOC_THRESHOLD(1) | |||
| 507 | if (!collected && MP_STATE_MEM(gc_alloc_amount)(mp_state_ctx.mem.gc_alloc_amount) >= MP_STATE_MEM(gc_alloc_threshold)(mp_state_ctx.mem.gc_alloc_threshold)) { | |||
| 508 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 509 | gc_collect(); | |||
| 510 | collected = 1; | |||
| 511 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 512 | collected = true1; | |||
| 513 | } | |||
| 514 | #endif | |||
| 515 | ||||
| 516 | bool_Bool keep_looking = true1; | |||
| 517 | ||||
| 518 | // When we start searching on the other side of the crossover block we make sure to | |||
| 519 | // perform a collect. That way we'll get the closest free block in our section. | |||
| 520 | size_t crossover_block = BLOCK_FROM_PTR(MP_STATE_MEM(gc_lowest_long_lived_ptr))(((byte*)((mp_state_ctx.mem.gc_lowest_long_lived_ptr)) - (mp_state_ctx .mem.gc_pool_start)) / ((4 * (sizeof(mp_uint_t))))); | |||
| 521 | while (keep_looking) { | |||
| 522 | int8_t direction = 1; | |||
| 523 | size_t start = MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index); | |||
| 524 | if (long_lived) { | |||
| 525 | direction = -1; | |||
| 526 | start = MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index); | |||
| 527 | } | |||
| 528 | n_free = 0; | |||
| 529 | // look for a run of n_blocks available blocks | |||
| 530 | for (size_t i = start; keep_looking && MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index) <= i && i <= MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index); i += direction) { | |||
| 531 | byte a = MP_STATE_MEM(gc_alloc_table_start)(mp_state_ctx.mem.gc_alloc_table_start)[i]; | |||
| 532 | // Four ATB states are packed into a single byte. | |||
| 533 | int j = 0; | |||
| 534 | if (direction == -1) { | |||
| 535 | j = 3; | |||
| 536 | } | |||
| 537 | for (; keep_looking && 0 <= j && j <= 3; j += direction) { | |||
| 538 | if ((a & (0x3 << (j * 2))) == 0) { | |||
| 539 | if (++n_free >= n_blocks) { | |||
| 540 | found_block = i * BLOCKS_PER_ATB(4) + j; | |||
| 541 | keep_looking = false0; | |||
| 542 | } | |||
| 543 | } else { | |||
| 544 | if (!collected) { | |||
| 545 | size_t block = i * BLOCKS_PER_ATB(4) + j; | |||
| 546 | if ((direction == 1 && block >= crossover_block) || | |||
| 547 | (direction == -1 && block < crossover_block)) { | |||
| 548 | keep_looking = false0; | |||
| 549 | } | |||
| 550 | } | |||
| 551 | n_free = 0; | |||
| 552 | } | |||
| 553 | } | |||
| 554 | } | |||
| 555 | if (n_free >= n_blocks) { | |||
| 556 | break; | |||
| 557 | } | |||
| 558 | ||||
| 559 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 560 | // nothing found! | |||
| 561 | if (collected) { | |||
| 562 | return NULL((void*)0); | |||
| 563 | } | |||
| 564 | DEBUG_printf("gc_alloc(" UINT_FMT "): no free mem, triggering GC\n", n_bytes)(void)0; | |||
| 565 | gc_collect(); | |||
| 566 | collected = true1; | |||
| 567 | // Try again since we've hopefully freed up space. | |||
| 568 | keep_looking = true1; | |||
| 569 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 570 | } | |||
| 571 | assert(found_block != 0xffffffff)((void) (0)); | |||
| 572 | ||||
| 573 | // Found free space ending at found_block inclusive. | |||
| 574 | // Also, set last free ATB index to block after last block we found, for start of | |||
| 575 | // next scan. To reduce fragmentation, we only do this if we were looking | |||
| 576 | // for a single free block, which guarantees that there are no free blocks | |||
| 577 | // before this one. Also, whenever we free or shrink a block we must check | |||
| 578 | // if this index needs adjusting (see gc_realloc and gc_free). | |||
| 579 | if (!long_lived) { | |||
| 580 | end_block = found_block; | |||
| 581 | start_block = found_block - n_free + 1; | |||
| 582 | if (n_blocks == 1) { | |||
| 583 | MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index) = (found_block + 1) / BLOCKS_PER_ATB(4); | |||
| 584 | } | |||
| 585 | } else { | |||
| 586 | start_block = found_block; | |||
| 587 | end_block = found_block + n_free - 1; | |||
| 588 | if (n_blocks == 1) { | |||
| 589 | MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index) = (found_block - 1) / BLOCKS_PER_ATB(4); | |||
| 590 | } | |||
| 591 | } | |||
| 592 | ||||
| 593 | #ifdef LOG_HEAP_ACTIVITY | |||
| 594 | gc_log_change(start_block, end_block - start_block + 1); | |||
| 595 | #endif | |||
| 596 | ||||
| 597 | // mark first block as used head | |||
| 598 | ATB_FREE_TO_HEAD(start_block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(start_block) / ( 4)] |= ((1) << (2 * ((start_block) & ((4) - 1)))); } while (0); | |||
| 599 | ||||
| 600 | // mark rest of blocks as used tail | |||
| 601 | // TODO for a run of many blocks can make this more efficient | |||
| 602 | for (size_t bl = start_block + 1; bl <= end_block; bl++) { | |||
| 603 | ATB_FREE_TO_TAIL(bl)do { (mp_state_ctx.mem.gc_alloc_table_start)[(bl) / (4)] |= ( (2) << (2 * ((bl) & ((4) - 1)))); } while (0); | |||
| 604 | } | |||
| 605 | ||||
| 606 | // get pointer to first block | |||
| 607 | // we must create this pointer before unlocking the GC so a collection can find it | |||
| 608 | void *ret_ptr = (void*)(MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start) + start_block * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 609 | DEBUG_printf("gc_alloc(%p)\n", ret_ptr)(void)0; | |||
| 610 | ||||
| 611 | // If the allocation was long live then update the lowest value. Its used to trigger early | |||
| 612 | // collects when allocations fail in their respective section. Its also used to ignore calls to | |||
| 613 | // gc_make_long_lived where the pointer is already in the long lived section. | |||
| 614 | if (long_lived && ret_ptr < MP_STATE_MEM(gc_lowest_long_lived_ptr)(mp_state_ctx.mem.gc_lowest_long_lived_ptr)) { | |||
| 615 | MP_STATE_MEM(gc_lowest_long_lived_ptr)(mp_state_ctx.mem.gc_lowest_long_lived_ptr) = ret_ptr; | |||
| 616 | } | |||
| 617 | ||||
| 618 | #if MICROPY_GC_ALLOC_THRESHOLD(1) | |||
| 619 | MP_STATE_MEM(gc_alloc_amount)(mp_state_ctx.mem.gc_alloc_amount) += n_blocks; | |||
| 620 | #endif | |||
| 621 | ||||
| 622 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 623 | ||||
| 624 | #if MICROPY_GC_CONSERVATIVE_CLEAR((1)) | |||
| 625 | // be conservative and zero out all the newly allocated blocks | |||
| 626 | memset((byte*)ret_ptr, 0, (end_block - start_block + 1) * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 627 | #else | |||
| 628 | // zero out the additional bytes of the newly allocated blocks | |||
| 629 | // This is needed because the blocks may have previously held pointers | |||
| 630 | // to the heap and will not be set to something else if the caller | |||
| 631 | // doesn't actually use the entire block. As such they will continue | |||
| 632 | // to point to the heap and may prevent other blocks from being reclaimed. | |||
| 633 | memset((byte*)ret_ptr + n_bytes, 0, (end_block - start_block + 1) * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) - n_bytes); | |||
| 634 | #endif | |||
| 635 | ||||
| 636 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 637 | if (has_finaliser) { | |||
| 638 | // clear type pointer in case it is never set | |||
| 639 | ((mp_obj_base_t*)ret_ptr)->type = NULL((void*)0); | |||
| 640 | // set mp_obj flag only if it has a finaliser | |||
| 641 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 642 | FTB_SET(start_block)do { (mp_state_ctx.mem.gc_finaliser_table_start)[(start_block ) / (8)] |= (1 << ((start_block) & 7)); } while (0); | |||
| 643 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 644 | } | |||
| 645 | #else | |||
| 646 | (void)has_finaliser; | |||
| 647 | #endif | |||
| 648 | ||||
| 649 | #if EXTENSIVE_HEAP_PROFILING(0) | |||
| 650 | gc_dump_alloc_table(); | |||
| 651 | #endif | |||
| 652 | ||||
| 653 | return ret_ptr; | |||
| 654 | } | |||
| 655 | ||||
| 656 | /* | |||
| 657 | void *gc_alloc(mp_uint_t n_bytes) { | |||
| 658 | return _gc_alloc(n_bytes, false); | |||
| 659 | } | |||
| 660 | ||||
| 661 | void *gc_alloc_with_finaliser(mp_uint_t n_bytes) { | |||
| 662 | return _gc_alloc(n_bytes, true); | |||
| 663 | } | |||
| 664 | */ | |||
| 665 | ||||
| 666 | // force the freeing of a piece of memory | |||
| 667 | // TODO: freeing here does not call finaliser | |||
| 668 | void gc_free(void *ptr) { | |||
| 669 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 670 | if (MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth) > 0) { | |||
| 671 | // TODO how to deal with this error? | |||
| 672 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 673 | return; | |||
| 674 | } | |||
| 675 | ||||
| 676 | DEBUG_printf("gc_free(%p)\n", ptr)(void)0; | |||
| 677 | ||||
| 678 | if (ptr == NULL((void*)0)) { | |||
| 679 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 680 | } else { | |||
| 681 | // get the GC block number corresponding to this pointer | |||
| 682 | assert(VERIFY_PTR(ptr))((void) (0)); | |||
| 683 | size_t block = BLOCK_FROM_PTR(ptr)(((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof (mp_uint_t))))); | |||
| 684 | assert(ATB_GET_KIND(block) == AT_HEAD)((void) (0)); | |||
| 685 | ||||
| 686 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 687 | FTB_CLEAR(block)do { (mp_state_ctx.mem.gc_finaliser_table_start)[(block) / (8 )] &= (~(1 << ((block) & 7))); } while (0); | |||
| 688 | #endif | |||
| 689 | ||||
| 690 | // set the last_free pointer to this block if it's earlier in the heap | |||
| 691 | if (block / BLOCKS_PER_ATB(4) < MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index)) { | |||
| 692 | MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index) = block / BLOCKS_PER_ATB(4); | |||
| 693 | } | |||
| 694 | if (block / BLOCKS_PER_ATB(4) > MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index)) { | |||
| 695 | MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index) = block / BLOCKS_PER_ATB(4); | |||
| 696 | } | |||
| 697 | ||||
| 698 | // free head and all of its tail blocks | |||
| 699 | #ifdef LOG_HEAP_ACTIVITY | |||
| 700 | gc_log_change(block, 0); | |||
| 701 | #endif | |||
| 702 | do { | |||
| 703 | ATB_ANY_TO_FREE(block)do { (mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] &= (~((3) << (2 * ((block) & ((4) - 1))))); } while ( 0); | |||
| 704 | block += 1; | |||
| 705 | } while (ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3) == AT_TAIL(2)); | |||
| 706 | ||||
| 707 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 708 | ||||
| 709 | #if EXTENSIVE_HEAP_PROFILING(0) | |||
| 710 | gc_dump_alloc_table(); | |||
| 711 | #endif | |||
| 712 | } | |||
| 713 | } | |||
| 714 | ||||
| 715 | size_t gc_nbytes(const void *ptr) { | |||
| 716 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 717 | if (VERIFY_PTR(ptr)( ((uintptr_t)(ptr) & (((4 * (sizeof(mp_uint_t)))) - 1)) == 0 && ptr >= (void*)(mp_state_ctx.mem.gc_pool_start ) && ptr < (void*)(mp_state_ctx.mem.gc_pool_end) )) { | |||
| 718 | size_t block = BLOCK_FROM_PTR(ptr)(((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof (mp_uint_t))))); | |||
| 719 | if (ATB_GET_KIND(block)(((mp_state_ctx.mem.gc_alloc_table_start)[(block) / (4)] >> (2 * ((block) & ((4) - 1)))) & 3) == AT_HEAD(1)) { | |||
| 720 | // work out number of consecutive blocks in the chain starting with this on | |||
| 721 | size_t n_blocks = 0; | |||
| 722 | do { | |||
| 723 | n_blocks += 1; | |||
| 724 | } while (ATB_GET_KIND(block + n_blocks)(((mp_state_ctx.mem.gc_alloc_table_start)[(block + n_blocks) / (4)] >> (2 * ((block + n_blocks) & ((4) - 1)))) & 3) == AT_TAIL(2)); | |||
| 725 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 726 | return n_blocks * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))); | |||
| 727 | } | |||
| 728 | } | |||
| 729 | ||||
| 730 | // invalid pointer | |||
| 731 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 732 | return 0; | |||
| 733 | } | |||
| 734 | ||||
| 735 | bool_Bool gc_has_finaliser(const void *ptr) { | |||
| 736 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 737 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 738 | if (VERIFY_PTR(ptr)( ((uintptr_t)(ptr) & (((4 * (sizeof(mp_uint_t)))) - 1)) == 0 && ptr >= (void*)(mp_state_ctx.mem.gc_pool_start ) && ptr < (void*)(mp_state_ctx.mem.gc_pool_end) )) { | |||
| 739 | bool_Bool has_finaliser = FTB_GET(BLOCK_FROM_PTR(ptr))(((mp_state_ctx.mem.gc_finaliser_table_start)[((((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof(mp_uint_t )))))) / (8)] >> (((((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start )) / ((4 * (sizeof(mp_uint_t)))))) & 7)) & 1); | |||
| 740 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 741 | return has_finaliser; | |||
| 742 | } | |||
| 743 | ||||
| 744 | // invalid pointer | |||
| 745 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 746 | #else | |||
| 747 | (void) ptr; | |||
| 748 | #endif | |||
| 749 | return false0; | |||
| 750 | } | |||
| 751 | ||||
| 752 | void *gc_make_long_lived(void *old_ptr) { | |||
| 753 | // If its already in the long lived section then don't bother moving it. | |||
| 754 | if (old_ptr >= MP_STATE_MEM(gc_lowest_long_lived_ptr)(mp_state_ctx.mem.gc_lowest_long_lived_ptr)) { | |||
| 755 | return old_ptr; | |||
| 756 | } | |||
| 757 | size_t n_bytes = gc_nbytes(old_ptr); | |||
| 758 | if (n_bytes == 0) { | |||
| 759 | return old_ptr; | |||
| 760 | } | |||
| 761 | bool_Bool has_finaliser = gc_has_finaliser(old_ptr); | |||
| 762 | ||||
| 763 | // Try and find a new area in the long lived section to copy the memory to. | |||
| 764 | void* new_ptr = gc_alloc(n_bytes, has_finaliser, true1); | |||
| 765 | if (new_ptr == NULL((void*)0)) { | |||
| 766 | return old_ptr; | |||
| 767 | } else if (old_ptr > new_ptr) { | |||
| 768 | // Return the old pointer if the new one is lower in the heap and free the new space. | |||
| 769 | gc_free(new_ptr); | |||
| 770 | return old_ptr; | |||
| 771 | } | |||
| 772 | // We copy everything over and let the garbage collection process delete the old copy. That way | |||
| 773 | // we ensure we don't delete memory that has a second reference. (Though if there is we may | |||
| 774 | // confuse things when its mutable.) | |||
| 775 | memcpy(new_ptr, old_ptr, n_bytes); | |||
| 776 | return new_ptr; | |||
| 777 | } | |||
| 778 | ||||
| 779 | #if 0 | |||
| 780 | // old, simple realloc that didn't expand memory in place | |||
| 781 | void *gc_realloc(void *ptr, mp_uint_t n_bytes) { | |||
| 782 | mp_uint_t n_existing = gc_nbytes(ptr); | |||
| 783 | if (n_bytes <= n_existing) { | |||
| 784 | return ptr; | |||
| 785 | } else { | |||
| 786 | bool_Bool has_finaliser; | |||
| 787 | if (ptr == NULL((void*)0)) { | |||
| 788 | has_finaliser = false0; | |||
| 789 | } else { | |||
| 790 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 791 | has_finaliser = FTB_GET(BLOCK_FROM_PTR((mp_uint_t)ptr))(((mp_state_ctx.mem.gc_finaliser_table_start)[((((byte*)((mp_uint_t )ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof(mp_uint_t )))))) / (8)] >> (((((byte*)((mp_uint_t)ptr) - (mp_state_ctx .mem.gc_pool_start)) / ((4 * (sizeof(mp_uint_t)))))) & 7) ) & 1); | |||
| 792 | #else | |||
| 793 | has_finaliser = false0; | |||
| 794 | #endif | |||
| 795 | } | |||
| 796 | void *ptr2 = gc_alloc(n_bytes, has_finaliser); | |||
| 797 | if (ptr2 == NULL((void*)0)) { | |||
| 798 | return ptr2; | |||
| 799 | } | |||
| 800 | memcpy(ptr2, ptr, n_existing); | |||
| 801 | gc_free(ptr); | |||
| 802 | return ptr2; | |||
| 803 | } | |||
| 804 | } | |||
| 805 | ||||
| 806 | #else // Alternative gc_realloc impl | |||
| 807 | ||||
| 808 | void *gc_realloc(void *ptr_in, size_t n_bytes, bool_Bool allow_move) { | |||
| 809 | // check for pure allocation | |||
| 810 | if (ptr_in == NULL((void*)0)) { | |||
| 811 | return gc_alloc(n_bytes, false0, false0); | |||
| 812 | } | |||
| 813 | ||||
| 814 | // check for pure free | |||
| 815 | if (n_bytes == 0) { | |||
| 816 | gc_free(ptr_in); | |||
| 817 | return NULL((void*)0); | |||
| 818 | } | |||
| 819 | ||||
| 820 | void *ptr = ptr_in; | |||
| 821 | ||||
| 822 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 823 | ||||
| 824 | if (MP_STATE_MEM(gc_lock_depth)(mp_state_ctx.mem.gc_lock_depth) > 0) { | |||
| 825 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 826 | return NULL((void*)0); | |||
| 827 | } | |||
| 828 | ||||
| 829 | // get the GC block number corresponding to this pointer | |||
| 830 | assert(VERIFY_PTR(ptr))((void) (0)); | |||
| 831 | size_t block = BLOCK_FROM_PTR(ptr)(((byte*)(ptr) - (mp_state_ctx.mem.gc_pool_start)) / ((4 * (sizeof (mp_uint_t))))); | |||
| 832 | assert(ATB_GET_KIND(block) == AT_HEAD)((void) (0)); | |||
| 833 | ||||
| 834 | // compute number of new blocks that are requested | |||
| 835 | size_t new_blocks = (n_bytes + BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) - 1) / BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))); | |||
| 836 | ||||
| 837 | // Get the total number of consecutive blocks that are already allocated to | |||
| 838 | // this chunk of memory, and then count the number of free blocks following | |||
| 839 | // it. Stop if we reach the end of the heap, or if we find enough extra | |||
| 840 | // free blocks to satisfy the realloc. Note that we need to compute the | |||
| 841 | // total size of the existing memory chunk so we can correctly and | |||
| 842 | // efficiently shrink it (see below for shrinking code). | |||
| 843 | size_t n_free = 0; | |||
| 844 | size_t n_blocks = 1; // counting HEAD block | |||
| 845 | size_t max_block = MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4); | |||
| 846 | for (size_t bl = block + n_blocks; bl < max_block; bl++) { | |||
| 847 | byte block_type = ATB_GET_KIND(bl)(((mp_state_ctx.mem.gc_alloc_table_start)[(bl) / (4)] >> (2 * ((bl) & ((4) - 1)))) & 3); | |||
| 848 | if (block_type == AT_TAIL(2)) { | |||
| 849 | n_blocks++; | |||
| 850 | continue; | |||
| 851 | } | |||
| 852 | if (block_type == AT_FREE(0)) { | |||
| 853 | n_free++; | |||
| 854 | if (n_blocks + n_free >= new_blocks) { | |||
| 855 | // stop as soon as we find enough blocks for n_bytes | |||
| 856 | break; | |||
| 857 | } | |||
| 858 | continue; | |||
| 859 | } | |||
| 860 | break; | |||
| 861 | } | |||
| 862 | ||||
| 863 | // return original ptr if it already has the requested number of blocks | |||
| 864 | if (new_blocks == n_blocks) { | |||
| 865 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 866 | return ptr_in; | |||
| 867 | } | |||
| 868 | ||||
| 869 | // check if we can shrink the allocated area | |||
| 870 | if (new_blocks < n_blocks) { | |||
| 871 | // free unneeded tail blocks | |||
| 872 | for (size_t bl = block + new_blocks, count = n_blocks - new_blocks; count > 0; bl++, count--) { | |||
| 873 | ATB_ANY_TO_FREE(bl)do { (mp_state_ctx.mem.gc_alloc_table_start)[(bl) / (4)] &= (~((3) << (2 * ((bl) & ((4) - 1))))); } while (0); | |||
| 874 | } | |||
| 875 | ||||
| 876 | // set the last_free pointer to end of this block if it's earlier in the heap | |||
| 877 | if ((block + new_blocks) / BLOCKS_PER_ATB(4) < MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index)) { | |||
| 878 | MP_STATE_MEM(gc_first_free_atb_index)(mp_state_ctx.mem.gc_first_free_atb_index) = (block + new_blocks) / BLOCKS_PER_ATB(4); | |||
| 879 | } | |||
| 880 | if ((block + new_blocks) / BLOCKS_PER_ATB(4) > MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index)) { | |||
| 881 | MP_STATE_MEM(gc_last_free_atb_index)(mp_state_ctx.mem.gc_last_free_atb_index) = (block + new_blocks) / BLOCKS_PER_ATB(4); | |||
| 882 | } | |||
| 883 | ||||
| 884 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 885 | ||||
| 886 | #if EXTENSIVE_HEAP_PROFILING(0) | |||
| 887 | gc_dump_alloc_table(); | |||
| 888 | #endif | |||
| 889 | ||||
| 890 | #ifdef LOG_HEAP_ACTIVITY | |||
| 891 | gc_log_change(block, new_blocks); | |||
| 892 | #endif | |||
| 893 | ||||
| 894 | return ptr_in; | |||
| 895 | } | |||
| 896 | ||||
| 897 | // check if we can expand in place | |||
| 898 | if (new_blocks <= n_blocks + n_free) { | |||
| 899 | // mark few more blocks as used tail | |||
| 900 | for (size_t bl = block + n_blocks; bl < block + new_blocks; bl++) { | |||
| 901 | assert(ATB_GET_KIND(bl) == AT_FREE)((void) (0)); | |||
| 902 | ATB_FREE_TO_TAIL(bl)do { (mp_state_ctx.mem.gc_alloc_table_start)[(bl) / (4)] |= ( (2) << (2 * ((bl) & ((4) - 1)))); } while (0); | |||
| 903 | } | |||
| 904 | ||||
| 905 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 906 | ||||
| 907 | #if MICROPY_GC_CONSERVATIVE_CLEAR((1)) | |||
| 908 | // be conservative and zero out all the newly allocated blocks | |||
| 909 | memset((byte*)ptr_in + n_blocks * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))), 0, (new_blocks - n_blocks) * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 910 | #else | |||
| 911 | // zero out the additional bytes of the newly allocated blocks (see comment above in gc_alloc) | |||
| 912 | memset((byte*)ptr_in + n_bytes, 0, new_blocks * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) - n_bytes); | |||
| 913 | #endif | |||
| 914 | ||||
| 915 | #if EXTENSIVE_HEAP_PROFILING(0) | |||
| 916 | gc_dump_alloc_table(); | |||
| 917 | #endif | |||
| 918 | ||||
| 919 | #ifdef LOG_HEAP_ACTIVITY | |||
| 920 | gc_log_change(block, new_blocks); | |||
| 921 | #endif | |||
| 922 | ||||
| 923 | return ptr_in; | |||
| 924 | } | |||
| 925 | ||||
| 926 | #if MICROPY_ENABLE_FINALISER(1) | |||
| 927 | bool_Bool ftb_state = FTB_GET(block)(((mp_state_ctx.mem.gc_finaliser_table_start)[(block) / (8)] >> ((block) & 7)) & 1); | |||
| 928 | #else | |||
| 929 | bool_Bool ftb_state = false0; | |||
| 930 | #endif | |||
| 931 | ||||
| 932 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 933 | ||||
| 934 | if (!allow_move) { | |||
| 935 | // not allowed to move memory block so return failure | |||
| 936 | return NULL((void*)0); | |||
| 937 | } | |||
| 938 | ||||
| 939 | // can't resize inplace; try to find a new contiguous chain | |||
| 940 | void *ptr_out = gc_alloc(n_bytes, ftb_state, false0); | |||
| 941 | ||||
| 942 | // check that the alloc succeeded | |||
| 943 | if (ptr_out == NULL((void*)0)) { | |||
| 944 | return NULL((void*)0); | |||
| 945 | } | |||
| 946 | ||||
| 947 | DEBUG_printf("gc_realloc(%p -> %p)\n", ptr_in, ptr_out)(void)0; | |||
| 948 | memcpy(ptr_out, ptr_in, n_blocks * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 949 | gc_free(ptr_in); | |||
| 950 | return ptr_out; | |||
| 951 | } | |||
| 952 | #endif // Alternative gc_realloc impl | |||
| 953 | ||||
| 954 | bool_Bool gc_never_free(void *ptr) { | |||
| 955 | // Check to make sure the pointer is on the heap in the first place. | |||
| 956 | if (gc_nbytes(ptr) == 0) { | |||
| ||||
| 957 | return false0; | |||
| 958 | } | |||
| 959 | // Pointers are stored in a linked list where each block is BYTES_PER_BLOCK long and the first | |||
| 960 | // pointer is the next block of pointers. | |||
| 961 | void ** current_reference_block = MP_STATE_MEM(permanent_pointers)(mp_state_ctx.mem.permanent_pointers); | |||
| 962 | while (current_reference_block != NULL((void*)0)) { | |||
| 963 | for (size_t i = 1; i < BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))) / sizeof(void*); i++) { | |||
| 964 | if (current_reference_block[i] == NULL((void*)0)) { | |||
| 965 | current_reference_block[i] = ptr; | |||
| 966 | return true1; | |||
| 967 | } | |||
| 968 | } | |||
| 969 | current_reference_block = current_reference_block[0]; | |||
| 970 | } | |||
| 971 | void** next_block = gc_alloc(BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t)))), false0, true1); | |||
| 972 | if (next_block == NULL((void*)0)) { | |||
| 973 | return false0; | |||
| 974 | } | |||
| 975 | if (MP_STATE_MEM(permanent_pointers)(mp_state_ctx.mem.permanent_pointers) == NULL((void*)0)) { | |||
| 976 | MP_STATE_MEM(permanent_pointers)(mp_state_ctx.mem.permanent_pointers) = next_block; | |||
| 977 | } else { | |||
| 978 | current_reference_block[0] = next_block; | |||
| ||||
| 979 | } | |||
| 980 | next_block[1] = ptr; | |||
| 981 | return true1; | |||
| 982 | } | |||
| 983 | ||||
| 984 | void gc_dump_info(void) { | |||
| 985 | gc_info_t info; | |||
| 986 | gc_info(&info); | |||
| 987 | mp_printf(&mp_plat_print, "GC: total: %u, used: %u, free: %u\n", | |||
| 988 | (uint)info.total, (uint)info.used, (uint)info.free); | |||
| 989 | mp_printf(&mp_plat_print, " No. of 1-blocks: %u, 2-blocks: %u, max blk sz: %u, max free sz: %u\n", | |||
| 990 | (uint)info.num_1block, (uint)info.num_2block, (uint)info.max_block, (uint)info.max_free); | |||
| 991 | } | |||
| 992 | ||||
| 993 | void gc_dump_alloc_table(void) { | |||
| 994 | GC_ENTER()mp_thread_mutex_lock(&(mp_state_ctx.mem.gc_mutex), 1); | |||
| 995 | static const size_t DUMP_BYTES_PER_LINE = 64; | |||
| 996 | #if !EXTENSIVE_HEAP_PROFILING(0) | |||
| 997 | // When comparing heap output we don't want to print the starting | |||
| 998 | // pointer of the heap because it changes from run to run. | |||
| 999 | mp_printf(&mp_plat_print, "GC memory layout; from %p:", MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start)); | |||
| 1000 | #endif | |||
| 1001 | for (size_t bl = 0; bl < MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4); bl++) { | |||
| 1002 | if (bl % DUMP_BYTES_PER_LINE == 0) { | |||
| 1003 | // a new line of blocks | |||
| 1004 | { | |||
| 1005 | // check if this line contains only free blocks | |||
| 1006 | size_t bl2 = bl; | |||
| 1007 | while (bl2 < MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4) && ATB_GET_KIND(bl2)(((mp_state_ctx.mem.gc_alloc_table_start)[(bl2) / (4)] >> (2 * ((bl2) & ((4) - 1)))) & 3) == AT_FREE(0)) { | |||
| 1008 | bl2++; | |||
| 1009 | } | |||
| 1010 | if (bl2 - bl >= 2 * DUMP_BYTES_PER_LINE) { | |||
| 1011 | // there are at least 2 lines containing only free blocks, so abbreviate their printing | |||
| 1012 | mp_printf(&mp_plat_print, "\n (%u lines all free)", (uint)(bl2 - bl) / DUMP_BYTES_PER_LINE); | |||
| 1013 | bl = bl2 & (~(DUMP_BYTES_PER_LINE - 1)); | |||
| 1014 | if (bl >= MP_STATE_MEM(gc_alloc_table_byte_len)(mp_state_ctx.mem.gc_alloc_table_byte_len) * BLOCKS_PER_ATB(4)) { | |||
| 1015 | // got to end of heap | |||
| 1016 | break; | |||
| 1017 | } | |||
| 1018 | } | |||
| 1019 | } | |||
| 1020 | // print header for new line of blocks | |||
| 1021 | // (the cast to uint32_t is for 16-bit ports) | |||
| 1022 | //mp_printf(&mp_plat_print, "\n%05x: ", (uint)(PTR_FROM_BLOCK(bl) & (uint32_t)0xfffff)); | |||
| 1023 | mp_printf(&mp_plat_print, "\n%05x: ", (uint)((bl * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))) & (uint32_t)0xfffff)); | |||
| 1024 | } | |||
| 1025 | int c = ' '; | |||
| 1026 | switch (ATB_GET_KIND(bl)(((mp_state_ctx.mem.gc_alloc_table_start)[(bl) / (4)] >> (2 * ((bl) & ((4) - 1)))) & 3)) { | |||
| 1027 | case AT_FREE(0): c = '.'; break; | |||
| 1028 | /* this prints out if the object is reachable from BSS or STACK (for unix only) | |||
| 1029 | case AT_HEAD: { | |||
| 1030 | c = 'h'; | |||
| 1031 | void **ptrs = (void**)(void*)&mp_state_ctx; | |||
| 1032 | mp_uint_t len = offsetof(mp_state_ctx_t, vm.stack_top) / sizeof(mp_uint_t); | |||
| 1033 | for (mp_uint_t i = 0; i < len; i++) { | |||
| 1034 | mp_uint_t ptr = (mp_uint_t)ptrs[i]; | |||
| 1035 | if (VERIFY_PTR(ptr) && BLOCK_FROM_PTR(ptr) == bl) { | |||
| 1036 | c = 'B'; | |||
| 1037 | break; | |||
| 1038 | } | |||
| 1039 | } | |||
| 1040 | if (c == 'h') { | |||
| 1041 | ptrs = (void**)&c; | |||
| 1042 | len = ((mp_uint_t)MP_STATE_THREAD(stack_top) - (mp_uint_t)&c) / sizeof(mp_uint_t); | |||
| 1043 | for (mp_uint_t i = 0; i < len; i++) { | |||
| 1044 | mp_uint_t ptr = (mp_uint_t)ptrs[i]; | |||
| 1045 | if (VERIFY_PTR(ptr) && BLOCK_FROM_PTR(ptr) == bl) { | |||
| 1046 | c = 'S'; | |||
| 1047 | break; | |||
| 1048 | } | |||
| 1049 | } | |||
| 1050 | } | |||
| 1051 | break; | |||
| 1052 | } | |||
| 1053 | */ | |||
| 1054 | /* this prints the uPy object type of the head block */ | |||
| 1055 | case AT_HEAD(1): { | |||
| 1056 | #pragma GCC diagnostic push | |||
| 1057 | #pragma GCC diagnostic ignored "-Wcast-align" | |||
| 1058 | void **ptr = (void**)(MP_STATE_MEM(gc_pool_start)(mp_state_ctx.mem.gc_pool_start) + bl * BYTES_PER_BLOCK((4 * (sizeof(mp_uint_t))))); | |||
| 1059 | #pragma GCC diagnostic pop | |||
| 1060 | if (*ptr == &mp_type_tuple) { c = 'T'; } | |||
| 1061 | else if (*ptr == &mp_type_list) { c = 'L'; } | |||
| 1062 | else if (*ptr == &mp_type_dict) { c = 'D'; } | |||
| 1063 | else if (*ptr == &mp_type_str || *ptr == &mp_type_bytes) { c = 'S'; } | |||
| 1064 | #if MICROPY_PY_BUILTINS_BYTEARRAY(1) | |||
| 1065 | else if (*ptr == &mp_type_bytearray) { c = 'A'; } | |||
| 1066 | #endif | |||
| 1067 | #if MICROPY_PY_ARRAY(1) | |||
| 1068 | else if (*ptr == &mp_type_array) { c = 'A'; } | |||
| 1069 | #endif | |||
| 1070 | #if MICROPY_PY_BUILTINS_FLOAT(1) | |||
| 1071 | else if (*ptr == &mp_type_float) { c = 'F'; } | |||
| 1072 | #endif | |||
| 1073 | else if (*ptr == &mp_type_fun_bc) { c = 'B'; } | |||
| 1074 | else if (*ptr == &mp_type_module) { c = 'M'; } | |||
| 1075 | else { | |||
| 1076 | c = 'h'; | |||
| 1077 | #if 0 | |||
| 1078 | // This code prints "Q" for qstr-pool data, and "q" for qstr-str | |||
| 1079 | // data. It can be useful to see how qstrs are being allocated, | |||
| 1080 | // but is disabled by default because it is very slow. | |||
| 1081 | for (qstr_pool_t *pool = MP_STATE_VM(last_pool)(mp_state_ctx.vm.last_pool); c == 'h' && pool != NULL((void*)0); pool = pool->prev) { | |||
| 1082 | if ((qstr_pool_t*)ptr == pool) { | |||
| 1083 | c = 'Q'; | |||
| 1084 | break; | |||
| 1085 | } | |||
| 1086 | for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) { | |||
| 1087 | if ((const byte*)ptr == *q) { | |||
| 1088 | c = 'q'; | |||
| 1089 | break; | |||
| 1090 | } | |||
| 1091 | } | |||
| 1092 | } | |||
| 1093 | #endif | |||
| 1094 | } | |||
| 1095 | break; | |||
| 1096 | } | |||
| 1097 | case AT_TAIL(2): c = '='; break; | |||
| 1098 | case AT_MARK(3): c = 'm'; break; | |||
| 1099 | } | |||
| 1100 | mp_printf(&mp_plat_print, "%c", c); | |||
| 1101 | } | |||
| 1102 | mp_print_str(&mp_plat_print, "\n"); | |||
| 1103 | GC_EXIT()mp_thread_mutex_unlock(&(mp_state_ctx.mem.gc_mutex)); | |||
| 1104 | } | |||
| 1105 | ||||
| 1106 | #if DEBUG_PRINT(0) | |||
| 1107 | void gc_test(void) { | |||
| 1108 | mp_uint_t len = 500; | |||
| 1109 | mp_uint_t *heap = malloc(len); | |||
| 1110 | gc_init(heap, heap + len / sizeof(mp_uint_t)); | |||
| 1111 | void *ptrs[100]; | |||
| 1112 | { | |||
| 1113 | mp_uint_t **p = gc_alloc(16, false0); | |||
| 1114 | p[0] = gc_alloc(64, false0); | |||
| 1115 | p[1] = gc_alloc(1, false0); | |||
| 1116 | p[2] = gc_alloc(1, false0); | |||
| 1117 | p[3] = gc_alloc(1, false0); | |||
| 1118 | mp_uint_t ***p2 = gc_alloc(16, false0); | |||
| 1119 | p2[0] = p; | |||
| 1120 | p2[1] = p; | |||
| 1121 | ptrs[0] = p2; | |||
| 1122 | } | |||
| 1123 | for (int i = 0; i < 25; i+=2) { | |||
| 1124 | mp_uint_t *p = gc_alloc(i, false0); | |||
| 1125 | printf("p=%p\n", p); | |||
| 1126 | if (i & 3) { | |||
| 1127 | //ptrs[i] = p; | |||
| 1128 | } | |||
| 1129 | } | |||
| 1130 | ||||
| 1131 | printf("Before GC:\n"); | |||
| 1132 | gc_dump_alloc_table(); | |||
| 1133 | printf("Starting GC...\n"); | |||
| 1134 | gc_collect_start(); | |||
| 1135 | gc_collect_root(ptrs, sizeof(ptrs) / sizeof(void*)); | |||
| 1136 | gc_collect_end(); | |||
| 1137 | printf("After GC:\n"); | |||
| 1138 | gc_dump_alloc_table(); | |||
| 1139 | } | |||
| 1140 | #endif | |||
| 1141 | ||||
| 1142 | #endif // MICROPY_ENABLE_GC |