From 4d43b0df1c39c72c18f4d32bf9d22259a5b5a0a5 Mon Sep 17 00:00:00 2001 From: X-Ryl669 Date: Mon, 19 Sep 2022 15:21:47 +0200 Subject: [PATCH 1/5] Multiple heap TLSF --- tlsf.c | 155 +++++++++++++++++++++++++++++++------------------- tlsf.h | 9 +-- tlsf_common.h | 60 ++++++++++--------- 3 files changed, 135 insertions(+), 89 deletions(-) diff --git a/tlsf.c b/tlsf.c index 4255d10..6ddf10f 100644 --- a/tlsf.c +++ b/tlsf.c @@ -35,6 +35,7 @@ ** ffs/fls return 1-32 by default, returning 0 for error. */ + /* ** Detect whether or not we are building for a 32- or 64-bit (LP/LLP) ** architecture. There is no reliable portable method at compile-time. @@ -215,12 +216,6 @@ tlsf_static_assert(sizeof(int) * CHAR_BIT == 32); tlsf_static_assert(sizeof(size_t) * CHAR_BIT >= 32); tlsf_static_assert(sizeof(size_t) * CHAR_BIT <= 64); -/* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */ -tlsf_static_assert(sizeof(unsigned int) * CHAR_BIT >= SL_INDEX_COUNT); - -/* Ensure we've properly tuned our sizes. */ -tlsf_static_assert(ALIGN_SIZE == SMALL_BLOCK_SIZE / SL_INDEX_COUNT); - static inline __attribute__((always_inline)) size_t align_up(size_t x, size_t align) { tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); @@ -229,7 +224,7 @@ static inline __attribute__((always_inline)) size_t align_up(size_t x, size_t al static inline __attribute__((always_inline)) size_t align_down(size_t x, size_t align) { - tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two "); return x - (x & (align - 1)); } @@ -245,7 +240,7 @@ static inline __attribute__((always_inline)) void* align_ptr(const void* ptr, si ** Adjust an allocation size to be aligned to word size, and no smaller ** than internal minimum. */ -static inline __attribute__((always_inline)) size_t adjust_request_size(size_t size, size_t align) +static inline __attribute__((always_inline)) size_t adjust_request_size(tlsf_t tlsf, size_t size, size_t align) { size_t adjust = 0; if (size) @@ -253,7 +248,7 @@ static inline __attribute__((always_inline)) size_t adjust_request_size(size_t s const size_t aligned = align_up(size, align); /* aligned sized must not exceed block_size_max or we'll go out of bounds on sl_bitmap */ - if (aligned < block_size_max) + if (aligned < tlsf_block_size_max(tlsf)) { adjust = tlsf_max(aligned, block_size_min); } @@ -266,34 +261,34 @@ static inline __attribute__((always_inline)) size_t adjust_request_size(size_t s ** the documentation found in the white paper. */ -static inline __attribute__((always_inline)) void mapping_insert(size_t size, int* fli, int* sli) +static inline __attribute__((always_inline)) void mapping_insert(control_t* control, size_t size, int* fli, int* sli) { int fl, sl; - if (size < SMALL_BLOCK_SIZE) + if (size < control->small_block_size) { /* Store small blocks in first list. */ fl = 0; - sl = tlsf_cast(int, size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT); + sl = tlsf_cast(int, size) / (control->small_block_size / control->sl_index_count); } else { fl = tlsf_fls_sizet(size); - sl = tlsf_cast(int, size >> (fl - SL_INDEX_COUNT_LOG2)) ^ (1 << SL_INDEX_COUNT_LOG2); - fl -= (FL_INDEX_SHIFT - 1); + sl = tlsf_cast(int, size >> (fl - control->sl_index_count_log2)) ^ (1 << control->sl_index_count_log2); + fl -= (control->fl_index_shift - 1); } *fli = fl; *sli = sl; } /* This version rounds up to the next block size (for allocations) */ -static inline __attribute__((always_inline)) void mapping_search(size_t size, int* fli, int* sli) +static inline __attribute__((always_inline)) void mapping_search(control_t* control, size_t size, int* fli, int* sli) { - if (size >= SMALL_BLOCK_SIZE) + if (size >= control->small_block_size) { - const size_t round = (1 << (tlsf_fls_sizet(size) - SL_INDEX_COUNT_LOG2)) - 1; + const size_t round = (1 << (tlsf_fls_sizet(size) - control->sl_index_count_log2)) - 1; size += round; } - mapping_insert(size, fli, sli); + mapping_insert(control, size, fli, sli); } static inline __attribute__((always_inline)) block_header_t* search_suitable_block(control_t* control, int* fli, int* sli) @@ -325,7 +320,7 @@ static inline __attribute__((always_inline)) block_header_t* search_suitable_blo *sli = sl; /* Return the first block in the free list. */ - return control->blocks[fl][sl]; + return control->blocks[fl * control->sl_index_count + sl]; } /* Remove a free block from the free list.*/ @@ -339,9 +334,9 @@ static inline __attribute__((always_inline)) void remove_free_block(control_t* c prev->next_free = next; /* If this block is the head of the free list, set new head. */ - if (control->blocks[fl][sl] == block) + if (control->blocks[fl * control->sl_index_count + sl] == block) { - control->blocks[fl][sl] = next; + control->blocks[fl * control->sl_index_count + sl] = next; /* If the new head is null, clear the bitmap. */ if (next == &control->block_null) @@ -360,20 +355,19 @@ static inline __attribute__((always_inline)) void remove_free_block(control_t* c /* Insert a free block into the free block list. */ static inline __attribute__((always_inline)) void insert_free_block(control_t* control, block_header_t* block, int fl, int sl) { - block_header_t* current = control->blocks[fl][sl]; + block_header_t* current = control->blocks[fl * control->sl_index_count + sl]; tlsf_assert(current && "free list cannot have a null entry"); tlsf_assert(block && "cannot insert a null entry into the free list"); block->next_free = current; block->prev_free = &control->block_null; current->prev_free = block; - tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE) - && "block not aligned properly"); + tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE) && "block not aligned properly"); /* ** Insert the new block at the head of the list, and mark the first- ** and second-level bitmaps appropriately. */ - control->blocks[fl][sl] = block; + control->blocks[fl * control->sl_index_count + sl] = block; control->fl_bitmap |= (1U << fl); control->sl_bitmap[fl] |= (1U << sl); } @@ -382,7 +376,7 @@ static inline __attribute__((always_inline)) void insert_free_block(control_t* c static inline __attribute__((always_inline)) void block_remove(control_t* control, block_header_t* block) { int fl, sl; - mapping_insert(block_size(block), &fl, &sl); + mapping_insert(control, block_size(block), &fl, &sl); remove_free_block(control, block, fl, sl); } @@ -390,7 +384,7 @@ static inline __attribute__((always_inline)) void block_remove(control_t* contro static inline __attribute__((always_inline)) void block_insert(control_t* control, block_header_t* block) { int fl, sl; - mapping_insert(block_size(block), &fl, &sl); + mapping_insert(control, block_size(block), &fl, &sl); insert_free_block(control, block, fl, sl); } @@ -413,8 +407,7 @@ static inline __attribute__((always_inline)) block_header_t* block_split(block_h * This field is NOT part of the size, so it has to be substracted from the calculation. */ const size_t remain_size = block_size(block) - (size + block_header_overhead); - tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE) - && "remaining block not aligned properly"); + tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE) && "remaining block not aligned properly"); tlsf_assert(block_size(block) == remain_size + size + block_header_overhead); block_set_size(remaining, remain_size); @@ -553,7 +546,7 @@ static inline __attribute__((always_inline)) block_header_t* block_locate_free(c if (size) { - mapping_search(size, &fl, &sl); + mapping_search(control, size, &fl, &sl); /* ** mapping_search can futz with the size, so for excessively large sizes it can sometimes wind up @@ -561,7 +554,7 @@ static inline __attribute__((always_inline)) block_header_t* block_locate_free(c ** So, we protect against that here, since this is the only callsite of mapping_search. ** Note that we don't need to check sl, since it comes from a modulo operation that guarantees it's always in range. */ - if (fl < FL_INDEX_COUNT) + if (fl < control->fl_index_count) { block = search_suitable_block(control, &fl, &sl); } @@ -590,20 +583,44 @@ static inline __attribute__((always_inline)) void* block_prepare_used(control_t* } /* Clear structure and point all empty lists at the null block. */ -static void control_construct(control_t* control) +static void control_construct(control_t* control, size_t bytes) { int i, j; control->block_null.next_free = &control->block_null; control->block_null.prev_free = &control->block_null; + /* Find the closest power of two for first layer */ + i = (bytes - 1) / (16 * 1024); + control->fl_index_max = FL_INDEX_MAX_MIN + sizeof(i) * 8 - __builtin_clz(i); + + /* Adapt second layer to the pool */ + if (bytes <= 16 * 1024) control->sl_index_count_log2 = 3; + else if (bytes <= 256 * 1024) control->sl_index_count_log2 = 4; + else control->sl_index_count_log2 = 5; + + control->fl_index_shift = (control->sl_index_count_log2 + ALIGN_SIZE_LOG2); + control->sl_index_count = 1 << control->sl_index_count_log2; + control->fl_index_count = control->fl_index_max - control->fl_index_shift + 1; + control->small_block_size = 1 << control->fl_index_shift; + control->fl_bitmap = 0; - for (i = 0; i < FL_INDEX_COUNT; ++i) + control->sl_bitmap = align_ptr(control + 1, sizeof(*control->sl_bitmap)); + control->blocks = align_ptr(control->sl_bitmap + control->fl_index_count, sizeof(*control->blocks)); + control->size = (void*) (control->blocks + control->sl_index_count * control->fl_index_count) - (void*) control; + + /* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */ + tlsf_assert(sizeof(unsigned int) * CHAR_BIT >= control->sl_index_count); //CHAR_BIT less than sl_index_count"); + + /* Ensure we've properly tuned our sizes. */ + tlsf_assert(ALIGN_SIZE == control->small_block_size / control->sl_index_count); //ALIGN_SIZE does not match"); + + for (i = 0; i < control->fl_index_count; ++i) { control->sl_bitmap[i] = 0; - for (j = 0; j < SL_INDEX_COUNT; ++j) + for (j = 0; j < control->sl_index_count; ++j) { - control->blocks[i][j] = &control->block_null; + control->blocks[i * control->sl_index_count + j] = &control->block_null; } } } @@ -645,14 +662,14 @@ int tlsf_check(tlsf_t tlsf) int status = 0; /* Check that the free lists and bitmaps are accurate. */ - for (i = 0; i < FL_INDEX_COUNT; ++i) + for (i = 0; i < control->fl_index_count; ++i) { - for (j = 0; j < SL_INDEX_COUNT; ++j) + for (j = 0; j < control->sl_index_count; ++j) { const int fl_map = control->fl_bitmap & (1U << i); const int sl_list = control->sl_bitmap[i]; const int sl_map = sl_list & (1U << j); - const block_header_t* block = control->blocks[i][j]; + const block_header_t* block = control->blocks[i * control->sl_index_count + j]; /* Check that first- and second-level lists agree. */ if (!fl_map) @@ -680,7 +697,7 @@ int tlsf_check(tlsf_t tlsf) tlsf_insist(block_is_prev_free(block_next(block)) && "block should be free"); tlsf_insist(block_size(block) >= block_size_min && "block not minimum size"); - mapping_insert(block_size(block), &fli, &sli); + mapping_insert(control, block_size(block), &fli, &sli); tlsf_insist(fli == i && sli == j && "block size indexed in wrong list"); if (tlsf_check_hook != NULL) @@ -753,13 +770,36 @@ int tlsf_check_pool(pool_t pool) return integ.status; } +size_t tlsf_fit_size(tlsf_t tlsf, size_t size) +{ + /* because it's GoodFit, allocable size is one range lower */ + if (size) + { + size_t sl_interval; + control_t* control = tlsf_cast(control_t*, tlsf); + sl_interval = (1 << ((sizeof(size_t) * 8 - 1) - __builtin_clz(size))) / control->sl_index_count; + return size & ~(sl_interval - 1); + } + + return 0; +} + /* ** Size of the TLSF structures in a given memory block passed to ** tlsf_create, equal to the size of a control_t */ -size_t tlsf_size(void) +size_t tlsf_size(tlsf_t tlsf) { - return sizeof(control_t); + if (tlsf) + { + control_t* control = tlsf_cast(control_t*, tlsf); + return control->size; + } + + /* no tlsf, we'll just return a min size */ + return sizeof(control_t) + + sizeof(int) * SL_INDEX_COUNT_MIN + + sizeof(block_header_t*) * SL_INDEX_COUNT_MIN * FL_INDEX_COUNT_MIN; } size_t tlsf_align_size(void) @@ -772,9 +812,10 @@ size_t tlsf_block_size_min(void) return block_size_min; } -size_t tlsf_block_size_max(void) +size_t tlsf_block_size_max(tlsf_t tlsf) { - return block_size_max; + control_t* control = tlsf_cast(control_t*, tlsf); + return tlsf_cast(size_t, 1) << control->fl_index_max; } /* @@ -807,16 +848,16 @@ pool_t tlsf_add_pool(tlsf_t tlsf, void* mem, size_t bytes) return 0; } - if (pool_bytes < block_size_min || pool_bytes > block_size_max) + if (pool_bytes < block_size_min || pool_bytes > tlsf_block_size_max(tlsf)) { #if defined (TLSF_64BIT) printf("tlsf_add_pool: Memory size must be between 0x%x and 0x%x00 bytes.\n", (unsigned int)(pool_overhead + block_size_min), - (unsigned int)((pool_overhead + block_size_max) / 256)); + (unsigned int)((pool_overhead + tlsf_block_size_max(tlsf)) / 256)); #else printf("tlsf_add_pool: Memory size must be between %u and %u bytes.\n", (unsigned int)(pool_overhead + block_size_min), - (unsigned int)(pool_overhead + block_size_max)); + (unsigned int)(pool_overhead + tlsf_block_size_max(tlsf))); #endif return 0; } @@ -852,7 +893,7 @@ void tlsf_remove_pool(tlsf_t tlsf, pool_t pool) tlsf_assert(!block_is_free(block_next(block)) && "next block should not be free"); tlsf_assert(block_size(block_next(block)) == 0 && "next block size should be zero"); - mapping_insert(block_size(block), &fl, &sl); + mapping_insert(control, block_size(block), &fl, &sl); remove_free_block(control, block, fl, sl); } @@ -888,7 +929,7 @@ int test_ffs_fls() } #endif -tlsf_t tlsf_create(void* mem) +tlsf_t tlsf_create(void* mem, size_t max_bytes) { #if _DEBUG if (test_ffs_fls()) @@ -904,15 +945,15 @@ tlsf_t tlsf_create(void* mem) return 0; } - control_construct(tlsf_cast(control_t*, mem)); + control_construct(tlsf_cast(control_t*, mem), max_bytes); return tlsf_cast(tlsf_t, mem); } -tlsf_t tlsf_create_with_pool(void* mem, size_t bytes) +tlsf_t tlsf_create_with_pool(void* mem, size_t pool_bytes, size_t max_bytes) { - tlsf_t tlsf = tlsf_create(mem); - tlsf_add_pool(tlsf, (char*)mem + tlsf_size(), bytes - tlsf_size()); + tlsf_t tlsf = tlsf_create(mem, max_bytes ? max_bytes : pool_bytes); + tlsf_add_pool(tlsf, (char*)mem + tlsf_size(tlsf), pool_bytes - tlsf_size(tlsf)); return tlsf; } @@ -924,13 +965,13 @@ void tlsf_destroy(tlsf_t tlsf) pool_t tlsf_get_pool(tlsf_t tlsf) { - return tlsf_cast(pool_t, (char*)tlsf + tlsf_size()); + return tlsf_cast(pool_t, (char*)tlsf + tlsf_size(tlsf)); } void* tlsf_malloc(tlsf_t tlsf, size_t size) { control_t* control = tlsf_cast(control_t*, tlsf); - const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE); block_header_t* block = block_locate_free(control, adjust); return block_prepare_used(control, block, adjust); } @@ -961,7 +1002,7 @@ void* tlsf_malloc(tlsf_t tlsf, size_t size) void* tlsf_memalign_offs(tlsf_t tlsf, size_t align, size_t size, size_t data_offset) { control_t* control = tlsf_cast(control_t*, tlsf); - const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE); const size_t off_adjust = align_up(data_offset, ALIGN_SIZE); /* @@ -976,7 +1017,7 @@ void* tlsf_memalign_offs(tlsf_t tlsf, size_t align, size_t size, size_t data_off /* The offset is included in both `adjust` and `gap_minimum`, so we ** need to subtract it once. */ - const size_t size_with_gap = adjust_request_size(adjust + align + gap_minimum - off_adjust, align); + const size_t size_with_gap = adjust_request_size(tlsf, adjust + align + gap_minimum - off_adjust, align); /* ** If alignment is less than or equal to base alignment, we're done, because @@ -1089,7 +1130,7 @@ void* tlsf_realloc(tlsf_t tlsf, void* ptr, size_t size) const size_t cursize = block_size(block); const size_t combined = cursize + block_size(next) + block_header_overhead; - const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE); tlsf_assert(!block_is_free(block) && "block already marked as free"); diff --git a/tlsf.h b/tlsf.h index 3dcdb1e..400e837 100644 --- a/tlsf.h +++ b/tlsf.h @@ -20,8 +20,8 @@ typedef void* tlsf_t; typedef void* pool_t; /* Create/destroy a memory pool. */ -tlsf_t tlsf_create(void* mem); -tlsf_t tlsf_create_with_pool(void* mem, size_t bytes); +tlsf_t tlsf_create(void* mem, size_t max_bytes); +tlsf_t tlsf_create_with_pool(void* mem, size_t pool_bytes, size_t max_bytes); void tlsf_destroy(tlsf_t tlsf); pool_t tlsf_get_pool(tlsf_t tlsf); @@ -40,12 +40,13 @@ void tlsf_free(tlsf_t tlsf, void* ptr); size_t tlsf_block_size(void* ptr); /* Overheads/limits of internal structures. */ -size_t tlsf_size(void); +size_t tlsf_size(tlsf_t tlsf); size_t tlsf_align_size(void); size_t tlsf_block_size_min(void); -size_t tlsf_block_size_max(void); +size_t tlsf_block_size_max(tlsf_t tlsf); size_t tlsf_pool_overhead(void); size_t tlsf_alloc_overhead(void); +size_t tlsf_fit_size(tlsf_t tlsf, size_t size); /* Debugging. */ typedef void (*tlsf_walker)(void* ptr, size_t size, int used, void* user); diff --git a/tlsf_common.h b/tlsf_common.h index 7af2a2a..7ef440c 100644 --- a/tlsf_common.h +++ b/tlsf_common.h @@ -37,9 +37,9 @@ enum tlsf_config { /* log2 of number of linear subdivisions of block sizes. Larger ** values require more memory in the control structure. Values of - ** 4 or 5 are typical. + ** 4 or 5 are typical, 3 is for very small pools. */ - SL_INDEX_COUNT_LOG2 = 5, + SL_INDEX_COUNT_LOG2_MIN = 3, /* All allocation sizes and addresses are aligned to 4 bytes. */ ALIGN_SIZE_LOG2 = 2, @@ -54,22 +54,16 @@ enum tlsf_config ** trying to split size ranges into more slots than we have available. ** Instead, we calculate the minimum threshold size, and place all ** blocks below that size into the 0th first-level list. + ** Values below are the absolute minimum to accept a pool addition. */ /* Tunning the first level, we can reduce TLSF pool overhead * in exchange of manage a pool smaller than 4GB */ - #ifdef FL_INDEX_MAX_PLATFORM - FL_INDEX_MAX = FL_INDEX_MAX_PLATFORM, - #else - FL_INDEX_MAX = 30, - #endif - - SL_INDEX_COUNT = (1 << SL_INDEX_COUNT_LOG2), - FL_INDEX_SHIFT = (SL_INDEX_COUNT_LOG2 + ALIGN_SIZE_LOG2), - FL_INDEX_COUNT = (FL_INDEX_MAX - FL_INDEX_SHIFT + 1), - - SMALL_BLOCK_SIZE = (1 << FL_INDEX_SHIFT), + FL_INDEX_MAX_MIN = 14, + SL_INDEX_COUNT_MIN = (1 << SL_INDEX_COUNT_LOG2_MIN), + FL_INDEX_SHIFT_MIN = (SL_INDEX_COUNT_LOG2_MIN + ALIGN_SIZE_LOG2), + FL_INDEX_COUNT_MIN = (FL_INDEX_MAX_MIN - FL_INDEX_SHIFT_MIN + 1), }; /* @@ -92,20 +86,6 @@ typedef struct block_header_t struct block_header_t* prev_free; } block_header_t; -/* The TLSF control structure. */ -typedef struct control_t -{ - /* Empty lists point at this block to indicate they are free. */ - block_header_t block_null; - - /* Bitmaps for free lists. */ - unsigned int fl_bitmap; - unsigned int sl_bitmap[FL_INDEX_COUNT]; - - /* Head of free lists. */ - block_header_t* blocks[FL_INDEX_COUNT][SL_INDEX_COUNT]; -} control_t; - /* ** Since block sizes are always at least a multiple of 4, the two least ** significant bits of the size field are used to store the block status: @@ -132,7 +112,31 @@ static const size_t block_start_offset = */ static const size_t block_size_min = sizeof(block_header_t) - sizeof(block_header_t*); -static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX; +static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX_MIN; + +/* The TLSF control structure. */ +typedef struct control_t +{ + /* Empty lists point at this block to indicate they are free. */ + block_header_t block_null; + + /* Local parameter for the pool */ + unsigned int fl_index_count; + unsigned int fl_index_shift; + unsigned int fl_index_max; + unsigned int sl_index_count; + unsigned int sl_index_count_log2; + unsigned int small_block_size; + size_t size; + + /* Bitmaps for free lists. */ + unsigned int fl_bitmap; + unsigned int *sl_bitmap; + + /* Head of free lists. */ + block_header_t** blocks; +} control_t; + #if defined(__cplusplus) }; From b13ce2f6bc3aec25bcd53a45c2fbe7d136888bf8 Mon Sep 17 00:00:00 2001 From: X-Ryl669 Date: Wed, 12 Oct 2022 16:08:12 +0200 Subject: [PATCH 2/5] Fix for long lines in asserts from Guillaume Souchere Signed-off-by: X-Ryl669 --- tlsf.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tlsf.c b/tlsf.c index 6ddf10f..42d50e7 100644 --- a/tlsf.c +++ b/tlsf.c @@ -35,7 +35,6 @@ ** ffs/fls return 1-32 by default, returning 0 for error. */ - /* ** Detect whether or not we are building for a 32- or 64-bit (LP/LLP) ** architecture. There is no reliable portable method at compile-time. @@ -224,7 +223,7 @@ static inline __attribute__((always_inline)) size_t align_up(size_t x, size_t al static inline __attribute__((always_inline)) size_t align_down(size_t x, size_t align) { - tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two "); + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); return x - (x & (align - 1)); } @@ -362,7 +361,8 @@ static inline __attribute__((always_inline)) void insert_free_block(control_t* c block->prev_free = &control->block_null; current->prev_free = block; - tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE) && "block not aligned properly"); + tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE) + && "block not aligned properly"); /* ** Insert the new block at the head of the list, and mark the first- ** and second-level bitmaps appropriately. @@ -407,7 +407,8 @@ static inline __attribute__((always_inline)) block_header_t* block_split(block_h * This field is NOT part of the size, so it has to be substracted from the calculation. */ const size_t remain_size = block_size(block) - (size + block_header_overhead); - tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE) && "remaining block not aligned properly"); + tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE) + && "remaining block not aligned properly"); tlsf_assert(block_size(block) == remain_size + size + block_header_overhead); block_set_size(remaining, remain_size); From 049f54327a7bb0320fa0eadc334ab16c3715788e Mon Sep 17 00:00:00 2001 From: Guillaume Souchere Date: Thu, 13 Oct 2022 15:52:03 +0200 Subject: [PATCH 3/5] tlsf: update calculation of fl index max The calculation of fl index max is changed to always be the smallest number that includes the size of the registered memory. The control_construct() function now checks for minimum size as the control structure parameters are calculated. There is no longer a minimum configuration for fl index max so the tlsf_config enum is striped down to remove unecessary compile time values. the tlsf_size() function will fail if no tlsf pointer is passed as parameter since there is no way to calculate a default tlsf size anymore. --- tlsf.c | 70 ++++++++++++++++++++++++++++++++++----------------- tlsf_common.h | 27 -------------------- 2 files changed, 47 insertions(+), 50 deletions(-) diff --git a/tlsf.c b/tlsf.c index 42d50e7..6b9bb30 100644 --- a/tlsf.c +++ b/tlsf.c @@ -584,16 +584,19 @@ static inline __attribute__((always_inline)) void* block_prepare_used(control_t* } /* Clear structure and point all empty lists at the null block. */ -static void control_construct(control_t* control, size_t bytes) +static control_t* control_construct(control_t* control, size_t bytes) { - int i, j; - - control->block_null.next_free = &control->block_null; - control->block_null.prev_free = &control->block_null; + // check that the requested size can at least hold the control_t. This will allow us + // to fill in the field of control_t necessary to determine the final size of + // the metadata overhead and check that the requested size can hold + // this data and at least a block of minimum size + if (bytes < sizeof(control_t)) + { + return NULL; + } /* Find the closest power of two for first layer */ - i = (bytes - 1) / (16 * 1024); - control->fl_index_max = FL_INDEX_MAX_MIN + sizeof(i) * 8 - __builtin_clz(i); + control->fl_index_max = 32 - __builtin_clz(bytes); /* Adapt second layer to the pool */ if (bytes <= 16 * 1024) control->sl_index_count_log2 = 3; @@ -604,11 +607,26 @@ static void control_construct(control_t* control, size_t bytes) control->sl_index_count = 1 << control->sl_index_count_log2; control->fl_index_count = control->fl_index_max - control->fl_index_shift + 1; control->small_block_size = 1 << control->fl_index_shift; + + // the total size fo the metadata overhead is the size of the control_t + // added to the size of the sl_bitmaps and the size of blocks + control->size = sizeof(control_t) + (sizeof(*control->sl_bitmap) * control->fl_index_count) + + (sizeof(*control->blocks) * (control->fl_index_count * control->sl_index_count)); + + // check that the requested size can hold the whole control structure and + // a small block at least + if (bytes < control->size + block_size_min) + { + return NULL; + } + + control->block_null.next_free = &control->block_null; + control->block_null.prev_free = &control->block_null; control->fl_bitmap = 0; control->sl_bitmap = align_ptr(control + 1, sizeof(*control->sl_bitmap)); control->blocks = align_ptr(control->sl_bitmap + control->fl_index_count, sizeof(*control->blocks)); - control->size = (void*) (control->blocks + control->sl_index_count * control->fl_index_count) - (void*) control; + /* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */ tlsf_assert(sizeof(unsigned int) * CHAR_BIT >= control->sl_index_count); //CHAR_BIT less than sl_index_count"); @@ -616,14 +634,16 @@ static void control_construct(control_t* control, size_t bytes) /* Ensure we've properly tuned our sizes. */ tlsf_assert(ALIGN_SIZE == control->small_block_size / control->sl_index_count); //ALIGN_SIZE does not match"); - for (i = 0; i < control->fl_index_count; ++i) + for (int i = 0; i < control->fl_index_count; ++i) { control->sl_bitmap[i] = 0; - for (j = 0; j < control->sl_index_count; ++j) + for (int j = 0; j < control->sl_index_count; ++j) { control->blocks[i * control->sl_index_count + j] = &control->block_null; } } + + return control; } /* @@ -791,16 +811,12 @@ size_t tlsf_fit_size(tlsf_t tlsf, size_t size) */ size_t tlsf_size(tlsf_t tlsf) { - if (tlsf) + if (tlsf == NULL) { - control_t* control = tlsf_cast(control_t*, tlsf); - return control->size; - } - - /* no tlsf, we'll just return a min size */ - return sizeof(control_t) + - sizeof(int) * SL_INDEX_COUNT_MIN + - sizeof(block_header_t*) * SL_INDEX_COUNT_MIN * FL_INDEX_COUNT_MIN; + return 0; + } + control_t* control = tlsf_cast(control_t*, tlsf); + return control->size; } size_t tlsf_align_size(void) @@ -946,15 +962,17 @@ tlsf_t tlsf_create(void* mem, size_t max_bytes) return 0; } - control_construct(tlsf_cast(control_t*, mem), max_bytes); - - return tlsf_cast(tlsf_t, mem); + control_t* control_ptr = control_construct(tlsf_cast(control_t*, mem), max_bytes); + return tlsf_cast(tlsf_t, control_ptr); } tlsf_t tlsf_create_with_pool(void* mem, size_t pool_bytes, size_t max_bytes) { tlsf_t tlsf = tlsf_create(mem, max_bytes ? max_bytes : pool_bytes); - tlsf_add_pool(tlsf, (char*)mem + tlsf_size(tlsf), pool_bytes - tlsf_size(tlsf)); + if (tlsf != NULL) + { + tlsf_add_pool(tlsf, (char*)mem + tlsf_size(tlsf), pool_bytes - tlsf_size(tlsf)); + } return tlsf; } @@ -1133,6 +1151,12 @@ void* tlsf_realloc(tlsf_t tlsf, void* ptr, size_t size) const size_t combined = cursize + block_size(next) + block_header_overhead; const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE); + // if adjust if equal to 0, the size is too big + if (adjust == 0) + { + return p; + } + tlsf_assert(!block_is_free(block) && "block already marked as free"); /* diff --git a/tlsf_common.h b/tlsf_common.h index 7ef440c..e7b41ff 100644 --- a/tlsf_common.h +++ b/tlsf_common.h @@ -35,35 +35,9 @@ extern "C" { enum tlsf_config { - /* log2 of number of linear subdivisions of block sizes. Larger - ** values require more memory in the control structure. Values of - ** 4 or 5 are typical, 3 is for very small pools. - */ - SL_INDEX_COUNT_LOG2_MIN = 3, - /* All allocation sizes and addresses are aligned to 4 bytes. */ ALIGN_SIZE_LOG2 = 2, ALIGN_SIZE = (1 << ALIGN_SIZE_LOG2), - - /* - ** We support allocations of sizes up to (1 << FL_INDEX_MAX) bits. - ** However, because we linearly subdivide the second-level lists, and - ** our minimum size granularity is 4 bytes, it doesn't make sense to - ** create first-level lists for sizes smaller than SL_INDEX_COUNT * 4, - ** or (1 << (SL_INDEX_COUNT_LOG2 + 2)) bytes, as there we will be - ** trying to split size ranges into more slots than we have available. - ** Instead, we calculate the minimum threshold size, and place all - ** blocks below that size into the 0th first-level list. - ** Values below are the absolute minimum to accept a pool addition. - */ - - /* Tunning the first level, we can reduce TLSF pool overhead - * in exchange of manage a pool smaller than 4GB - */ - FL_INDEX_MAX_MIN = 14, - SL_INDEX_COUNT_MIN = (1 << SL_INDEX_COUNT_LOG2_MIN), - FL_INDEX_SHIFT_MIN = (SL_INDEX_COUNT_LOG2_MIN + ALIGN_SIZE_LOG2), - FL_INDEX_COUNT_MIN = (FL_INDEX_MAX_MIN - FL_INDEX_SHIFT_MIN + 1), }; /* @@ -112,7 +86,6 @@ static const size_t block_start_offset = */ static const size_t block_size_min = sizeof(block_header_t) - sizeof(block_header_t*); -static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX_MIN; /* The TLSF control structure. */ typedef struct control_t From eaa11bbf5d15b4b33f992c30000eaca84b8149fe Mon Sep 17 00:00:00 2001 From: Guillaume Souchere Date: Fri, 14 Oct 2022 08:49:07 +0200 Subject: [PATCH 4/5] tlsf: fix input validation of tlsf_t parameter and comment tlsf_fit_size() --- tlsf.c | 17 +++++++++++++---- tlsf.h | 9 +++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tlsf.c b/tlsf.c index 6b9bb30..d9fdb4c 100644 --- a/tlsf.c +++ b/tlsf.c @@ -794,11 +794,11 @@ int tlsf_check_pool(pool_t pool) size_t tlsf_fit_size(tlsf_t tlsf, size_t size) { /* because it's GoodFit, allocable size is one range lower */ - if (size) + if (size && tlsf != NULL) { size_t sl_interval; control_t* control = tlsf_cast(control_t*, tlsf); - sl_interval = (1 << ((sizeof(size_t) * 8 - 1) - __builtin_clz(size))) / control->sl_index_count; + sl_interval = (1 << (32 - __builtin_clz(size) - 1)) / control->sl_index_count; return size & ~(sl_interval - 1); } @@ -831,6 +831,10 @@ size_t tlsf_block_size_min(void) size_t tlsf_block_size_max(tlsf_t tlsf) { + if (tlsf == NULL) + { + return 0; + } control_t* control = tlsf_cast(control_t*, tlsf); return tlsf_cast(size_t, 1) << control->fl_index_max; } @@ -951,15 +955,20 @@ tlsf_t tlsf_create(void* mem, size_t max_bytes) #if _DEBUG if (test_ffs_fls()) { - return 0; + return NULL; } #endif + if (mem == NULL) + { + return NULL; + } + if (((tlsfptr_t)mem % ALIGN_SIZE) != 0) { printf("tlsf_create: Memory must be aligned to %u bytes.\n", (unsigned int)ALIGN_SIZE); - return 0; + return NULL; } control_t* control_ptr = control_construct(tlsf_cast(control_t*, mem), max_bytes); diff --git a/tlsf.h b/tlsf.h index 400e837..7f6cc18 100644 --- a/tlsf.h +++ b/tlsf.h @@ -46,6 +46,15 @@ size_t tlsf_block_size_min(void); size_t tlsf_block_size_max(tlsf_t tlsf); size_t tlsf_pool_overhead(void); size_t tlsf_alloc_overhead(void); + +/** + * @brief Return the allocable size based on the size passed + * as parameter + * + * @param tlsf Pointer to the tlsf structure + * @param size The allocation size + * @return size_t The updated allocation size + */ size_t tlsf_fit_size(tlsf_t tlsf, size_t size); /* Debugging. */ From 1ca1da2407376624a96eda45454a92960b4d1527 Mon Sep 17 00:00:00 2001 From: Guillaume Souchere Date: Thu, 20 Oct 2022 12:09:26 +0200 Subject: [PATCH 5/5] tlsf: move control_t to tlsf.c - the control_t structure should not be available to the user so it was moved to tlsf.c. In addition bitfields are now used in control_t when possible which reduces the size of the structure from 56 bytes to 36 bytes. - fix an assert message to place it in the tlsf_assert - add comment about the index count log2 field in control_t --- tlsf.c | 3 ++- tlsf_common.h | 29 ++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tlsf.c b/tlsf.c index d9fdb4c..6f7d604 100644 --- a/tlsf.c +++ b/tlsf.c @@ -629,7 +629,8 @@ static control_t* control_construct(control_t* control, size_t bytes) /* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */ - tlsf_assert(sizeof(unsigned int) * CHAR_BIT >= control->sl_index_count); //CHAR_BIT less than sl_index_count"); + tlsf_assert(sizeof(unsigned int) * CHAR_BIT >= control->sl_index_count + && "CHAR_BIT less than sl_index_count"); /* Ensure we've properly tuned our sizes. */ tlsf_assert(ALIGN_SIZE == control->small_block_size / control->sl_index_count); //ALIGN_SIZE does not match"); diff --git a/tlsf_common.h b/tlsf_common.h index e7b41ff..35ab101 100644 --- a/tlsf_common.h +++ b/tlsf_common.h @@ -93,24 +93,35 @@ typedef struct control_t /* Empty lists point at this block to indicate they are free. */ block_header_t block_null; - /* Local parameter for the pool */ - unsigned int fl_index_count; - unsigned int fl_index_shift; - unsigned int fl_index_max; - unsigned int sl_index_count; - unsigned int sl_index_count_log2; - unsigned int small_block_size; + /* Local parameter for the pool. Given the maximum + * value of each field, all the following parameters + * can fit on 4 bytes when using bitfields + */ + unsigned int fl_index_count : 5; // 5 cumulated bits + unsigned int fl_index_shift : 3; // 8 cumulated bits + unsigned int fl_index_max : 6; // 14 cumulated bits + unsigned int sl_index_count : 6; // 20 cumulated bits + + /* log2 of number of linear subdivisions of block sizes. Larger + ** values require more memory in the control structure. Values of + ** 4 or 5 are typical. + */ + unsigned int sl_index_count_log2 : 3; // 23 cumulated bits + unsigned int small_block_size : 8; // 31 cumulated bits + + /* size of the metadata ( size of control block, + * sl_bitmap and blocks ) + */ size_t size; /* Bitmaps for free lists. */ unsigned int fl_bitmap; - unsigned int *sl_bitmap; + unsigned int *sl_bitmap; /* Head of free lists. */ block_header_t** blocks; } control_t; - #if defined(__cplusplus) }; #endif