From 4cd9c116924b8fcafbbcc375c268677ecee9bcab Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Fri, 17 Nov 2023 18:58:55 -0500 Subject: [PATCH] lib: heap: split heap-validate.c Split heap-validate.c into smaller chunks. We have been adding all kind of new APIs under this file and building it unconditionally whether those APIs are needed/used or not. Many of those APIs have nothing to do with the validation part. Signed-off-by: Anas Nashif --- lib/heap/CMakeLists.txt | 2 + lib/heap/heap-info.c | 76 +++++++++++++ lib/heap/heap-stress.c | 153 ++++++++++++++++++++++++++ lib/heap/heap-validate.c | 228 +-------------------------------------- lib/heap/heap.h | 17 +++ 5 files changed, 249 insertions(+), 227 deletions(-) create mode 100644 lib/heap/heap-info.c create mode 100644 lib/heap/heap-stress.c diff --git a/lib/heap/CMakeLists.txt b/lib/heap/CMakeLists.txt index bc5f30b4841..fa297fbdca6 100644 --- a/lib/heap/CMakeLists.txt +++ b/lib/heap/CMakeLists.txt @@ -3,6 +3,8 @@ zephyr_sources( heap.c heap-validate.c + heap-stress.c + heap-info.c multi_heap.c ) diff --git a/lib/heap/heap-info.c b/lib/heap/heap-info.c new file mode 100644 index 00000000000..15d383f41e3 --- /dev/null +++ b/lib/heap/heap-info.c @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ +#include +#include +#include +#include "heap.h" + +/* + * Print heap info for debugging / analysis purpose + */ +void heap_print_info(struct z_heap *h, bool dump_chunks) +{ + int i, nb_buckets = bucket_idx(h, h->end_chunk) + 1; + size_t free_bytes, allocated_bytes, total, overhead; + + printk("Heap at %p contains %d units in %d buckets\n\n", + chunk_buf(h), h->end_chunk, nb_buckets); + + printk(" bucket# min units total largest largest\n" + " threshold chunks (units) (bytes)\n" + " -----------------------------------------------------------\n"); + for (i = 0; i < nb_buckets; i++) { + chunkid_t first = h->buckets[i].next; + chunksz_t largest = 0; + int count = 0; + + if (first) { + chunkid_t curr = first; + + do { + count++; + largest = MAX(largest, chunk_size(h, curr)); + curr = next_free_chunk(h, curr); + } while (curr != first); + } + if (count) { + printk("%9d %12d %12d %12d %12zd\n", + i, (1 << i) - 1 + min_chunk_size(h), count, + largest, chunksz_to_bytes(h, largest)); + } + } + + if (dump_chunks) { + printk("\nChunk dump:\n"); + for (chunkid_t c = 0; ; c = right_chunk(h, c)) { + printk("chunk %4d: [%c] size=%-4d left=%-4d right=%d\n", + c, + chunk_used(h, c) ? '*' + : solo_free_header(h, c) ? '.' + : '-', + chunk_size(h, c), + left_chunk(h, c), + right_chunk(h, c)); + if (c == h->end_chunk) { + break; + } + } + } + + get_alloc_info(h, &allocated_bytes, &free_bytes); + /* The end marker chunk has a header. It is part of the overhead. */ + total = h->end_chunk * CHUNK_UNIT + chunk_header_bytes(h); + overhead = total - free_bytes - allocated_bytes; + printk("\n%zd free bytes, %zd allocated bytes, overhead = %zd bytes (%zd.%zd%%)\n", + free_bytes, allocated_bytes, overhead, + (1000 * overhead + total/2) / total / 10, + (1000 * overhead + total/2) / total % 10); +} + +void sys_heap_print_info(struct sys_heap *heap, bool dump_chunks) +{ + heap_print_info(heap->heap, dump_chunks); +} diff --git a/lib/heap/heap-stress.c b/lib/heap/heap-stress.c new file mode 100644 index 00000000000..443ffc9e7eb --- /dev/null +++ b/lib/heap/heap-stress.c @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2019 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ +#include +#include +#include +#include "heap.h" + +struct z_heap_stress_rec { + void *(*alloc_fn)(void *arg, size_t bytes); + void (*free_fn)(void *arg, void *p); + void *arg; + size_t total_bytes; + struct z_heap_stress_block *blocks; + size_t nblocks; + size_t blocks_alloced; + size_t bytes_alloced; + uint32_t target_percent; +}; + +struct z_heap_stress_block { + void *ptr; + size_t sz; +}; + +/* Very simple LCRNG (from https://nuclear.llnl.gov/CNP/rng/rngman/node4.html) + * + * Here to guarantee cross-platform test repeatability. + */ +static uint32_t rand32(void) +{ + static uint64_t state = 123456789; /* seed */ + + state = state * 2862933555777941757UL + 3037000493UL; + + return (uint32_t)(state >> 32); +} + +static bool rand_alloc_choice(struct z_heap_stress_rec *sr) +{ + /* Edge cases: no blocks allocated, and no space for a new one */ + if (sr->blocks_alloced == 0) { + return true; + } else if (sr->blocks_alloced >= sr->nblocks) { + return false; + } else { + + /* The way this works is to scale the chance of choosing to + * allocate vs. free such that it's even odds when the heap is + * at the target percent, with linear tapering on the low + * slope (i.e. we choose to always allocate with an empty + * heap, allocate 50% of the time when the heap is exactly at + * the target, and always free when above the target). In + * practice, the operations aren't quite symmetric (you can + * always free, but your allocation might fail), and the units + * aren't matched (we're doing math based on bytes allocated + * and ignoring the overhead) but this is close enough. And + * yes, the math here is coarse (in units of percent), but + * that's good enough and fits well inside 32 bit quantities. + * (Note precision issue when heap size is above 40MB + * though!). + */ + __ASSERT(sr->total_bytes < 0xffffffffU / 100, "too big for u32!"); + uint32_t full_pct = (100 * sr->bytes_alloced) / sr->total_bytes; + uint32_t target = sr->target_percent ? sr->target_percent : 1; + uint32_t free_chance = 0xffffffffU; + + if (full_pct < sr->target_percent) { + free_chance = full_pct * (0x80000000U / target); + } + + return rand32() > free_chance; + } +} + +/* Chooses a size of block to allocate, logarithmically favoring + * smaller blocks (i.e. blocks twice as large are half as frequent + */ +static size_t rand_alloc_size(struct z_heap_stress_rec *sr) +{ + ARG_UNUSED(sr); + + /* Min scale of 4 means that the half of the requests in the + * smallest size have an average size of 8 + */ + int scale = 4 + __builtin_clz(rand32()); + + return rand32() & BIT_MASK(scale); +} + +/* Returns the index of a randomly chosen block to free */ +static size_t rand_free_choice(struct z_heap_stress_rec *sr) +{ + return rand32() % sr->blocks_alloced; +} + +/* General purpose heap stress test. Takes function pointers to allow + * for testing multiple heap APIs with the same rig. The alloc and + * free functions are passed back the argument as a context pointer. + * The "log" function is for readable user output. The total_bytes + * argument should reflect the size of the heap being tested. The + * scratch array is used to store temporary state and should be sized + * about half as large as the heap itself. Returns true on success. + */ +void sys_heap_stress(void *(*alloc_fn)(void *arg, size_t bytes), + void (*free_fn)(void *arg, void *p), + void *arg, size_t total_bytes, + uint32_t op_count, + void *scratch_mem, size_t scratch_bytes, + int target_percent, + struct z_heap_stress_result *result) +{ + struct z_heap_stress_rec sr = { + .alloc_fn = alloc_fn, + .free_fn = free_fn, + .arg = arg, + .total_bytes = total_bytes, + .blocks = scratch_mem, + .nblocks = scratch_bytes / sizeof(struct z_heap_stress_block), + .target_percent = target_percent, + }; + + *result = (struct z_heap_stress_result) {0}; + + for (uint32_t i = 0; i < op_count; i++) { + if (rand_alloc_choice(&sr)) { + size_t sz = rand_alloc_size(&sr); + void *p = sr.alloc_fn(sr.arg, sz); + + result->total_allocs++; + if (p != NULL) { + result->successful_allocs++; + sr.blocks[sr.blocks_alloced].ptr = p; + sr.blocks[sr.blocks_alloced].sz = sz; + sr.blocks_alloced++; + sr.bytes_alloced += sz; + } + } else { + int b = rand_free_choice(&sr); + void *p = sr.blocks[b].ptr; + size_t sz = sr.blocks[b].sz; + + result->total_frees++; + sr.blocks[b] = sr.blocks[sr.blocks_alloced - 1]; + sr.blocks_alloced--; + sr.bytes_alloced -= sz; + sr.free_fn(sr.arg, p); + } + result->accumulated_in_use_bytes += sr.bytes_alloced; + } +} diff --git a/lib/heap/heap-validate.c b/lib/heap/heap-validate.c index 18aafe71804..3d52a0d15c4 100644 --- a/lib/heap/heap-validate.c +++ b/lib/heap/heap-validate.c @@ -68,23 +68,6 @@ static inline void check_nexts(struct z_heap *h, int bidx) } } -static void get_alloc_info(struct z_heap *h, size_t *alloc_bytes, - size_t *free_bytes) -{ - chunkid_t c; - - *alloc_bytes = 0; - *free_bytes = 0; - - for (c = right_chunk(h, 0); c < h->end_chunk; c = right_chunk(h, c)) { - if (chunk_used(h, c)) { - *alloc_bytes += chunksz_to_bytes(h, chunk_size(h, c)); - } else if (!solo_free_header(h, c)) { - *free_bytes += chunksz_to_bytes(h, chunk_size(h, c)); - } - } -} - bool sys_heap_validate(struct sys_heap *heap) { struct z_heap *h = heap->heap; @@ -157,6 +140,7 @@ bool sys_heap_validate(struct sys_heap *heap) * USED. */ chunkid_t prev_chunk = 0; + for (c = right_chunk(h, 0); c < h->end_chunk; c = right_chunk(h, c)) { if (!chunk_used(h, c) && !solo_free_header(h, c)) { return false; @@ -201,216 +185,6 @@ bool sys_heap_validate(struct sys_heap *heap) return true; } -struct z_heap_stress_rec { - void *(*alloc_fn)(void *arg, size_t bytes); - void (*free_fn)(void *arg, void *p); - void *arg; - size_t total_bytes; - struct z_heap_stress_block *blocks; - size_t nblocks; - size_t blocks_alloced; - size_t bytes_alloced; - uint32_t target_percent; -}; - -struct z_heap_stress_block { - void *ptr; - size_t sz; -}; - -/* Very simple LCRNG (from https://nuclear.llnl.gov/CNP/rng/rngman/node4.html) - * - * Here to guarantee cross-platform test repeatability. - */ -static uint32_t rand32(void) -{ - static uint64_t state = 123456789; /* seed */ - - state = state * 2862933555777941757UL + 3037000493UL; - - return (uint32_t)(state >> 32); -} - -static bool rand_alloc_choice(struct z_heap_stress_rec *sr) -{ - /* Edge cases: no blocks allocated, and no space for a new one */ - if (sr->blocks_alloced == 0) { - return true; - } else if (sr->blocks_alloced >= sr->nblocks) { - return false; - } else { - - /* The way this works is to scale the chance of choosing to - * allocate vs. free such that it's even odds when the heap is - * at the target percent, with linear tapering on the low - * slope (i.e. we choose to always allocate with an empty - * heap, allocate 50% of the time when the heap is exactly at - * the target, and always free when above the target). In - * practice, the operations aren't quite symmetric (you can - * always free, but your allocation might fail), and the units - * aren't matched (we're doing math based on bytes allocated - * and ignoring the overhead) but this is close enough. And - * yes, the math here is coarse (in units of percent), but - * that's good enough and fits well inside 32 bit quantities. - * (Note precision issue when heap size is above 40MB - * though!). - */ - __ASSERT(sr->total_bytes < 0xffffffffU / 100, "too big for u32!"); - uint32_t full_pct = (100 * sr->bytes_alloced) / sr->total_bytes; - uint32_t target = sr->target_percent ? sr->target_percent : 1; - uint32_t free_chance = 0xffffffffU; - - if (full_pct < sr->target_percent) { - free_chance = full_pct * (0x80000000U / target); - } - - return rand32() > free_chance; - } -} - -/* Chooses a size of block to allocate, logarithmically favoring - * smaller blocks (i.e. blocks twice as large are half as frequent - */ -static size_t rand_alloc_size(struct z_heap_stress_rec *sr) -{ - ARG_UNUSED(sr); - - /* Min scale of 4 means that the half of the requests in the - * smallest size have an average size of 8 - */ - int scale = 4 + __builtin_clz(rand32()); - - return rand32() & BIT_MASK(scale); -} - -/* Returns the index of a randomly chosen block to free */ -static size_t rand_free_choice(struct z_heap_stress_rec *sr) -{ - return rand32() % sr->blocks_alloced; -} - -/* General purpose heap stress test. Takes function pointers to allow - * for testing multiple heap APIs with the same rig. The alloc and - * free functions are passed back the argument as a context pointer. - * The "log" function is for readable user output. The total_bytes - * argument should reflect the size of the heap being tested. The - * scratch array is used to store temporary state and should be sized - * about half as large as the heap itself. Returns true on success. - */ -void sys_heap_stress(void *(*alloc_fn)(void *arg, size_t bytes), - void (*free_fn)(void *arg, void *p), - void *arg, size_t total_bytes, - uint32_t op_count, - void *scratch_mem, size_t scratch_bytes, - int target_percent, - struct z_heap_stress_result *result) -{ - struct z_heap_stress_rec sr = { - .alloc_fn = alloc_fn, - .free_fn = free_fn, - .arg = arg, - .total_bytes = total_bytes, - .blocks = scratch_mem, - .nblocks = scratch_bytes / sizeof(struct z_heap_stress_block), - .target_percent = target_percent, - }; - - *result = (struct z_heap_stress_result) {0}; - - for (uint32_t i = 0; i < op_count; i++) { - if (rand_alloc_choice(&sr)) { - size_t sz = rand_alloc_size(&sr); - void *p = sr.alloc_fn(sr.arg, sz); - - result->total_allocs++; - if (p != NULL) { - result->successful_allocs++; - sr.blocks[sr.blocks_alloced].ptr = p; - sr.blocks[sr.blocks_alloced].sz = sz; - sr.blocks_alloced++; - sr.bytes_alloced += sz; - } - } else { - int b = rand_free_choice(&sr); - void *p = sr.blocks[b].ptr; - size_t sz = sr.blocks[b].sz; - - result->total_frees++; - sr.blocks[b] = sr.blocks[sr.blocks_alloced - 1]; - sr.blocks_alloced--; - sr.bytes_alloced -= sz; - sr.free_fn(sr.arg, p); - } - result->accumulated_in_use_bytes += sr.bytes_alloced; - } -} - -/* - * Print heap info for debugging / analysis purpose - */ -void heap_print_info(struct z_heap *h, bool dump_chunks) -{ - int i, nb_buckets = bucket_idx(h, h->end_chunk) + 1; - size_t free_bytes, allocated_bytes, total, overhead; - - printk("Heap at %p contains %d units in %d buckets\n\n", - chunk_buf(h), h->end_chunk, nb_buckets); - - printk(" bucket# min units total largest largest\n" - " threshold chunks (units) (bytes)\n" - " -----------------------------------------------------------\n"); - for (i = 0; i < nb_buckets; i++) { - chunkid_t first = h->buckets[i].next; - chunksz_t largest = 0; - int count = 0; - - if (first) { - chunkid_t curr = first; - do { - count++; - largest = MAX(largest, chunk_size(h, curr)); - curr = next_free_chunk(h, curr); - } while (curr != first); - } - if (count) { - printk("%9d %12d %12d %12d %12zd\n", - i, (1 << i) - 1 + min_chunk_size(h), count, - largest, chunksz_to_bytes(h, largest)); - } - } - - if (dump_chunks) { - printk("\nChunk dump:\n"); - for (chunkid_t c = 0; ; c = right_chunk(h, c)) { - printk("chunk %4d: [%c] size=%-4d left=%-4d right=%d\n", - c, - chunk_used(h, c) ? '*' - : solo_free_header(h, c) ? '.' - : '-', - chunk_size(h, c), - left_chunk(h, c), - right_chunk(h, c)); - if (c == h->end_chunk) { - break; - } - } - } - - get_alloc_info(h, &allocated_bytes, &free_bytes); - /* The end marker chunk has a header. It is part of the overhead. */ - total = h->end_chunk * CHUNK_UNIT + chunk_header_bytes(h); - overhead = total - free_bytes - allocated_bytes; - printk("\n%zd free bytes, %zd allocated bytes, overhead = %zd bytes (%zd.%zd%%)\n", - free_bytes, allocated_bytes, overhead, - (1000 * overhead + total/2) / total / 10, - (1000 * overhead + total/2) / total % 10); -} - -void sys_heap_print_info(struct sys_heap *heap, bool dump_chunks) -{ - heap_print_info(heap->heap, dump_chunks); -} - #ifdef CONFIG_SYS_HEAP_RUNTIME_STATS int sys_heap_runtime_stats_get(struct sys_heap *heap, diff --git a/lib/heap/heap.h b/lib/heap/heap.h index a29f4adfbd6..9e0354b0b21 100644 --- a/lib/heap/heap.h +++ b/lib/heap/heap.h @@ -262,6 +262,23 @@ static inline bool size_too_big(struct z_heap *h, size_t bytes) return (bytes / CHUNK_UNIT) >= h->end_chunk; } +static inline void get_alloc_info(struct z_heap *h, size_t *alloc_bytes, + size_t *free_bytes) +{ + chunkid_t c; + + *alloc_bytes = 0; + *free_bytes = 0; + + for (c = right_chunk(h, 0); c < h->end_chunk; c = right_chunk(h, c)) { + if (chunk_used(h, c)) { + *alloc_bytes += chunksz_to_bytes(h, chunk_size(h, c)); + } else if (!solo_free_header(h, c)) { + *free_bytes += chunksz_to_bytes(h, chunk_size(h, c)); + } + } +} + /* For debugging */ void heap_print_info(struct z_heap *h, bool dump_chunks);