From 9da06456f208a092fd930a95cbe04b163fd5448f Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Mon, 10 Mar 2025 17:26:40 -0400 Subject: [PATCH] kernel: kheap: decouple simple alloc from aligned_alloc When k_heap_alloc() is expressed in terms of k_heap_aligned_alloc() it invokes a longer aligned allocation code path with an extra runtime overhead even though no alignment is necessary. Let's reference and invoke the aligned allocation code path only when an actual aligned allocation is requested. This opens the possibility for the linker to garbage-collect the aligning code otherwise. Signed-off-by: Nicolas Pitre --- include/zephyr/sys/sys_heap.h | 12 +++++++ include/zephyr/tracing/tracing.h | 2 +- kernel/kheap.c | 33 +++++++++++++------ lib/heap/heap.c | 7 ++++ subsys/tracing/ctf/tracing_ctf.h | 2 +- subsys/tracing/sysview/tracing_sysview.h | 2 +- .../tracing/test/tracing_string_format_test.c | 2 +- subsys/tracing/test/tracing_test.h | 6 ++-- subsys/tracing/user/tracing_user.h | 2 +- 9 files changed, 50 insertions(+), 18 deletions(-) diff --git a/include/zephyr/sys/sys_heap.h b/include/zephyr/sys/sys_heap.h index 0c859a5a567..f559e58958b 100644 --- a/include/zephyr/sys/sys_heap.h +++ b/include/zephyr/sys/sys_heap.h @@ -138,6 +138,18 @@ void *sys_heap_alloc(struct sys_heap *heap, size_t bytes); */ void *sys_heap_aligned_alloc(struct sys_heap *heap, size_t align, size_t bytes); +/** @brief Allocate memory from a sys_heap + * + * This is a wrapper for sys_heap_alloc() whose purpose is to provide the same + * function signature as sys_heap_aligned_alloc(). + * + * @param heap Heap from which to allocate + * @param align Ignored placeholder + * @param bytes Number of bytes requested + * @return Pointer to memory the caller can now use + */ +void *sys_heap_noalign_alloc(struct sys_heap *heap, size_t align, size_t bytes); + /** @brief Free memory into a sys_heap * * De-allocates a pointer to memory previously returned from diff --git a/include/zephyr/tracing/tracing.h b/include/zephyr/tracing/tracing.h index 355e49e0d61..13b27cdbddc 100644 --- a/include/zephyr/tracing/tracing.h +++ b/include/zephyr/tracing/tracing.h @@ -1753,7 +1753,7 @@ * @param h Heap object * @param timeout Timeout period */ -#define sys_port_trace_k_heap_aligned_alloc_blocking(h, timeout) +#define sys_port_trace_k_heap_alloc_helper_blocking(h, timeout) /** * @brief Trace Heap align alloc attempt outcome diff --git a/kernel/kheap.c b/kernel/kheap.c index 245558c85fe..548a417b4d9 100644 --- a/kernel/kheap.c +++ b/kernel/kheap.c @@ -63,22 +63,23 @@ SYS_INIT_NAMED(statics_init_pre, statics_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_ SYS_INIT_NAMED(statics_init_post, statics_init, POST_KERNEL, 0); #endif /* CONFIG_DEMAND_PAGING && !CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */ -void *k_heap_aligned_alloc(struct k_heap *heap, size_t align, size_t bytes, - k_timeout_t timeout) +typedef void * (sys_heap_allocator_t)(struct sys_heap *heap, size_t align, size_t bytes); + +static void *z_heap_alloc_helper(struct k_heap *heap, size_t align, size_t bytes, + k_timeout_t timeout, + sys_heap_allocator_t *sys_heap_allocator) { k_timepoint_t end = sys_timepoint_calc(timeout); void *ret = NULL; k_spinlock_key_t key = k_spin_lock(&heap->lock); - SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, aligned_alloc, heap, timeout); - __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), ""); bool blocked_alloc = false; while (ret == NULL) { - ret = sys_heap_aligned_alloc(&heap->heap, align, bytes); + ret = sys_heap_allocator(&heap->heap, align, bytes); if (!IS_ENABLED(CONFIG_MULTITHREADING) || (ret != NULL) || K_TIMEOUT_EQ(timeout, K_NO_WAIT)) { @@ -88,7 +89,7 @@ void *k_heap_aligned_alloc(struct k_heap *heap, size_t align, size_t bytes, if (!blocked_alloc) { blocked_alloc = true; - SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_heap, aligned_alloc, heap, timeout); + SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_heap, alloc_helper, heap, timeout); } else { /** * @todo Trace attempt to avoid empty trace segments @@ -100,8 +101,6 @@ void *k_heap_aligned_alloc(struct k_heap *heap, size_t align, size_t bytes, key = k_spin_lock(&heap->lock); } - SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, aligned_alloc, heap, timeout, ret); - k_spin_unlock(&heap->lock, key); return ret; } @@ -110,13 +109,27 @@ void *k_heap_alloc(struct k_heap *heap, size_t bytes, k_timeout_t timeout) { SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, alloc, heap, timeout); - void *ret = k_heap_aligned_alloc(heap, sizeof(void *), bytes, timeout); + void *ret = z_heap_alloc_helper(heap, 0, bytes, timeout, + sys_heap_noalign_alloc); SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, alloc, heap, timeout, ret); return ret; } +void *k_heap_aligned_alloc(struct k_heap *heap, size_t align, size_t bytes, + k_timeout_t timeout) +{ + SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, aligned_alloc, heap, timeout); + + void *ret = z_heap_alloc_helper(heap, align, bytes, timeout, + sys_heap_aligned_alloc); + + SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, aligned_alloc, heap, timeout, ret); + + return ret; +} + void *k_heap_calloc(struct k_heap *heap, size_t num, size_t size, k_timeout_t timeout) { SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, calloc, heap, timeout); @@ -148,7 +161,7 @@ void *k_heap_realloc(struct k_heap *heap, void *ptr, size_t bytes, k_timeout_t t __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), ""); while (ret == NULL) { - ret = sys_heap_aligned_realloc(&heap->heap, ptr, sizeof(void *), bytes); + ret = sys_heap_realloc(&heap->heap, ptr, bytes); if (!IS_ENABLED(CONFIG_MULTITHREADING) || (ret != NULL) || K_TIMEOUT_EQ(timeout, K_NO_WAIT)) { diff --git a/lib/heap/heap.c b/lib/heap/heap.c index 517c43f5085..743485f2854 100644 --- a/lib/heap/heap.c +++ b/lib/heap/heap.c @@ -298,6 +298,13 @@ void *sys_heap_alloc(struct sys_heap *heap, size_t bytes) return mem; } +void *sys_heap_noalign_alloc(struct sys_heap *heap, size_t align, size_t bytes) +{ + ARG_UNUSED(align); + + return sys_heap_alloc(heap, bytes); +} + void *sys_heap_aligned_alloc(struct sys_heap *heap, size_t align, size_t bytes) { struct z_heap *h = heap->heap; diff --git a/subsys/tracing/ctf/tracing_ctf.h b/subsys/tracing/ctf/tracing_ctf.h index 3b12f8401b8..c1d8b65bd6b 100644 --- a/subsys/tracing/ctf/tracing_ctf.h +++ b/subsys/tracing/ctf/tracing_ctf.h @@ -311,7 +311,7 @@ extern "C" { #define sys_port_trace_k_heap_init(heap) #define sys_port_trace_k_heap_aligned_alloc_enter(heap, timeout) -#define sys_port_trace_k_heap_aligned_alloc_blocking(heap, timeout) +#define sys_port_trace_k_heap_alloc_helper_blocking(heap, timeout) #define sys_port_trace_k_heap_aligned_alloc_exit(heap, timeout, ret) #define sys_port_trace_k_heap_alloc_enter(heap, timeout) #define sys_port_trace_k_heap_alloc_exit(heap, timeout, ret) diff --git a/subsys/tracing/sysview/tracing_sysview.h b/subsys/tracing/sysview/tracing_sysview.h index 15fd161af52..d13ab42dd8d 100644 --- a/subsys/tracing/sysview/tracing_sysview.h +++ b/subsys/tracing/sysview/tracing_sysview.h @@ -573,7 +573,7 @@ void sys_trace_thread_info(struct k_thread *thread); SEGGER_SYSVIEW_RecordU32x2(TID_HEAP_ALIGNED_ALLOC, (uint32_t)(uintptr_t)heap, \ (uint32_t)timeout.ticks) -#define sys_port_trace_k_heap_aligned_alloc_blocking(heap, timeout) +#define sys_port_trace_k_heap_alloc_helper_blocking(heap, timeout) #define sys_port_trace_k_heap_aligned_alloc_exit(heap, timeout, ret) \ SEGGER_SYSVIEW_RecordEndCallU32(TID_HEAP_ALIGNED_ALLOC, (uint32_t)ret) diff --git a/subsys/tracing/test/tracing_string_format_test.c b/subsys/tracing/test/tracing_string_format_test.c index c18d5b2ed14..8a53e0bcdb8 100644 --- a/subsys/tracing/test/tracing_string_format_test.c +++ b/subsys/tracing/test/tracing_string_format_test.c @@ -381,7 +381,7 @@ void sys_trace_k_heap_realloc_exit(struct k_heap *h, void *ptr, size_t bytes, k_ TRACING_STRING("%s: %p\n", __func__, h); } -void sys_trace_k_heap_aligned_alloc_blocking(struct k_heap *h, size_t bytes, k_timeout_t timeout) +void sys_trace_k_heap_alloc_helper_blocking(struct k_heap *h, size_t bytes, k_timeout_t timeout) { TRACING_STRING("%s: %p\n", __func__, h); } diff --git a/subsys/tracing/test/tracing_test.h b/subsys/tracing/test/tracing_test.h index 8604bcd2391..a6208337e38 100644 --- a/subsys/tracing/test/tracing_test.h +++ b/subsys/tracing/test/tracing_test.h @@ -388,8 +388,8 @@ #define sys_port_trace_k_heap_init(h) sys_trace_k_heap_init(h, mem, bytes) #define sys_port_trace_k_heap_aligned_alloc_enter(h, timeout) \ sys_trace_k_heap_aligned_alloc_enter(h, bytes, timeout) -#define sys_port_trace_k_heap_aligned_alloc_blocking(h, timeout) \ - sys_trace_k_heap_aligned_alloc_blocking(h, bytes, timeout) +#define sys_port_trace_k_heap_alloc_helper_blocking(h, timeout) \ + sys_trace_k_heap_alloc_helper_blocking(h, bytes, timeout) #define sys_port_trace_k_heap_aligned_alloc_exit(h, timeout, ret) \ sys_trace_k_heap_aligned_alloc_exit(h, bytes, timeout, ret) #define sys_port_trace_k_heap_alloc_enter(h, timeout) \ @@ -697,7 +697,7 @@ void sys_trace_k_heap_calloc_enter(struct k_heap *h, size_t num, size_t size, k_ void sys_trace_k_heap_calloc_exit(struct k_heap *h, size_t num, size_t size, k_timeout_t timeout, void *ret); void sys_trace_k_heap_aligned_alloc_enter(struct k_heap *h, size_t bytes, k_timeout_t timeout); -void sys_trace_k_heap_aligned_alloc_blocking(struct k_heap *h, size_t bytes, k_timeout_t timeout); +void sys_trace_k_heap_alloc_helper_blocking(struct k_heap *h, size_t bytes, k_timeout_t timeout); void sys_trace_k_heap_aligned_alloc_exit(struct k_heap *h, size_t bytes, k_timeout_t timeout, void *ret); void sys_trace_k_heap_free(struct k_heap *h, void *mem); diff --git a/subsys/tracing/user/tracing_user.h b/subsys/tracing/user/tracing_user.h index 863a22f499b..9d1fd6f8c7b 100644 --- a/subsys/tracing/user/tracing_user.h +++ b/subsys/tracing/user/tracing_user.h @@ -352,7 +352,7 @@ void sys_trace_gpio_fire_callback_user(const struct device *port, struct gpio_ca #define sys_port_trace_k_heap_init(heap) #define sys_port_trace_k_heap_aligned_alloc_enter(heap, timeout) -#define sys_port_trace_k_heap_aligned_alloc_blocking(heap, timeout) +#define sys_port_trace_k_heap_alloc_helper_blocking(heap, timeout) #define sys_port_trace_k_heap_aligned_alloc_exit(heap, timeout, ret) #define sys_port_trace_k_heap_alloc_enter(heap, timeout) #define sys_port_trace_k_heap_alloc_exit(heap, timeout, ret)