From 01f8e0fa2daa50ddd710300d769dc9ac9dfac681 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 12 Nov 2024 14:06:11 -0800 Subject: [PATCH] demand_paging: eviction: add kconfig CONFIG_EVICTION_TRACKING This adds a new kconfig for eviction algorithm which needs page tracking. When enabled, k_mem_paging_eviction_add()/_remove() and k_mem_paging_eviction_accessed() must be implemented. If an algorithm does not do page tracking, there is no need to implement these functions, and no need for the kernel MMU code to call into empty functions. This should save a few function calls and some CPU cycles. Note that arm64 unconditionally calls those functions so forces CONFIG_EVICTION_TRACKING to be enabled there. Signed-off-by: Daniel Leung --- arch/Kconfig | 8 +++++ .../memory_management/demand_paging.rst | 8 +++-- include/zephyr/kernel/mm/demand_paging.h | 21 +++++++++++ kernel/mmu.c | 35 +++++++++++++------ subsys/demand_paging/eviction/Kconfig | 10 ++++++ subsys/demand_paging/eviction/nru.c | 7 +++- 6 files changed, 75 insertions(+), 14 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 355a6247af3..774840dda0f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -55,6 +55,8 @@ config ARM64 select ARCH_HAS_DIRECTED_IPIS select ARCH_HAS_DEMAND_PAGING select ARCH_HAS_DEMAND_MAPPING + select ARCH_SUPPORTS_EVICTION_TRACKING + select EVICTION_TRACKING if DEMAND_PAGING help ARM64 (AArch64) architecture @@ -694,6 +696,12 @@ config ARCH_SUPPORTS_ROM_START config ARCH_SUPPORTS_EMPTY_IRQ_SPURIOUS bool +config ARCH_SUPPORTS_EVICTION_TRACKING + bool + help + Architecture code supports page tracking for eviction algorithms + when demand paging is enabled. + config ARCH_HAS_EXTRA_EXCEPTION_INFO bool diff --git a/doc/kernel/memory_management/demand_paging.rst b/doc/kernel/memory_management/demand_paging.rst index db068397a14..76e962bd8e8 100644 --- a/doc/kernel/memory_management/demand_paging.rst +++ b/doc/kernel/memory_management/demand_paging.rst @@ -156,8 +156,12 @@ Two eviction algorithms are currently available: to the NRU code but also considerably more efficient. This is recommended for production use. -To implement a new eviction algorithm, the five functions mentioned -above must be implemented. +To implement a new eviction algorithm, :c:func:`k_mem_paging_eviction_init()` +and :c:func:`k_mem_paging_eviction_select()` must be implemented. +If :kconfig:option:`CONFIG_EVICTION_TRACKING` is enabled for an algorithm, +these additional functions must also be implemented, +:c:func:`k_mem_paging_eviction_add()`, :c:func:`k_mem_paging_eviction_remove()`, +:c:func:`k_mem_paging_eviction_accessed()`. Backing Store ************* diff --git a/include/zephyr/kernel/mm/demand_paging.h b/include/zephyr/kernel/mm/demand_paging.h index 120ee3299a3..10a2e7918ac 100644 --- a/include/zephyr/kernel/mm/demand_paging.h +++ b/include/zephyr/kernel/mm/demand_paging.h @@ -217,6 +217,8 @@ __syscall void k_mem_paging_histogram_backing_store_page_out_get( * @{ */ +#if defined(CONFIG_EVICTION_TRACKING) || defined(__DOXYGEN__) + /** * Submit a page frame for eviction candidate tracking * @@ -261,6 +263,25 @@ void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf); */ void k_mem_paging_eviction_accessed(uintptr_t phys); +#else /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */ + +static inline void k_mem_paging_eviction_add(struct k_mem_page_frame *pf) +{ + ARG_UNUSED(pf); +} + +static inline void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf) +{ + ARG_UNUSED(pf); +} + +static inline void k_mem_paging_eviction_accessed(uintptr_t phys) +{ + ARG_UNUSED(phys); +} + +#endif /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */ + /** * Select a page frame for eviction * diff --git a/kernel/mmu.c b/kernel/mmu.c index b03ff978786..788f30ff730 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -556,7 +556,7 @@ static int map_anon_page(void *addr, uint32_t flags) } frame_mapped_set(pf, addr); #ifdef CONFIG_DEMAND_PAGING - if (!lock) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && (!lock)) { k_mem_paging_eviction_add(pf); } #endif @@ -784,7 +784,8 @@ void k_mem_unmap_phys_guard(void *addr, size_t size, bool is_anon) arch_mem_unmap(pos, CONFIG_MMU_PAGE_SIZE); #ifdef CONFIG_DEMAND_PAGING - if (!k_mem_page_frame_is_pinned(pf)) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && + (!k_mem_page_frame_is_pinned(pf))) { k_mem_paging_eviction_remove(pf); } #endif @@ -1041,7 +1042,8 @@ static void mark_linker_section_pinned(void *start_addr, void *end_addr, } else { k_mem_page_frame_clear(pf, K_MEM_PAGE_FRAME_PINNED); #ifdef CONFIG_DEMAND_PAGING - if (k_mem_page_frame_is_evictable(pf)) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && + k_mem_page_frame_is_evictable(pf)) { k_mem_paging_eviction_add(pf); } #endif @@ -1147,10 +1149,13 @@ void z_mem_manage_init(void) #endif /* CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM */ k_mem_paging_backing_store_init(); k_mem_paging_eviction_init(); - /* start tracking evictable page installed above if any */ - K_MEM_PAGE_FRAME_FOREACH(phys, pf) { - if (k_mem_page_frame_is_evictable(pf)) { - k_mem_paging_eviction_add(pf); + + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + /* start tracking evictable page installed above if any */ + K_MEM_PAGE_FRAME_FOREACH(phys, pf) { + if (k_mem_page_frame_is_evictable(pf)) { + k_mem_paging_eviction_add(pf); + } } } #endif /* CONFIG_DEMAND_PAGING */ @@ -1347,7 +1352,10 @@ static int page_frame_prepare_locked(struct k_mem_page_frame *pf, bool *dirty_pt return -ENOMEM; } arch_mem_page_out(k_mem_page_frame_to_virt(pf), *location_ptr); - k_mem_paging_eviction_remove(pf); + + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + k_mem_paging_eviction_remove(pf); + } } else { /* Shouldn't happen unless this function is mis-used */ __ASSERT(!dirty, "un-mapped page determined to be dirty"); @@ -1683,7 +1691,9 @@ static bool do_page_fault(void *addr, bool pin) pf = k_mem_phys_to_page_frame(phys); if (!k_mem_page_frame_is_pinned(pf)) { - k_mem_paging_eviction_remove(pf); + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + k_mem_paging_eviction_remove(pf); + } k_mem_page_frame_set(pf, K_MEM_PAGE_FRAME_PINNED); } } @@ -1738,7 +1748,7 @@ static bool do_page_fault(void *addr, bool pin) arch_mem_page_in(addr, k_mem_page_frame_to_phys(pf)); k_mem_paging_backing_store_page_finalize(pf, page_in_location); - if (!pin) { + if (IS_ENABLED(CONFIG_EVICTION_TRACKING) && (!pin)) { k_mem_paging_eviction_add(pf); } out: @@ -1807,7 +1817,10 @@ static void do_mem_unpin(void *addr) pf = k_mem_phys_to_page_frame(phys); if (k_mem_page_frame_is_pinned(pf)) { k_mem_page_frame_clear(pf, K_MEM_PAGE_FRAME_PINNED); - k_mem_paging_eviction_add(pf); + + if (IS_ENABLED(CONFIG_EVICTION_TRACKING)) { + k_mem_paging_eviction_add(pf); + } } } k_spin_unlock(&z_mm_lock, key); diff --git a/subsys/demand_paging/eviction/Kconfig b/subsys/demand_paging/eviction/Kconfig index 3d4044b1fff..d920f814a0e 100644 --- a/subsys/demand_paging/eviction/Kconfig +++ b/subsys/demand_paging/eviction/Kconfig @@ -11,6 +11,7 @@ choice EVICTION_CHOICE config EVICTION_CUSTOM bool "Custom eviction algorithm" + imply EVICTION_TRACKING help This option is chosen when the eviction algorithm will be implemented by the application, instead of using one included in Zephyr. @@ -30,6 +31,7 @@ config EVICTION_NRU config EVICTION_LRU bool "Least Recently Used (LRU) page eviction algorithm" + select EVICTION_TRACKING help This implements a Least Recently Used page eviction algorithm. Usage is tracked based on MMU protection making pages unaccessible @@ -49,3 +51,11 @@ config EVICTION_NRU_PERIOD pages that are capable of being paged out. At eviction time, if a page still has the accessed property, it will be considered as recently used. endif # EVICTION_NRU + +config EVICTION_TRACKING + bool + depends on ARCH_SUPPORTS_EVICTION_TRACKING + help + Selected by eviction algorithms which needs page tracking and need to + implement the following functions: k_mem_paging_eviction_add(), + k_mem_paging_eviction_remove() and k_mem_paging_eviction_accessed(). diff --git a/subsys/demand_paging/eviction/nru.c b/subsys/demand_paging/eviction/nru.c index 2fd92ccb884..246d2d12300 100644 --- a/subsys/demand_paging/eviction/nru.c +++ b/subsys/demand_paging/eviction/nru.c @@ -111,8 +111,11 @@ void k_mem_paging_eviction_init(void) K_MSEC(CONFIG_EVICTION_NRU_PERIOD)); } +#ifdef CONFIG_EVICTION_TRACKING /* - * unused interfaces + * Empty functions defined here so that architectures unconditionally + * implement eviction tracking can still use this algorithm for + * testing. */ void k_mem_paging_eviction_add(struct k_mem_page_frame *pf) @@ -129,3 +132,5 @@ void k_mem_paging_eviction_accessed(uintptr_t phys) { ARG_UNUSED(phys); } + +#endif /* CONFIG_EVICTION_TRACKING */