diff --git a/MAINTAINERS.yml b/MAINTAINERS.yml index 02ff92cb9f8..b5718196750 100644 --- a/MAINTAINERS.yml +++ b/MAINTAINERS.yml @@ -3138,6 +3138,7 @@ Userspace: - include/zephyr/sys/mem_manage.h - include/zephyr/kernel/mm.h - include/zephyr/kernel/internal/mm.h + - include/zephyr/kernel/mm/demand_paging.h labels: - "area: Userspace" diff --git a/include/zephyr/kernel/mm.h b/include/zephyr/kernel/mm.h index b277449ea3e..1056857c4fd 100644 --- a/include/zephyr/kernel/mm.h +++ b/include/zephyr/kernel/mm.h @@ -14,6 +14,7 @@ #endif #include +#include /** * @brief Kernel Memory Management @@ -69,46 +70,6 @@ #include #include -struct k_mem_paging_stats_t { -#ifdef CONFIG_DEMAND_PAGING_STATS - struct { - /** Number of page faults */ - unsigned long cnt; - - /** Number of page faults with IRQ locked */ - unsigned long irq_locked; - - /** Number of page faults with IRQ unlocked */ - unsigned long irq_unlocked; - -#ifndef CONFIG_DEMAND_PAGING_ALLOW_IRQ - /** Number of page faults while in ISR */ - unsigned long in_isr; -#endif - } pagefaults; - - struct { - /** Number of clean pages selected for eviction */ - unsigned long clean; - - /** Number of dirty pages selected for eviction */ - unsigned long dirty; - } eviction; -#endif /* CONFIG_DEMAND_PAGING_STATS */ -}; - -struct k_mem_paging_histogram_t { -#ifdef CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM - /* Counts for each bin in timing histogram */ - unsigned long counts[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS]; - - /* Bounds for the bins in timing histogram, - * excluding the first and last (hence, NUM_SLOTS - 1). - */ - unsigned long bounds[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS]; -#endif /* CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM */ -}; - #ifdef __cplusplus extern "C" { #endif @@ -222,304 +183,6 @@ void k_mem_unmap(void *addr, size_t size); size_t k_mem_region_align(uintptr_t *aligned_addr, size_t *aligned_size, uintptr_t addr, size_t size, size_t align); -/** - * @defgroup demand_paging Demand Paging - * @ingroup kernel_memory_management - */ -/** - * @defgroup mem-demand-paging Demand Paging APIs - * @ingroup demand_paging - * @{ - */ - -/** - * Evict a page-aligned virtual memory region to the backing store - * - * Useful if it is known that a memory region will not be used for some time. - * All the data pages within the specified region will be evicted to the - * backing store if they weren't already, with their associated page frames - * marked as available for mappings or page-ins. - * - * None of the associated page frames mapped to the provided region should - * be pinned. - * - * Note that there are no guarantees how long these pages will be evicted, - * they could take page faults immediately. - * - * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be - * called by ISRs as the backing store may be in-use. - * - * @param addr Base page-aligned virtual address - * @param size Page-aligned data region size - * @retval 0 Success - * @retval -ENOMEM Insufficient space in backing store to satisfy request. - * The region may be partially paged out. - */ -int k_mem_page_out(void *addr, size_t size); - -/** - * Load a virtual data region into memory - * - * After the function completes, all the page frames associated with this - * function will be paged in. However, they are not guaranteed to stay there. - * This is useful if the region is known to be used soon. - * - * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be - * called by ISRs as the backing store may be in-use. - * - * @param addr Base page-aligned virtual address - * @param size Page-aligned data region size - */ -void k_mem_page_in(void *addr, size_t size); - -/** - * Pin an aligned virtual data region, paging in as necessary - * - * After the function completes, all the page frames associated with this - * region will be resident in memory and pinned such that they stay that way. - * This is a stronger version of z_mem_page_in(). - * - * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be - * called by ISRs as the backing store may be in-use. - * - * @param addr Base page-aligned virtual address - * @param size Page-aligned data region size - */ -void k_mem_pin(void *addr, size_t size); - -/** - * Un-pin an aligned virtual data region - * - * After the function completes, all the page frames associated with this - * region will be no longer marked as pinned. This does not evict the region, - * follow this with z_mem_page_out() if you need that. - * - * @param addr Base page-aligned virtual address - * @param size Page-aligned data region size - */ -void k_mem_unpin(void *addr, size_t size); - -/** - * Get the paging statistics since system startup - * - * This populates the paging statistics struct being passed in - * as argument. - * - * @param[in,out] stats Paging statistics struct to be filled. - */ -__syscall void k_mem_paging_stats_get(struct k_mem_paging_stats_t *stats); - -struct k_thread; -/** - * Get the paging statistics since system startup for a thread - * - * This populates the paging statistics struct being passed in - * as argument for a particular thread. - * - * @param[in] thread Thread - * @param[in,out] stats Paging statistics struct to be filled. - */ -__syscall -void k_mem_paging_thread_stats_get(struct k_thread *thread, - struct k_mem_paging_stats_t *stats); - -/** - * Get the eviction timing histogram - * - * This populates the timing histogram struct being passed in - * as argument. - * - * @param[in,out] hist Timing histogram struct to be filled. - */ -__syscall void k_mem_paging_histogram_eviction_get( - struct k_mem_paging_histogram_t *hist); - -/** - * Get the backing store page-in timing histogram - * - * This populates the timing histogram struct being passed in - * as argument. - * - * @param[in,out] hist Timing histogram struct to be filled. - */ -__syscall void k_mem_paging_histogram_backing_store_page_in_get( - struct k_mem_paging_histogram_t *hist); - -/** - * Get the backing store page-out timing histogram - * - * This populates the timing histogram struct being passed in - * as argument. - * - * @param[in,out] hist Timing histogram struct to be filled. - */ -__syscall void k_mem_paging_histogram_backing_store_page_out_get( - struct k_mem_paging_histogram_t *hist); - -#include - -/** @} */ - -/** - * Eviction algorithm APIs - * - * @defgroup mem-demand-paging-eviction Eviction Algorithm APIs - * @ingroup demand_paging - * @{ - */ - -/** - * Select a page frame for eviction - * - * The kernel will invoke this to choose a page frame to evict if there - * are no free page frames. - * - * This function will never be called before the initial - * k_mem_paging_eviction_init(). - * - * This function is invoked with interrupts locked. - * - * @param [out] dirty Whether the page to evict is dirty - * @return The page frame to evict - */ -struct z_page_frame *k_mem_paging_eviction_select(bool *dirty); - -/** - * Initialization function - * - * Called at POST_KERNEL to perform any necessary initialization tasks for the - * eviction algorithm. k_mem_paging_eviction_select() is guaranteed to never be - * called until this has returned, and this will only be called once. - */ -void k_mem_paging_eviction_init(void); - -/** @} */ - -/** - * Backing store APIs - * - * @defgroup mem-demand-paging-backing-store Backing Store APIs - * @ingroup demand_paging - * @{ - */ - -/** - * Reserve or fetch a storage location for a data page loaded into a page frame - * - * The returned location token must be unique to the mapped virtual address. - * This location will be used in the backing store to page out data page - * contents for later retrieval. The location value must be page-aligned. - * - * This function may be called multiple times on the same data page. If its - * page frame has its Z_PAGE_FRAME_BACKED bit set, it is expected to return - * the previous backing store location for the data page containing a cached - * clean copy. This clean copy may be updated on page-out, or used to - * discard clean pages without needing to write out their contents. - * - * If the backing store is full, some other backing store location which caches - * a loaded data page may be selected, in which case its associated page frame - * will have the Z_PAGE_FRAME_BACKED bit cleared (as it is no longer cached). - * - * pf->addr will indicate the virtual address the page is currently mapped to. - * Large, sparse backing stores which can contain the entire address space - * may simply generate location tokens purely as a function of pf->addr with no - * other management necessary. - * - * This function distinguishes whether it was called on behalf of a page - * fault. A free backing store location must always be reserved in order for - * page faults to succeed. If the page_fault parameter is not set, this - * function should return -ENOMEM even if one location is available. - * - * This function is invoked with interrupts locked. - * - * @param pf Virtual address to obtain a storage location - * @param [out] location storage location token - * @param page_fault Whether this request was for a page fault - * @return 0 Success - * @return -ENOMEM Backing store is full - */ -int k_mem_paging_backing_store_location_get(struct z_page_frame *pf, - uintptr_t *location, - bool page_fault); - -/** - * Free a backing store location - * - * Any stored data may be discarded, and the location token associated with - * this address may be re-used for some other data page. - * - * This function is invoked with interrupts locked. - * - * @param location Location token to free - */ -void k_mem_paging_backing_store_location_free(uintptr_t location); - -/** - * Copy a data page from Z_SCRATCH_PAGE to the specified location - * - * Immediately before this is called, Z_SCRATCH_PAGE will be mapped read-write - * to the intended source page frame for the calling context. - * - * Calls to this and k_mem_paging_backing_store_page_in() will always be - * serialized, but interrupts may be enabled. - * - * @param location Location token for the data page, for later retrieval - */ -void k_mem_paging_backing_store_page_out(uintptr_t location); - -/** - * Copy a data page from the provided location to Z_SCRATCH_PAGE. - * - * Immediately before this is called, Z_SCRATCH_PAGE will be mapped read-write - * to the intended destination page frame for the calling context. - * - * Calls to this and k_mem_paging_backing_store_page_out() will always be - * serialized, but interrupts may be enabled. - * - * @param location Location token for the data page - */ -void k_mem_paging_backing_store_page_in(uintptr_t location); - -/** - * Update internal accounting after a page-in - * - * This is invoked after k_mem_paging_backing_store_page_in() and interrupts - * have been* re-locked, making it safe to access the z_page_frame data. - * The location value will be the same passed to - * k_mem_paging_backing_store_page_in(). - * - * The primary use-case for this is to update custom fields for the backing - * store in the page frame, to reflect where the data should be evicted to - * if it is paged out again. This may be a no-op in some implementations. - * - * If the backing store caches paged-in data pages, this is the appropriate - * time to set the Z_PAGE_FRAME_BACKED bit. The kernel only skips paging - * out clean data pages if they are noted as clean in the page tables and the - * Z_PAGE_FRAME_BACKED bit is set in their associated page frame. - * - * @param pf Page frame that was loaded in - * @param location Location of where the loaded data page was retrieved - */ -void k_mem_paging_backing_store_page_finalize(struct z_page_frame *pf, - uintptr_t location); - -/** - * Backing store initialization function. - * - * The implementation may expect to receive page in/out calls as soon as this - * returns, but not before that. Called at POST_KERNEL. - * - * This function is expected to do two things: - * - Initialize any internal data structures and accounting for the backing - * store. - * - If the backing store already contains all or some loaded kernel data pages - * at boot time, Z_PAGE_FRAME_BACKED should be appropriately set for their - * associated page frames, and any internal accounting set up appropriately. - */ -void k_mem_paging_backing_store_init(void); - -/** @} */ - #ifdef __cplusplus } #endif diff --git a/include/zephyr/kernel/mm/demand_paging.h b/include/zephyr/kernel/mm/demand_paging.h new file mode 100644 index 00000000000..fb20a2c734f --- /dev/null +++ b/include/zephyr/kernel/mm/demand_paging.h @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2020 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H +#define ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H + +#include + +#include +#include + +/** + * @defgroup demand_paging Demand Paging + * @ingroup kernel_memory_management + */ + +/** + * @defgroup mem-demand-paging Demand Paging APIs + * @ingroup demand_paging + * @{ + */ + +#ifndef _ASMLANGUAGE +#include +#include +#include +#include + +struct k_mem_paging_stats_t { +#ifdef CONFIG_DEMAND_PAGING_STATS + struct { + /** Number of page faults */ + unsigned long cnt; + + /** Number of page faults with IRQ locked */ + unsigned long irq_locked; + + /** Number of page faults with IRQ unlocked */ + unsigned long irq_unlocked; + +#ifndef CONFIG_DEMAND_PAGING_ALLOW_IRQ + /** Number of page faults while in ISR */ + unsigned long in_isr; +#endif + } pagefaults; + + struct { + /** Number of clean pages selected for eviction */ + unsigned long clean; + + /** Number of dirty pages selected for eviction */ + unsigned long dirty; + } eviction; +#endif /* CONFIG_DEMAND_PAGING_STATS */ +}; + +struct k_mem_paging_histogram_t { +#ifdef CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM + /* Counts for each bin in timing histogram */ + unsigned long counts[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS]; + + /* Bounds for the bins in timing histogram, + * excluding the first and last (hence, NUM_SLOTS - 1). + */ + unsigned long bounds[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS]; +#endif /* CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM */ +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Evict a page-aligned virtual memory region to the backing store + * + * Useful if it is known that a memory region will not be used for some time. + * All the data pages within the specified region will be evicted to the + * backing store if they weren't already, with their associated page frames + * marked as available for mappings or page-ins. + * + * None of the associated page frames mapped to the provided region should + * be pinned. + * + * Note that there are no guarantees how long these pages will be evicted, + * they could take page faults immediately. + * + * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be + * called by ISRs as the backing store may be in-use. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + * @retval 0 Success + * @retval -ENOMEM Insufficient space in backing store to satisfy request. + * The region may be partially paged out. + */ +int k_mem_page_out(void *addr, size_t size); + +/** + * Load a virtual data region into memory + * + * After the function completes, all the page frames associated with this + * function will be paged in. However, they are not guaranteed to stay there. + * This is useful if the region is known to be used soon. + * + * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be + * called by ISRs as the backing store may be in-use. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + */ +void k_mem_page_in(void *addr, size_t size); + +/** + * Pin an aligned virtual data region, paging in as necessary + * + * After the function completes, all the page frames associated with this + * region will be resident in memory and pinned such that they stay that way. + * This is a stronger version of z_mem_page_in(). + * + * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be + * called by ISRs as the backing store may be in-use. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + */ +void k_mem_pin(void *addr, size_t size); + +/** + * Un-pin an aligned virtual data region + * + * After the function completes, all the page frames associated with this + * region will be no longer marked as pinned. This does not evict the region, + * follow this with z_mem_page_out() if you need that. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + */ +void k_mem_unpin(void *addr, size_t size); + +/** + * Get the paging statistics since system startup + * + * This populates the paging statistics struct being passed in + * as argument. + * + * @param[in,out] stats Paging statistics struct to be filled. + */ +__syscall void k_mem_paging_stats_get(struct k_mem_paging_stats_t *stats); + +struct k_thread; +/** + * Get the paging statistics since system startup for a thread + * + * This populates the paging statistics struct being passed in + * as argument for a particular thread. + * + * @param[in] thread Thread + * @param[in,out] stats Paging statistics struct to be filled. + */ +__syscall +void k_mem_paging_thread_stats_get(struct k_thread *thread, + struct k_mem_paging_stats_t *stats); + +/** + * Get the eviction timing histogram + * + * This populates the timing histogram struct being passed in + * as argument. + * + * @param[in,out] hist Timing histogram struct to be filled. + */ +__syscall void k_mem_paging_histogram_eviction_get( + struct k_mem_paging_histogram_t *hist); + +/** + * Get the backing store page-in timing histogram + * + * This populates the timing histogram struct being passed in + * as argument. + * + * @param[in,out] hist Timing histogram struct to be filled. + */ +__syscall void k_mem_paging_histogram_backing_store_page_in_get( + struct k_mem_paging_histogram_t *hist); + +/** + * Get the backing store page-out timing histogram + * + * This populates the timing histogram struct being passed in + * as argument. + * + * @param[in,out] hist Timing histogram struct to be filled. + */ +__syscall void k_mem_paging_histogram_backing_store_page_out_get( + struct k_mem_paging_histogram_t *hist); + +#include + +/** @} */ + +/** + * Eviction algorithm APIs + * + * @defgroup mem-demand-paging-eviction Eviction Algorithm APIs + * @ingroup demand_paging + * @{ + */ + +/** + * Select a page frame for eviction + * + * The kernel will invoke this to choose a page frame to evict if there + * are no free page frames. + * + * This function will never be called before the initial + * k_mem_paging_eviction_init(). + * + * This function is invoked with interrupts locked. + * + * @param [out] dirty Whether the page to evict is dirty + * @return The page frame to evict + */ +struct z_page_frame *k_mem_paging_eviction_select(bool *dirty); + +/** + * Initialization function + * + * Called at POST_KERNEL to perform any necessary initialization tasks for the + * eviction algorithm. k_mem_paging_eviction_select() is guaranteed to never be + * called until this has returned, and this will only be called once. + */ +void k_mem_paging_eviction_init(void); + +/** @} */ + +/** + * Backing store APIs + * + * @defgroup mem-demand-paging-backing-store Backing Store APIs + * @ingroup demand_paging + * @{ + */ + +/** + * Reserve or fetch a storage location for a data page loaded into a page frame + * + * The returned location token must be unique to the mapped virtual address. + * This location will be used in the backing store to page out data page + * contents for later retrieval. The location value must be page-aligned. + * + * This function may be called multiple times on the same data page. If its + * page frame has its Z_PAGE_FRAME_BACKED bit set, it is expected to return + * the previous backing store location for the data page containing a cached + * clean copy. This clean copy may be updated on page-out, or used to + * discard clean pages without needing to write out their contents. + * + * If the backing store is full, some other backing store location which caches + * a loaded data page may be selected, in which case its associated page frame + * will have the Z_PAGE_FRAME_BACKED bit cleared (as it is no longer cached). + * + * pf->addr will indicate the virtual address the page is currently mapped to. + * Large, sparse backing stores which can contain the entire address space + * may simply generate location tokens purely as a function of pf->addr with no + * other management necessary. + * + * This function distinguishes whether it was called on behalf of a page + * fault. A free backing store location must always be reserved in order for + * page faults to succeed. If the page_fault parameter is not set, this + * function should return -ENOMEM even if one location is available. + * + * This function is invoked with interrupts locked. + * + * @param pf Virtual address to obtain a storage location + * @param [out] location storage location token + * @param page_fault Whether this request was for a page fault + * @return 0 Success + * @return -ENOMEM Backing store is full + */ +int k_mem_paging_backing_store_location_get(struct z_page_frame *pf, + uintptr_t *location, + bool page_fault); + +/** + * Free a backing store location + * + * Any stored data may be discarded, and the location token associated with + * this address may be re-used for some other data page. + * + * This function is invoked with interrupts locked. + * + * @param location Location token to free + */ +void k_mem_paging_backing_store_location_free(uintptr_t location); + +/** + * Copy a data page from Z_SCRATCH_PAGE to the specified location + * + * Immediately before this is called, Z_SCRATCH_PAGE will be mapped read-write + * to the intended source page frame for the calling context. + * + * Calls to this and k_mem_paging_backing_store_page_in() will always be + * serialized, but interrupts may be enabled. + * + * @param location Location token for the data page, for later retrieval + */ +void k_mem_paging_backing_store_page_out(uintptr_t location); + +/** + * Copy a data page from the provided location to Z_SCRATCH_PAGE. + * + * Immediately before this is called, Z_SCRATCH_PAGE will be mapped read-write + * to the intended destination page frame for the calling context. + * + * Calls to this and k_mem_paging_backing_store_page_out() will always be + * serialized, but interrupts may be enabled. + * + * @param location Location token for the data page + */ +void k_mem_paging_backing_store_page_in(uintptr_t location); + +/** + * Update internal accounting after a page-in + * + * This is invoked after k_mem_paging_backing_store_page_in() and interrupts + * have been* re-locked, making it safe to access the z_page_frame data. + * The location value will be the same passed to + * k_mem_paging_backing_store_page_in(). + * + * The primary use-case for this is to update custom fields for the backing + * store in the page frame, to reflect where the data should be evicted to + * if it is paged out again. This may be a no-op in some implementations. + * + * If the backing store caches paged-in data pages, this is the appropriate + * time to set the Z_PAGE_FRAME_BACKED bit. The kernel only skips paging + * out clean data pages if they are noted as clean in the page tables and the + * Z_PAGE_FRAME_BACKED bit is set in their associated page frame. + * + * @param pf Page frame that was loaded in + * @param location Location of where the loaded data page was retrieved + */ +void k_mem_paging_backing_store_page_finalize(struct z_page_frame *pf, + uintptr_t location); + +/** + * Backing store initialization function. + * + * The implementation may expect to receive page in/out calls as soon as this + * returns, but not before that. Called at POST_KERNEL. + * + * This function is expected to do two things: + * - Initialize any internal data structures and accounting for the backing + * store. + * - If the backing store already contains all or some loaded kernel data pages + * at boot time, Z_PAGE_FRAME_BACKED should be appropriately set for their + * associated page frames, and any internal accounting set up appropriately. + */ +void k_mem_paging_backing_store_init(void); + +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* !_ASMLANGUAGE */ +#endif /* ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H */ diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 367b83ab04a..bec410968cc 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -25,6 +25,11 @@ zephyr_syscall_header_ifdef( ${ZEPHYR_BASE}/include/zephyr/sys/mem_manage.h ) +zephyr_syscall_header_ifdef( + CONFIG_DEMAND_PAGING + ${ZEPHYR_BASE}/include/zephyr/kernel/mm/demand_paging.h +) + # If a pre-built static library containing kernel code exists in # this directory, libkernel.a, link it with the application code # instead of building from source.