kernel: add k_mem_map() interface
Allows applications to increase the data space available to Zephyr
via anonymous memory mappings. Loosely based on mmap().
Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
diff --git a/include/sys/mem_manage.h b/include/sys/mem_manage.h
index fdc2793..ee93f87 100644
--- a/include/sys/mem_manage.h
+++ b/include/sys/mem_manage.h
@@ -14,13 +14,13 @@
*/
/** No caching. Most drivers want this. */
-#define K_MEM_CACHE_NONE 0
+#define K_MEM_CACHE_NONE 2
/** Write-through caching. Used by certain drivers. */
#define K_MEM_CACHE_WT 1
/** Full write-back caching. Any RAM mapped wants this. */
-#define K_MEM_CACHE_WB 2
+#define K_MEM_CACHE_WB 0
/** Reserved bits for cache modes in k_map() flags argument */
#define K_MEM_CACHE_MASK (BIT(3) - 1)
@@ -94,6 +94,87 @@
void z_phys_map(uint8_t **virt_ptr, uintptr_t phys, size_t size,
uint32_t flags);
+/*
+ * k_mem_map() control flags
+ */
+
+/**
+ * @def K_MEM_MAP_UNINIT
+ *
+ * @brief The mapped region is not guaranteed to be zeroed.
+ *
+ * This may improve performance. The associated page frames may contain
+ * indeterminate data, zeroes, or even sensitive information.
+ *
+ * This may not be used with K_MEM_PERM_USER as there are no circumstances
+ * where this is safe.
+ */
+#define K_MEM_MAP_UNINIT BIT(16)
+
+/**
+ * @def K_MEM_MAP_LOCK
+ *
+ * Region will be pinned in memory and never paged
+ *
+ * Such memory is guaranteed to never produce a page fault due to page-outs
+ * or copy-on-write once the mapping call has returned. Physical page frames
+ * will be pre-fetched as necessary and pinned.
+ */
+#define K_MEM_MAP_LOCK BIT(17)
+
+/**
+ * @def K_MEM_MAP_GUARD
+ *
+ * A un-mapped virtual guard page will be placed in memory immediately preceding
+ * the mapped region. This page will still be noted as being used by the
+ * virtual memory manager. The total size of the allocation will be the
+ * requested size plus the size of this guard page. The returned address
+ * pointer will not include the guard page immediately below it. The typical
+ * use-case is downward-growing thread stacks.
+ *
+ * Zephyr treats page faults on this guard page as a fatal K_ERR_STACK_CHK_FAIL
+ * if it determines it immediately precedes a stack buffer, this is
+ * implemented in the architecture layer.
+ */
+#define K_MEM_MAP_GUARD BIT(18)
+
+/**
+ * Map anonymous memory into Zephyr's address space
+ *
+ * This function effectively increases the data space available to Zephyr.
+ * The kernel will choose a base virtual address and return it to the caller.
+ * The memory will have access permissions for all contexts set per the
+ * provided flags argument.
+ *
+ * If user thread access control needs to be managed in any way, do not enable
+ * K_MEM_PERM_USER flags here; instead manage the region's permissions
+ * with memory domain APIs after the mapping has been established. Setting
+ * K_MEM_PERM_USER here will allow all user threads to access this memory
+ * which is usually undesirable.
+ *
+ * Unless K_MEM_MAP_UNINIT is used, the returned memory will be zeroed.
+ *
+ * The mapped region is not guaranteed to be physically contiguous in memory.
+ * Physically contiguous buffers should be allocated statically and pinned
+ * at build time.
+ *
+ * Pages mapped in this way have write-back cache settings.
+ *
+ * The returned virtual memory pointer will be page-aligned. The size
+ * parameter, and any base address for re-mapping purposes must be page-
+ * aligned.
+ *
+ * Many K_MEM_MAP_* flags have been implemented to alter the behavior of this
+ * function, with details in the documentation for these flags.
+ *
+ * @param size Size of the memory mapping. This must be page-aligned.
+ * @param flags K_MEM_PERM_*, K_MEM_MAP_* control flags.
+ * @return The mapped memory location, or NULL if insufficient virtual address
+ * space, insufficient physical memory to establish the mapping,
+ * or insufficient memory for paging structures.
+ */
+void *k_mem_map(size_t size, uint32_t flags);
+
/**
* Given an arbitrary region, provide a aligned region that covers it
*
diff --git a/kernel/mmu.c b/kernel/mmu.c
index 200ae42..9dece53 100644
--- a/kernel/mmu.c
+++ b/kernel/mmu.c
@@ -252,7 +252,7 @@
* local ontology (and do some assertions while we're at it)
*/
static void frame_mapped_set(struct z_page_frame *pf, void *addr)
-{
+{
PF_ASSERT(pf, !z_page_frame_is_reserved(pf),
"attempted to map a reserved page frame");
@@ -266,9 +266,102 @@
pf->flags |= Z_PAGE_FRAME_MAPPED;
pf->addr = addr;
- pf->refcount++;
}
+/* Allocate a free page frame, and map it to a specified virtual address
+ *
+ * TODO: Add optional support for copy-on-write mappings to a zero page instead
+ * of allocating, in which case page frames will be allocated lazily as
+ * the mappings to the zero page get touched.
+ */
+static int map_anon_page(void *addr, uint32_t flags)
+{
+ int ret;
+ struct z_page_frame *pf;
+ uintptr_t phys;
+ bool lock = (flags & K_MEM_MAP_LOCK) != 0;
+
+ pf = free_page_frame_list_get();
+ if (pf == NULL) {
+ return -ENOMEM;
+ }
+ phys = z_page_frame_to_phys(pf);
+
+ ret = arch_mem_map(addr, phys, CONFIG_MMU_PAGE_SIZE,
+ flags | K_MEM_CACHE_WB);
+ if (ret != 0) {
+ free_page_frame_list_put(pf);
+ return -ENOMEM;
+ }
+ if (lock) {
+ pf->flags |= Z_PAGE_FRAME_PINNED;
+ }
+ frame_mapped_set(pf, addr);
+
+ return 0;
+}
+
+void *k_mem_map(size_t size, uint32_t flags)
+{;
+ uint8_t *dst;
+ size_t total_size = size;
+ int ret;
+ k_spinlock_key_t key;
+ bool uninit = (flags & K_MEM_MAP_UNINIT) != 0;
+ bool guard = (flags & K_MEM_MAP_GUARD) != 0;
+ uint8_t *pos;
+
+ __ASSERT(!(((flags & K_MEM_PERM_USER) != 0) && uninit),
+ "user access to anonymous uninitialized pages is forbidden");
+ __ASSERT(size % CONFIG_MMU_PAGE_SIZE == 0,
+ "unaligned size %zu passed to %s", size, __func__);
+ __ASSERT(size != 0, "zero sized memory mapping");
+ __ASSERT(page_frames_initialized, "%s called too early", __func__);
+ __ASSERT((flags & K_MEM_CACHE_MASK) == 0,
+ "%s does not support explicit cache settings", __func__);
+
+ key = k_spin_lock(&z_mm_lock);
+
+ if (guard) {
+ /* Need extra virtual page for the guard which we
+ * won't map
+ */
+ total_size += CONFIG_MMU_PAGE_SIZE;
+ }
+
+ dst = virt_region_get(total_size);
+ if (dst == NULL) {
+ /* Address space has no free region */
+ goto out;
+ }
+ if (guard) {
+ /* Skip over the guard page in returned address. */
+ dst += CONFIG_MMU_PAGE_SIZE;
+ }
+
+ VIRT_FOREACH(dst, size, pos) {
+ ret = map_anon_page(pos, flags);
+
+ if (ret != 0) {
+ /* TODO: call k_mem_unmap(dst, pos - dst) when
+ * implmented in #28990 and release any guard virtual
+ * page as well.
+ */
+ dst = NULL;
+ goto out;
+ }
+ }
+
+ if (!uninit) {
+ /* If we later implement mappings to a copy-on-write zero
+ * page, won't need this step
+ */
+ memset(dst, 0, size);
+ }
+out:
+ k_spin_unlock(&z_mm_lock, key);
+ return dst;
+}
/* This may be called from arch early boot code before z_cstart() is invoked.
* Data will be copied and BSS zeroed, but this must not rely on any
