| /* |
| * Copyright (c) 2011-2014 Wind River Systems, Inc. |
| * Copyright (c) 2017 Intel Corporation |
| * |
| * SPDX-License-Identifier: Apache-2.0 |
| */ |
| #include <kernel.h> |
| #include <arch/x86/mmustructs.h> |
| #include <linker/linker-defs.h> |
| #include <kernel_internal.h> |
| #include <kernel_structs.h> |
| #include <init.h> |
| #include <ctype.h> |
| #include <string.h> |
| |
| /* Despite our use of PAE page tables, we do not (and will never) actually |
| * support PAE. Use a 64-bit x86 target if you have that much RAM. |
| */ |
| BUILD_ASSERT(DT_PHYS_RAM_ADDR + (DT_RAM_SIZE * 1024ULL) - 1ULL <= |
| (unsigned long long)UINTPTR_MAX); |
| |
| /* Common regions for all x86 processors. |
| * Peripheral I/O ranges configured at the SOC level |
| */ |
| |
| /* Mark text and rodata as read-only. |
| * Userspace may read all text and rodata. |
| */ |
| MMU_BOOT_REGION(&_image_text_start, &_image_text_size, |
| Z_X86_MMU_US); |
| |
| MMU_BOOT_REGION(&_image_rodata_start, &_image_rodata_size, |
| Z_X86_MMU_US | Z_X86_MMU_XD); |
| |
| #ifdef CONFIG_USERSPACE |
| MMU_BOOT_REGION(&_app_smem_start, &_app_smem_size, |
| Z_X86_MMU_RW | Z_X86_MMU_XD); |
| #endif |
| |
| #ifdef CONFIG_COVERAGE_GCOV |
| MMU_BOOT_REGION(&__gcov_bss_start, &__gcov_bss_size, |
| Z_X86_MMU_RW | Z_X86_MMU_US | Z_X86_MMU_XD); |
| #endif |
| |
| #ifdef CONFIG_X86_64 |
| extern char _locore_start[]; |
| extern char _locore_size[]; |
| extern char _lorodata_start[]; |
| extern char _lorodata_size[]; |
| extern char _lodata_start[]; |
| extern char _lodata_size[]; |
| |
| /* Early boot regions that need to be in low memory to be comprehensible |
| * by the CPU in 16-bit mode |
| */ |
| |
| MMU_BOOT_REGION(&_locore_start, &_locore_size, 0); |
| MMU_BOOT_REGION(&_lorodata_start, &_lorodata_size, Z_X86_MMU_XD); |
| MMU_BOOT_REGION(&_lodata_start, &_lodata_size, Z_X86_MMU_RW | Z_X86_MMU_XD); |
| #endif |
| |
| /* __kernel_ram_size includes all unused memory, which is used for heaps. |
| * User threads cannot access this unless granted at runtime. This is done |
| * automatically for stacks. |
| */ |
| MMU_BOOT_REGION(&__kernel_ram_start, &__kernel_ram_size, |
| Z_X86_MMU_RW | Z_X86_MMU_XD); |
| |
| /* |
| * Inline functions for setting memory addresses in page table structures |
| */ |
| |
| #ifdef CONFIG_X86_64 |
| static inline void pml4e_update_pdpt(u64_t *pml4e, struct x86_mmu_pdpt *pdpt) |
| { |
| uintptr_t pdpt_addr = (uintptr_t)pdpt; |
| |
| *pml4e = ((*pml4e & ~Z_X86_MMU_PML4E_PDPT_MASK) | |
| (pdpt_addr & Z_X86_MMU_PML4E_PDPT_MASK)); |
| } |
| #endif /* CONFIG_X86_64 */ |
| |
| static inline void pdpte_update_pd(u64_t *pdpte, struct x86_mmu_pd *pd) |
| { |
| uintptr_t pd_addr = (uintptr_t)pd; |
| |
| #ifdef CONFIG_X86_64 |
| __ASSERT((*pdpte & Z_X86_MMU_PS) == 0, "PDPT is for 1GB page"); |
| #endif |
| *pdpte = ((*pdpte & ~Z_X86_MMU_PDPTE_PD_MASK) | |
| (pd_addr & Z_X86_MMU_PDPTE_PD_MASK)); |
| } |
| |
| static inline void pde_update_pt(u64_t *pde, struct x86_mmu_pt *pt) |
| { |
| uintptr_t pt_addr = (uintptr_t)pt; |
| |
| __ASSERT((*pde & Z_X86_MMU_PS) == 0, "pde is for 2MB page"); |
| |
| *pde = ((*pde & ~Z_X86_MMU_PDE_PT_MASK) | |
| (pt_addr & Z_X86_MMU_PDE_PT_MASK)); |
| } |
| |
| static inline void pte_update_addr(u64_t *pte, uintptr_t addr) |
| { |
| *pte = ((*pte & ~Z_X86_MMU_PTE_ADDR_MASK) | |
| (addr & Z_X86_MMU_PTE_ADDR_MASK)); |
| } |
| |
| /* |
| * Functions for dumping page tables to console |
| */ |
| |
| /* Works for PDPT, PD, PT entries, the bits we check here are all the same. |
| * |
| * Not trying to capture every flag, just the most interesting stuff, |
| * Present, write, XD, user, in typically encountered combinations. |
| */ |
| static char get_entry_code(u64_t value) |
| { |
| char ret; |
| |
| if ((value & Z_X86_MMU_P) == 0) { |
| ret = '.'; |
| } else { |
| if ((value & Z_X86_MMU_RW) != 0) { |
| /* Writable page */ |
| if ((value & Z_X86_MMU_XD) != 0) { |
| /* RW */ |
| ret = 'w'; |
| } else { |
| /* RWX */ |
| ret = 'a'; |
| } |
| } else { |
| if ((value & Z_X86_MMU_XD) != 0) { |
| /* R */ |
| ret = 'r'; |
| } else { |
| /* RX */ |
| ret = 'x'; |
| } |
| } |
| |
| if ((value & Z_X86_MMU_US) != 0) { |
| /* Uppercase indicates user mode access */ |
| ret = toupper(ret); |
| } |
| } |
| |
| return ret; |
| } |
| |
| static void print_entries(u64_t entries_array[], size_t count) |
| { |
| int column = 0; |
| |
| for (int i = 0; i < count; i++) { |
| printk("%c", get_entry_code(entries_array[i])); |
| |
| column++; |
| if (column == 64) { |
| column = 0; |
| printk("\n"); |
| } |
| } |
| |
| if (column != 0) { |
| printk("\n"); |
| } |
| } |
| |
| static void z_x86_dump_pt(struct x86_mmu_pt *pt, uintptr_t base, int index) |
| { |
| printk("Page table %d for 0x%016lX - 0x%016lX at %p\n", |
| index, base, base + Z_X86_PT_AREA - 1, pt); |
| |
| print_entries(pt->entry, Z_X86_NUM_PT_ENTRIES); |
| } |
| |
| static void z_x86_dump_pd(struct x86_mmu_pd *pd, uintptr_t base, int index) |
| { |
| printk("Page directory %d for 0x%016lX - 0x%016lX at %p\n", |
| index, base, base + Z_X86_PD_AREA - 1, pd); |
| |
| print_entries(pd->entry, Z_X86_NUM_PD_ENTRIES); |
| |
| for (int i = 0; i < Z_X86_NUM_PD_ENTRIES; i++) { |
| struct x86_mmu_pt *pt; |
| u64_t pde = pd->entry[i]; |
| |
| if (((pde & Z_X86_MMU_P) == 0) || ((pde & Z_X86_MMU_PS) != 0)) { |
| /* Skip non-present, or 2MB directory entries, there's |
| * no page table to examine */ |
| continue; |
| } |
| pt = z_x86_pde_get_pt(pde); |
| |
| z_x86_dump_pt(pt, base + (i * Z_X86_PT_AREA), i); |
| } |
| } |
| |
| static void z_x86_dump_pdpt(struct x86_mmu_pdpt *pdpt, uintptr_t base, |
| int index) |
| { |
| printk("Page directory pointer table %d for 0x%0816lX - 0x%016lX at %p\n", |
| index, base, base + Z_X86_PDPT_AREA - 1, pdpt); |
| |
| print_entries(pdpt->entry, Z_X86_NUM_PDPT_ENTRIES); |
| |
| for (int i = 0; i < Z_X86_NUM_PDPT_ENTRIES; i++) { |
| struct x86_mmu_pd *pd; |
| u64_t pdpte = pdpt->entry[i]; |
| |
| if ((pdpte & Z_X86_MMU_P) == 0) { |
| continue; |
| } |
| #ifdef CONFIG_X86_64 |
| if ((pdpte & Z_X86_MMU_PS) != 0) { |
| continue; |
| } |
| #endif |
| pd = z_x86_pdpte_get_pd(pdpte); |
| z_x86_dump_pd(pd, base + (i * Z_X86_PD_AREA), i); |
| } |
| } |
| |
| #ifdef CONFIG_X86_64 |
| static void z_x86_dump_pml4(struct x86_mmu_pml4 *pml4) |
| { |
| printk("Page mapping level 4 at %p for all memory addresses\n", pml4); |
| |
| print_entries(pml4->entry, Z_X86_NUM_PML4_ENTRIES); |
| |
| for (int i = 0; i < Z_X86_NUM_PML4_ENTRIES; i++) { |
| struct x86_mmu_pdpt *pdpt; |
| u64_t pml4e = pml4->entry[i]; |
| |
| if ((pml4e & Z_X86_MMU_P) == 0) { |
| continue; |
| } |
| |
| pdpt = z_x86_pml4e_get_pdpt(pml4e); |
| z_x86_dump_pdpt(pdpt, i * Z_X86_PDPT_AREA, i); |
| } |
| } |
| |
| void z_x86_dump_page_tables(struct x86_page_tables *ptables) |
| { |
| z_x86_dump_pml4(z_x86_get_pml4(ptables)); |
| } |
| |
| #else |
| void z_x86_dump_page_tables(struct x86_page_tables *ptables) |
| { |
| z_x86_dump_pdpt(z_x86_get_pdpt(ptables, 0), 0, 0); |
| } |
| #endif |
| |
| void z_x86_mmu_get_flags(struct x86_page_tables *ptables, void *addr, |
| u64_t *pde_flags, u64_t *pte_flags) |
| { |
| *pde_flags = *z_x86_get_pde(ptables, (uintptr_t)addr) & |
| ~Z_X86_MMU_PDE_PT_MASK; |
| |
| if ((*pde_flags & Z_X86_MMU_P) != 0) { |
| *pte_flags = *z_x86_get_pte(ptables, (uintptr_t)addr) & |
| ~Z_X86_MMU_PTE_ADDR_MASK; |
| } else { |
| *pte_flags = 0; |
| } |
| } |
| |
| /* Given an address/size pair, which corresponds to some memory address |
| * within a table of table_size, return the maximum number of bytes to |
| * examine so we look just to the end of the table and no further. |
| * |
| * If size fits entirely within the table, just return size. |
| */ |
| static size_t get_table_max(uintptr_t addr, size_t size, size_t table_size) |
| { |
| size_t table_remaining; |
| |
| addr &= (table_size - 1); |
| table_remaining = table_size - addr; |
| |
| if (size < table_remaining) { |
| return size; |
| } else { |
| return table_remaining; |
| } |
| } |
| |
| /* Range [addr, addr + size) must fall within the bounds of the pt */ |
| static int x86_mmu_validate_pt(struct x86_mmu_pt *pt, uintptr_t addr, |
| size_t size, bool write) |
| { |
| uintptr_t pos = addr; |
| size_t remaining = size; |
| int ret = 0; |
| |
| while (true) { |
| u64_t pte = *z_x86_pt_get_pte(pt, pos); |
| |
| if ((pte & Z_X86_MMU_P) == 0 || (pte & Z_X86_MMU_US) == 0 || |
| (write && (pte & Z_X86_MMU_RW) == 0)) { |
| ret = -1; |
| break; |
| } |
| |
| if (remaining <= MMU_PAGE_SIZE) { |
| break; |
| } |
| |
| remaining -= MMU_PAGE_SIZE; |
| pos += MMU_PAGE_SIZE; |
| } |
| |
| return ret; |
| } |
| |
| /* Range [addr, addr + size) must fall within the bounds of the pd */ |
| static int x86_mmu_validate_pd(struct x86_mmu_pd *pd, uintptr_t addr, |
| size_t size, bool write) |
| { |
| uintptr_t pos = addr; |
| size_t remaining = size; |
| int ret = 0; |
| size_t to_examine; |
| |
| while (remaining) { |
| u64_t pde = *z_x86_pd_get_pde(pd, pos); |
| |
| if ((pde & Z_X86_MMU_P) == 0 || (pde & Z_X86_MMU_US) == 0 || |
| (write && (pde & Z_X86_MMU_RW) == 0)) { |
| ret = -1; |
| break; |
| } |
| |
| to_examine = get_table_max(pos, remaining, Z_X86_PT_AREA); |
| |
| if ((pde & Z_X86_MMU_PS) == 0) { |
| /* Not a 2MB PDE. Need to check all the linked |
| * tables for this entry |
| */ |
| struct x86_mmu_pt *pt; |
| |
| pt = z_x86_pde_get_pt(pde); |
| ret = x86_mmu_validate_pt(pt, pos, to_examine, write); |
| if (ret != 0) { |
| break; |
| } |
| } else { |
| ret = 0; |
| } |
| |
| remaining -= to_examine; |
| pos += to_examine; |
| } |
| |
| return ret; |
| } |
| |
| /* Range [addr, addr + size) must fall within the bounds of the pdpt */ |
| static int x86_mmu_validate_pdpt(struct x86_mmu_pdpt *pdpt, uintptr_t addr, |
| size_t size, bool write) |
| { |
| uintptr_t pos = addr; |
| size_t remaining = size; |
| int ret = 0; |
| size_t to_examine; |
| |
| while (remaining) { |
| u64_t pdpte = *z_x86_pdpt_get_pdpte(pdpt, pos); |
| |
| if ((pdpte & Z_X86_MMU_P) == 0) { |
| /* Non-present */ |
| ret = -1; |
| break; |
| } |
| |
| #ifdef CONFIG_X86_64 |
| if ((pdpte & Z_X86_MMU_US) == 0 || |
| (write && (pdpte & Z_X86_MMU_RW) == 0)) { |
| ret = -1; |
| break; |
| } |
| #endif |
| to_examine = get_table_max(pos, remaining, Z_X86_PD_AREA); |
| |
| #ifdef CONFIG_X86_64 |
| /* Check if 1GB page, if not, examine linked page directory */ |
| if ((pdpte & Z_X86_MMU_PS) == 0) { |
| #endif |
| struct x86_mmu_pd *pd = z_x86_pdpte_get_pd(pdpte); |
| |
| ret = x86_mmu_validate_pd(pd, pos, to_examine, write); |
| if (ret != 0) { |
| break; |
| } |
| #ifdef CONFIG_X86_64 |
| } else { |
| ret = 0; |
| } |
| #endif |
| remaining -= to_examine; |
| pos += to_examine; |
| } |
| |
| return ret; |
| } |
| |
| #ifdef CONFIG_X86_64 |
| static int x86_mmu_validate_pml4(struct x86_mmu_pml4 *pml4, uintptr_t addr, |
| size_t size, bool write) |
| { |
| uintptr_t pos = addr; |
| size_t remaining = size; |
| int ret = 0; |
| size_t to_examine; |
| |
| while (remaining) { |
| u64_t pml4e = *z_x86_pml4_get_pml4e(pml4, pos); |
| struct x86_mmu_pdpt *pdpt; |
| |
| if ((pml4e & Z_X86_MMU_P) == 0 || (pml4e & Z_X86_MMU_US) == 0 || |
| (write && (pml4e & Z_X86_MMU_RW) == 0)) { |
| ret = -1; |
| break; |
| } |
| |
| to_examine = get_table_max(pos, remaining, Z_X86_PDPT_AREA); |
| pdpt = z_x86_pml4e_get_pdpt(pml4e); |
| |
| ret = x86_mmu_validate_pdpt(pdpt, pos, to_examine, write); |
| if (ret != 0) { |
| break; |
| } |
| |
| remaining -= to_examine; |
| pos += to_examine; |
| } |
| |
| return ret; |
| } |
| #endif /* CONFIG_X86_64 */ |
| |
| int z_x86_mmu_validate(struct x86_page_tables *ptables, void *addr, size_t size, |
| bool write) |
| { |
| int ret; |
| |
| #ifdef CONFIG_X86_64 |
| struct x86_mmu_pml4 *pml4 = z_x86_get_pml4(ptables); |
| |
| ret = x86_mmu_validate_pml4(pml4, (uintptr_t)addr, size, write); |
| #else |
| struct x86_mmu_pdpt *pdpt = z_x86_get_pdpt(ptables, (uintptr_t)addr); |
| |
| ret = x86_mmu_validate_pdpt(pdpt, (uintptr_t)addr, size, write); |
| #endif |
| |
| #ifdef CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION |
| __asm__ volatile ("lfence" : : : "memory"); |
| #endif |
| |
| return ret; |
| } |
| |
| static inline void tlb_flush_page(void *addr) |
| { |
| /* Invalidate TLB entries corresponding to the page containing the |
| * specified address |
| */ |
| char *page = (char *)addr; |
| |
| __asm__ ("invlpg %0" :: "m" (*page)); |
| } |
| |
| #ifdef CONFIG_X86_64 |
| #define PML4E_FLAGS_MASK (Z_X86_MMU_RW | Z_X86_MMU_US | Z_X86_MMU_P) |
| |
| #define PDPTE_FLAGS_MASK PML4E_FLAGS_MASK |
| |
| #define PDE_FLAGS_MASK PDPTE_FLAGS_MASK |
| #else |
| #define PDPTE_FLAGS_MASK Z_X86_MMU_P |
| |
| #define PDE_FLAGS_MASK (Z_X86_MMU_RW | Z_X86_MMU_US | \ |
| PDPTE_FLAGS_MASK) |
| #endif |
| |
| #define PTE_FLAGS_MASK (PDE_FLAGS_MASK | Z_X86_MMU_XD | \ |
| Z_X86_MMU_PWT | \ |
| Z_X86_MMU_PCD) |
| |
| void z_x86_mmu_set_flags(struct x86_page_tables *ptables, void *ptr, |
| size_t size, u64_t flags, u64_t mask, bool flush) |
| { |
| uintptr_t addr = (uintptr_t)ptr; |
| |
| __ASSERT((addr & MMU_PAGE_MASK) == 0U, "unaligned address provided"); |
| __ASSERT((size & MMU_PAGE_MASK) == 0U, "unaligned size provided"); |
| |
| /* L1TF mitigation: non-present PTEs will have address fields |
| * zeroed. Expand the mask to include address bits if we are changing |
| * the present bit. |
| */ |
| if ((mask & Z_X86_MMU_P) != 0) { |
| mask |= Z_X86_MMU_PTE_ADDR_MASK; |
| } |
| |
| /* NOTE: All of this code assumes that 2MB or 1GB pages are not being |
| * modified. |
| */ |
| while (size != 0) { |
| u64_t *pte; |
| u64_t *pde; |
| u64_t *pdpte; |
| #ifdef CONFIG_X86_64 |
| u64_t *pml4e; |
| #endif |
| u64_t cur_flags = flags; |
| bool exec = (flags & Z_X86_MMU_XD) == 0; |
| |
| #ifdef CONFIG_X86_64 |
| pml4e = z_x86_pml4_get_pml4e(z_x86_get_pml4(ptables), addr); |
| __ASSERT((*pml4e & Z_X86_MMU_P) != 0, |
| "set flags on non-present PML4e"); |
| *pml4e |= (flags & PML4E_FLAGS_MASK); |
| |
| if (exec) { |
| *pml4e &= ~Z_X86_MMU_XD; |
| } |
| |
| pdpte = z_x86_pdpt_get_pdpte(z_x86_pml4e_get_pdpt(*pml4e), |
| addr); |
| #else |
| pdpte = z_x86_pdpt_get_pdpte(z_x86_get_pdpt(ptables, addr), |
| addr); |
| #endif |
| __ASSERT((*pdpte & Z_X86_MMU_P) != 0, |
| "set flags on non-present PDPTE"); |
| *pdpte |= (flags & PDPTE_FLAGS_MASK); |
| #ifdef CONFIG_X86_64 |
| if (exec) { |
| *pdpte &= ~Z_X86_MMU_XD; |
| } |
| #endif |
| pde = z_x86_pd_get_pde(z_x86_pdpte_get_pd(*pdpte), addr); |
| __ASSERT((*pde & Z_X86_MMU_P) != 0, |
| "set flags on non-present PDE"); |
| *pde |= (flags & PDE_FLAGS_MASK); |
| |
| /* If any flags enable execution, clear execute disable at the |
| * page directory level |
| */ |
| if (exec) { |
| *pde &= ~Z_X86_MMU_XD; |
| } |
| |
| pte = z_x86_pt_get_pte(z_x86_pde_get_pt(*pde), addr); |
| |
| /* If we're setting the present bit, restore the address |
| * field. If we're clearing it, then the address field |
| * will be zeroed instead, mapping the PTE to the NULL page. |
| */ |
| if ((mask & Z_X86_MMU_P) != 0 && ((flags & Z_X86_MMU_P) != 0)) { |
| cur_flags |= addr; |
| } |
| |
| *pte = (*pte & ~mask) | cur_flags; |
| if (flush) { |
| tlb_flush_page((void *)addr); |
| } |
| |
| size -= MMU_PAGE_SIZE; |
| addr += MMU_PAGE_SIZE; |
| } |
| } |
| |
| static char __aligned(MMU_PAGE_SIZE) |
| page_pool[MMU_PAGE_SIZE * CONFIG_X86_MMU_PAGE_POOL_PAGES]; |
| |
| static char *page_pos = page_pool + sizeof(page_pool); |
| |
| static void *get_page(void) |
| { |
| page_pos -= MMU_PAGE_SIZE; |
| |
| __ASSERT(page_pos >= page_pool, "out of MMU pages\n"); |
| |
| return page_pos; |
| } |
| |
| #ifdef CONFIG_X86_64 |
| #define PTABLES_ALIGN 4096 |
| #else |
| #define PTABLES_ALIGN 32 |
| #endif |
| |
| __aligned(PTABLES_ALIGN) struct x86_page_tables z_x86_kernel_ptables; |
| #ifdef CONFIG_X86_KPTI |
| __aligned(PTABLES_ALIGN) struct x86_page_tables z_x86_user_ptables; |
| #endif |
| |
| extern char z_shared_kernel_page_start[]; |
| |
| static inline bool is_within_system_ram(uintptr_t addr) |
| { |
| return (addr >= DT_PHYS_RAM_ADDR) && |
| (addr < (DT_PHYS_RAM_ADDR + (DT_RAM_SIZE * 1024U))); |
| } |
| |
| /* Ignored bit posiition at all levels */ |
| #define IGNORED BIT64(11) |
| |
| static void maybe_clear_xd(u64_t *entry, bool exec) |
| { |
| /* Execute disable bit needs special handling, we should only set it at |
| * intermediate levels if ALL containing pages have XD set (instead of |
| * just one). |
| * |
| * Use an ignored bit position in the PDE to store a marker on whether |
| * any configured region allows execution. |
| */ |
| if (exec) { |
| *entry |= IGNORED; |
| *entry &= ~Z_X86_MMU_XD; |
| } else if ((*entry & IGNORED) == 0) { |
| *entry |= Z_X86_MMU_XD; |
| } |
| } |
| |
| static void add_mmu_region_page(struct x86_page_tables *ptables, |
| uintptr_t addr, u64_t flags, bool user_table) |
| { |
| #ifdef CONFIG_X86_64 |
| u64_t *pml4e; |
| #endif |
| struct x86_mmu_pdpt *pdpt; |
| u64_t *pdpte; |
| struct x86_mmu_pd *pd; |
| u64_t *pde; |
| struct x86_mmu_pt *pt; |
| u64_t *pte; |
| bool exec = (flags & Z_X86_MMU_XD) == 0; |
| |
| #ifdef CONFIG_X86_KPTI |
| /* If we are generating a page table for user mode, and this address |
| * does not have the user flag set, and this address falls outside |
| * of system RAM, then don't bother generating any tables for it, |
| * we will never need them later as memory domains are limited to |
| * regions within system RAM. |
| */ |
| if (user_table && (flags & Z_X86_MMU_US) == 0 && |
| !is_within_system_ram(addr)) { |
| return; |
| } |
| #endif |
| |
| #ifdef CONFIG_X86_64 |
| pml4e = z_x86_pml4_get_pml4e(z_x86_get_pml4(ptables), addr); |
| if ((*pml4e & Z_X86_MMU_P) == 0) { |
| pdpt = get_page(); |
| pml4e_update_pdpt(pml4e, pdpt); |
| } else { |
| pdpt = z_x86_pml4e_get_pdpt(*pml4e); |
| } |
| *pml4e |= (flags & PML4E_FLAGS_MASK); |
| maybe_clear_xd(pml4e, exec); |
| #else |
| pdpt = z_x86_get_pdpt(ptables, addr); |
| #endif |
| |
| /* Setup the PDPTE entry for the address, creating a page directory |
| * if one didn't exist |
| */ |
| pdpte = z_x86_pdpt_get_pdpte(pdpt, addr); |
| if ((*pdpte & Z_X86_MMU_P) == 0) { |
| pd = get_page(); |
| pdpte_update_pd(pdpte, pd); |
| } else { |
| pd = z_x86_pdpte_get_pd(*pdpte); |
| } |
| *pdpte |= (flags & PDPTE_FLAGS_MASK); |
| #ifdef CONFIG_X86_64 |
| maybe_clear_xd(pdpte, exec); |
| #endif |
| |
| /* Setup the PDE entry for the address, creating a page table |
| * if necessary |
| */ |
| pde = z_x86_pd_get_pde(pd, addr); |
| if ((*pde & Z_X86_MMU_P) == 0) { |
| pt = get_page(); |
| pde_update_pt(pde, pt); |
| } else { |
| pt = z_x86_pde_get_pt(*pde); |
| } |
| *pde |= (flags & PDE_FLAGS_MASK); |
| maybe_clear_xd(pde, exec); |
| |
| #ifdef CONFIG_X86_KPTI |
| if (user_table && (flags & Z_X86_MMU_US) == 0 && |
| addr != (uintptr_t)(&z_shared_kernel_page_start)) { |
| /* All non-user accessible pages except the shared page |
| * are marked non-present in the page table. |
| */ |
| return; |
| } |
| #else |
| ARG_UNUSED(user_table); |
| #endif |
| |
| /* Finally set up the page table entry */ |
| pte = z_x86_pt_get_pte(pt, addr); |
| pte_update_addr(pte, addr); |
| *pte |= (flags & PTE_FLAGS_MASK); |
| } |
| |
| static void add_mmu_region(struct x86_page_tables *ptables, |
| struct mmu_region *rgn, |
| bool user_table) |
| { |
| size_t size; |
| u64_t flags; |
| uintptr_t addr; |
| |
| __ASSERT((rgn->address & MMU_PAGE_MASK) == 0U, |
| "unaligned address provided"); |
| __ASSERT((rgn->size & MMU_PAGE_MASK) == 0U, |
| "unaligned size provided"); |
| addr = rgn->address; |
| flags = rgn->flags | Z_X86_MMU_P; |
| |
| /* Iterate through the region a page at a time, creating entries as |
| * necessary. |
| */ |
| size = rgn->size; |
| while (size > 0) { |
| add_mmu_region_page(ptables, addr, flags, user_table); |
| |
| size -= MMU_PAGE_SIZE; |
| addr += MMU_PAGE_SIZE; |
| } |
| } |
| |
| /* Called from x86's arch_kernel_init() */ |
| void z_x86_paging_init(void) |
| { |
| size_t pages_free; |
| |
| Z_STRUCT_SECTION_FOREACH(mmu_region, rgn) { |
| add_mmu_region(&z_x86_kernel_ptables, rgn, false); |
| #ifdef CONFIG_X86_KPTI |
| add_mmu_region(&z_x86_user_ptables, rgn, true); |
| #endif |
| } |
| |
| pages_free = (page_pos - page_pool) / MMU_PAGE_SIZE; |
| |
| if (pages_free != 0) { |
| printk("Optimal CONFIG_X86_MMU_PAGE_POOL_PAGES %zu\n", |
| CONFIG_X86_MMU_PAGE_POOL_PAGES - pages_free); |
| } |
| |
| #ifdef CONFIG_X86_64 |
| /* MMU already enabled at boot for long mode, we just need to |
| * program CR3 with our newly generated page tables. |
| */ |
| __asm__ volatile("movq %0, %%cr3\n\t" |
| : : "r" (&z_x86_kernel_ptables) : "memory"); |
| #else |
| z_x86_enable_paging(); |
| #endif |
| } |
| |
| #ifdef CONFIG_X86_USERSPACE |
| int arch_buffer_validate(void *addr, size_t size, int write) |
| { |
| return z_x86_mmu_validate(z_x86_thread_page_tables_get(_current), addr, |
| size, write != 0); |
| } |
| |
| static uintptr_t thread_pd_create(uintptr_t pages, |
| struct x86_page_tables *thread_ptables, |
| struct x86_page_tables *master_ptables) |
| { |
| uintptr_t pos = pages, phys_addr = Z_X86_PD_START; |
| |
| for (int i = 0; i < Z_X86_NUM_PD; i++, phys_addr += Z_X86_PD_AREA) { |
| u64_t *pdpte; |
| struct x86_mmu_pd *master_pd, *dest_pd; |
| |
| /* Obtain PD in master tables for the address range and copy |
| * into the per-thread PD for this range |
| */ |
| master_pd = z_x86_get_pd(master_ptables, phys_addr); |
| dest_pd = (struct x86_mmu_pd *)pos; |
| |
| (void)memcpy(dest_pd, master_pd, sizeof(struct x86_mmu_pd)); |
| |
| /* Update pointer in per-thread pdpt to point to the per-thread |
| * directory we just copied |
| */ |
| pdpte = z_x86_get_pdpte(thread_ptables, phys_addr); |
| pdpte_update_pd(pdpte, dest_pd); |
| pos += MMU_PAGE_SIZE; |
| } |
| |
| return pos; |
| } |
| |
| /* thread_ptables must be initialized, as well as all the page directories */ |
| static uintptr_t thread_pt_create(uintptr_t pages, |
| struct x86_page_tables *thread_ptables, |
| struct x86_page_tables *master_ptables) |
| { |
| uintptr_t pos = pages, phys_addr = Z_X86_PT_START; |
| |
| for (int i = 0; i < Z_X86_NUM_PT; i++, phys_addr += Z_X86_PT_AREA) { |
| u64_t *pde; |
| struct x86_mmu_pt *master_pt, *dest_pt; |
| |
| /* Same as we did with the directories, obtain PT in master |
| * tables for the address range and copy into per-thread PT |
| * for this range |
| */ |
| master_pt = z_x86_get_pt(master_ptables, phys_addr); |
| dest_pt = (struct x86_mmu_pt *)pos; |
| (void)memcpy(dest_pt, master_pt, sizeof(struct x86_mmu_pt)); |
| |
| /* And then wire this up to the relevant per-thread |
| * page directory entry |
| */ |
| pde = z_x86_get_pde(thread_ptables, phys_addr); |
| pde_update_pt(pde, dest_pt); |
| pos += MMU_PAGE_SIZE; |
| } |
| |
| return pos; |
| } |
| |
| /* Initialize the page tables for a thread. This will contain, once done, |
| * the boot-time configuration for a user thread page tables. There are |
| * no pre-conditions on the existing state of the per-thread tables. |
| */ |
| static void copy_page_tables(struct k_thread *thread, |
| struct x86_page_tables *master_ptables) |
| { |
| uintptr_t pos, start; |
| struct x86_page_tables *thread_ptables = |
| z_x86_thread_page_tables_get(thread); |
| struct z_x86_thread_stack_header *header = |
| (struct z_x86_thread_stack_header *)thread->stack_obj; |
| |
| __ASSERT(thread->stack_obj != NULL, "no stack object assigned"); |
| __ASSERT(z_x86_page_tables_get() != thread_ptables, |
| "tables are active"); |
| __ASSERT(((uintptr_t)thread_ptables & 0x1f) == 0, |
| "unaligned page tables at %p", thread_ptables); |
| |
| (void)memcpy(thread_ptables, master_ptables, |
| sizeof(struct x86_page_tables)); |
| |
| /* pos represents the page we are working with in the reserved area |
| * in the stack buffer for per-thread tables. As we create tables in |
| * this area, pos is incremented to the next free page. |
| * |
| * The layout of the stack object, when this is done: |
| * |
| * +---------------------------+ <- thread->stack_obj |
| * | PDE(0) | |
| * +---------------------------+ |
| * | ... | |
| * +---------------------------+ |
| * | PDE(Z_X86_NUM_PD - 1) | |
| * +---------------------------+ |
| * | PTE(0) | |
| * +---------------------------+ |
| * | ... | |
| * +---------------------------+ |
| * | PTE(Z_X86_NUM_PT - 1) | |
| * +---------------------------+ <- pos once this logic completes |
| * | Stack guard | |
| * +---------------------------+ |
| * | Privilege elevation stack | |
| * | PDPT | |
| * +---------------------------+ <- thread->stack_info.start |
| * | Thread stack | |
| * | ... | |
| * |
| */ |
| start = (uintptr_t)(&header->page_tables); |
| pos = thread_pd_create(start, thread_ptables, master_ptables); |
| pos = thread_pt_create(pos, thread_ptables, master_ptables); |
| |
| __ASSERT(pos == (start + Z_X86_THREAD_PT_AREA), |
| "wrong amount of stack object memory used"); |
| } |
| |
| static void reset_mem_partition(struct x86_page_tables *thread_ptables, |
| struct k_mem_partition *partition) |
| { |
| uintptr_t addr = partition->start; |
| size_t size = partition->size; |
| |
| __ASSERT((addr & MMU_PAGE_MASK) == 0U, "unaligned address provided"); |
| __ASSERT((size & MMU_PAGE_MASK) == 0U, "unaligned size provided"); |
| |
| while (size != 0) { |
| u64_t *thread_pte, *master_pte; |
| |
| thread_pte = z_x86_get_pte(thread_ptables, addr); |
| master_pte = z_x86_get_pte(&USER_PTABLES, addr); |
| |
| *thread_pte = *master_pte; |
| |
| size -= MMU_PAGE_SIZE; |
| addr += MMU_PAGE_SIZE; |
| } |
| } |
| |
| static void apply_mem_partition(struct x86_page_tables *ptables, |
| struct k_mem_partition *partition) |
| { |
| u64_t x86_attr; |
| u64_t mask; |
| |
| if (IS_ENABLED(CONFIG_X86_KPTI)) { |
| x86_attr = partition->attr | Z_X86_MMU_P; |
| mask = K_MEM_PARTITION_PERM_MASK | Z_X86_MMU_P; |
| } else { |
| x86_attr = partition->attr; |
| mask = K_MEM_PARTITION_PERM_MASK; |
| } |
| |
| __ASSERT(partition->start >= DT_PHYS_RAM_ADDR, |
| "region at %08lx[%zu] extends below system ram start 0x%08x", |
| partition->start, partition->size, DT_PHYS_RAM_ADDR); |
| __ASSERT(((partition->start + partition->size) <= |
| (DT_PHYS_RAM_ADDR + (DT_RAM_SIZE * 1024U))), |
| "region at %08lx[%zu] end at %08lx extends beyond system ram end 0x%08x", |
| partition->start, partition->size, |
| partition->start + partition->size, |
| (DT_PHYS_RAM_ADDR + (DT_RAM_SIZE * 1024U))); |
| |
| z_x86_mmu_set_flags(ptables, (void *)partition->start, partition->size, |
| x86_attr, mask, false); |
| } |
| |
| void z_x86_apply_mem_domain(struct x86_page_tables *ptables, |
| struct k_mem_domain *mem_domain) |
| { |
| for (int i = 0, pcount = 0; pcount < mem_domain->num_partitions; i++) { |
| struct k_mem_partition *partition; |
| |
| partition = &mem_domain->partitions[i]; |
| if (partition->size == 0) { |
| continue; |
| } |
| pcount++; |
| |
| apply_mem_partition(ptables, partition); |
| } |
| } |
| |
| /* Called on creation of a user thread or when a supervisor thread drops to |
| * user mode. |
| * |
| * Sets up the per-thread page tables, such that when they are activated on |
| * context switch, everything is ready to go. |
| */ |
| void z_x86_thread_pt_init(struct k_thread *thread) |
| { |
| struct x86_page_tables *ptables = z_x86_thread_page_tables_get(thread); |
| |
| /* USER_PDPT contains the page tables with the boot time memory |
| * policy. We use it as a template to set up the per-thread page |
| * tables. |
| * |
| * With KPTI, this is a distinct set of tables z_x86_user_pdpt from the |
| * kernel page tables in z_x86_kernel_pdpt; it has all non user |
| * accessible pages except the trampoline page marked as non-present. |
| * Without KPTI, they are the same object. |
| */ |
| copy_page_tables(thread, &USER_PTABLES); |
| |
| /* Enable access to the thread's own stack buffer */ |
| z_x86_mmu_set_flags(ptables, (void *)thread->stack_info.start, |
| ROUND_UP(thread->stack_info.size, MMU_PAGE_SIZE), |
| Z_X86_MMU_P | K_MEM_PARTITION_P_RW_U_RW, |
| Z_X86_MMU_P | K_MEM_PARTITION_PERM_MASK, |
| false); |
| } |
| |
| /* |
| * Memory domain interface |
| * |
| * In all cases, if one of these APIs is called on a supervisor thread, |
| * we don't need to do anything. If the thread later drops into supervisor |
| * mode the per-thread page tables will be generated and the memory domain |
| * configuration applied. |
| */ |
| void arch_mem_domain_partition_remove(struct k_mem_domain *domain, |
| u32_t partition_id) |
| { |
| sys_dnode_t *node, *next_node; |
| |
| /* Removing a partition. Need to reset the relevant memory range |
| * to the defaults in USER_PDPT for each thread. |
| */ |
| SYS_DLIST_FOR_EACH_NODE_SAFE(&domain->mem_domain_q, node, next_node) { |
| struct k_thread *thread = |
| CONTAINER_OF(node, struct k_thread, mem_domain_info); |
| |
| if ((thread->base.user_options & K_USER) == 0) { |
| continue; |
| } |
| |
| reset_mem_partition(z_x86_thread_page_tables_get(thread), |
| &domain->partitions[partition_id]); |
| } |
| } |
| |
| void arch_mem_domain_destroy(struct k_mem_domain *domain) |
| { |
| for (int i = 0, pcount = 0; pcount < domain->num_partitions; i++) { |
| struct k_mem_partition *partition; |
| |
| partition = &domain->partitions[i]; |
| if (partition->size == 0) { |
| continue; |
| } |
| pcount++; |
| |
| arch_mem_domain_partition_remove(domain, i); |
| } |
| } |
| |
| void arch_mem_domain_thread_remove(struct k_thread *thread) |
| { |
| struct k_mem_domain *domain = thread->mem_domain_info.mem_domain; |
| |
| /* Non-user threads don't have per-thread page tables set up */ |
| if ((thread->base.user_options & K_USER) == 0) { |
| return; |
| } |
| |
| for (int i = 0, pcount = 0; pcount < domain->num_partitions; i++) { |
| struct k_mem_partition *partition; |
| |
| partition = &domain->partitions[i]; |
| if (partition->size == 0) { |
| continue; |
| } |
| pcount++; |
| |
| reset_mem_partition(z_x86_thread_page_tables_get(thread), |
| partition); |
| } |
| } |
| |
| void arch_mem_domain_partition_add(struct k_mem_domain *domain, |
| u32_t partition_id) |
| { |
| sys_dnode_t *node, *next_node; |
| |
| SYS_DLIST_FOR_EACH_NODE_SAFE(&domain->mem_domain_q, node, next_node) { |
| struct k_thread *thread = |
| CONTAINER_OF(node, struct k_thread, mem_domain_info); |
| |
| if ((thread->base.user_options & K_USER) == 0) { |
| continue; |
| } |
| |
| apply_mem_partition(z_x86_thread_page_tables_get(thread), |
| &domain->partitions[partition_id]); |
| } |
| } |
| |
| void arch_mem_domain_thread_add(struct k_thread *thread) |
| { |
| if ((thread->base.user_options & K_USER) == 0) { |
| return; |
| } |
| |
| z_x86_apply_mem_domain(z_x86_thread_page_tables_get(thread), |
| thread->mem_domain_info.mem_domain); |
| } |
| |
| int arch_mem_domain_max_partitions_get(void) |
| { |
| return CONFIG_MAX_DOMAIN_PARTITIONS; |
| } |
| #endif /* CONFIG_X86_USERSPACE*/ |