| /* |
| * Copyright 2019 Broadcom |
| * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. |
| * |
| * Copyright (c) 2021 BayLibre, SAS |
| * |
| * SPDX-License-Identifier: Apache-2.0 |
| */ |
| |
| #include <zephyr/cache.h> |
| #include <zephyr/device.h> |
| #include <zephyr/init.h> |
| #include <zephyr/kernel.h> |
| #include <kernel_arch_func.h> |
| #include <kernel_arch_interface.h> |
| #include <kernel_internal.h> |
| #include <zephyr/logging/log.h> |
| #include <zephyr/arch/arm64/cpu.h> |
| #include <zephyr/arch/arm64/lib_helpers.h> |
| #include <zephyr/arch/arm64/mm.h> |
| #include <zephyr/linker/linker-defs.h> |
| #include <zephyr/spinlock.h> |
| #include <zephyr/sys/util.h> |
| |
| #include "mmu.h" |
| |
| LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); |
| |
| static uint64_t xlat_tables[CONFIG_MAX_XLAT_TABLES * Ln_XLAT_NUM_ENTRIES] |
| __aligned(Ln_XLAT_NUM_ENTRIES * sizeof(uint64_t)); |
| static uint16_t xlat_use_count[CONFIG_MAX_XLAT_TABLES]; |
| static struct k_spinlock xlat_lock; |
| |
| /* Returns a reference to a free table */ |
| static uint64_t *new_table(void) |
| { |
| uint64_t *table; |
| unsigned int i; |
| |
| /* Look for a free table. */ |
| for (i = 0U; i < CONFIG_MAX_XLAT_TABLES; i++) { |
| if (xlat_use_count[i] == 0U) { |
| table = &xlat_tables[i * Ln_XLAT_NUM_ENTRIES]; |
| xlat_use_count[i] = 1U; |
| MMU_DEBUG("allocating table [%d]%p\n", i, table); |
| return table; |
| } |
| } |
| |
| LOG_ERR("CONFIG_MAX_XLAT_TABLES, too small"); |
| return NULL; |
| } |
| |
| static inline unsigned int table_index(uint64_t *pte) |
| { |
| unsigned int i = (pte - xlat_tables) / Ln_XLAT_NUM_ENTRIES; |
| |
| __ASSERT(i < CONFIG_MAX_XLAT_TABLES, "table %p out of range", pte); |
| return i; |
| } |
| |
| /* Makes a table free for reuse. */ |
| static void free_table(uint64_t *table) |
| { |
| unsigned int i = table_index(table); |
| |
| MMU_DEBUG("freeing table [%d]%p\n", i, table); |
| __ASSERT(xlat_use_count[i] == 1U, "table still in use"); |
| xlat_use_count[i] = 0U; |
| } |
| |
| /* Adjusts usage count and returns current count. */ |
| static int table_usage(uint64_t *table, int adjustment) |
| { |
| unsigned int i = table_index(table); |
| |
| xlat_use_count[i] += adjustment; |
| __ASSERT(xlat_use_count[i] > 0, "usage count underflow"); |
| return xlat_use_count[i]; |
| } |
| |
| static inline bool is_table_unused(uint64_t *table) |
| { |
| return table_usage(table, 0) == 1; |
| } |
| |
| static inline bool is_free_desc(uint64_t desc) |
| { |
| return (desc & PTE_DESC_TYPE_MASK) == PTE_INVALID_DESC; |
| } |
| |
| static inline bool is_table_desc(uint64_t desc, unsigned int level) |
| { |
| return level != XLAT_LAST_LEVEL && |
| (desc & PTE_DESC_TYPE_MASK) == PTE_TABLE_DESC; |
| } |
| |
| static inline bool is_block_desc(uint64_t desc) |
| { |
| return (desc & PTE_DESC_TYPE_MASK) == PTE_BLOCK_DESC; |
| } |
| |
| static inline uint64_t *pte_desc_table(uint64_t desc) |
| { |
| uint64_t address = desc & GENMASK(47, PAGE_SIZE_SHIFT); |
| |
| return (uint64_t *)address; |
| } |
| |
| static inline bool is_desc_block_aligned(uint64_t desc, unsigned int level_size) |
| { |
| uint64_t mask = GENMASK(47, PAGE_SIZE_SHIFT); |
| bool aligned = !((desc & mask) & (level_size - 1)); |
| |
| if (!aligned) { |
| MMU_DEBUG("misaligned desc 0x%016llx for block size 0x%x\n", |
| desc, level_size); |
| } |
| |
| return aligned; |
| } |
| |
| static inline bool is_desc_superset(uint64_t desc1, uint64_t desc2, |
| unsigned int level) |
| { |
| uint64_t mask = DESC_ATTRS_MASK | GENMASK(47, LEVEL_TO_VA_SIZE_SHIFT(level)); |
| |
| return (desc1 & mask) == (desc2 & mask); |
| } |
| |
| #if DUMP_PTE |
| static void debug_show_pte(uint64_t *pte, unsigned int level) |
| { |
| MMU_DEBUG("%.*s", level * 2U, ". . . "); |
| MMU_DEBUG("[%d]%p: ", table_index(pte), pte); |
| |
| if (is_free_desc(*pte)) { |
| MMU_DEBUG("---\n"); |
| return; |
| } |
| |
| if (is_table_desc(*pte, level)) { |
| uint64_t *table = pte_desc_table(*pte); |
| |
| MMU_DEBUG("[Table] [%d]%p\n", table_index(table), table); |
| return; |
| } |
| |
| if (is_block_desc(*pte)) { |
| MMU_DEBUG("[Block] "); |
| } else { |
| MMU_DEBUG("[Page] "); |
| } |
| |
| uint8_t mem_type = (*pte >> 2) & MT_TYPE_MASK; |
| |
| MMU_DEBUG((mem_type == MT_NORMAL) ? "MEM" : |
| ((mem_type == MT_NORMAL_NC) ? "NC" : "DEV")); |
| MMU_DEBUG((*pte & PTE_BLOCK_DESC_AP_RO) ? "-RO" : "-RW"); |
| MMU_DEBUG((*pte & PTE_BLOCK_DESC_NS) ? "-NS" : "-S"); |
| MMU_DEBUG((*pte & PTE_BLOCK_DESC_AP_ELx) ? "-ELx" : "-ELh"); |
| MMU_DEBUG((*pte & PTE_BLOCK_DESC_PXN) ? "-PXN" : "-PX"); |
| MMU_DEBUG((*pte & PTE_BLOCK_DESC_UXN) ? "-UXN" : "-UX"); |
| MMU_DEBUG("\n"); |
| } |
| #else |
| static inline void debug_show_pte(uint64_t *pte, unsigned int level) { } |
| #endif |
| |
| static void set_pte_table_desc(uint64_t *pte, uint64_t *table, unsigned int level) |
| { |
| /* Point pte to new table */ |
| *pte = PTE_TABLE_DESC | (uint64_t)table; |
| debug_show_pte(pte, level); |
| } |
| |
| static void set_pte_block_desc(uint64_t *pte, uint64_t desc, unsigned int level) |
| { |
| if (desc) { |
| desc |= (level == XLAT_LAST_LEVEL) ? PTE_PAGE_DESC : PTE_BLOCK_DESC; |
| } |
| *pte = desc; |
| debug_show_pte(pte, level); |
| } |
| |
| static uint64_t *expand_to_table(uint64_t *pte, unsigned int level) |
| { |
| uint64_t *table; |
| |
| __ASSERT(level < XLAT_LAST_LEVEL, "can't expand last level"); |
| |
| table = new_table(); |
| if (!table) { |
| return NULL; |
| } |
| |
| if (!is_free_desc(*pte)) { |
| /* |
| * If entry at current level was already populated |
| * then we need to reflect that in the new table. |
| */ |
| uint64_t desc = *pte; |
| unsigned int i, stride_shift; |
| |
| MMU_DEBUG("expanding PTE 0x%016llx into table [%d]%p\n", |
| desc, table_index(table), table); |
| __ASSERT(is_block_desc(desc), ""); |
| |
| if (level + 1 == XLAT_LAST_LEVEL) { |
| desc |= PTE_PAGE_DESC; |
| } |
| |
| stride_shift = LEVEL_TO_VA_SIZE_SHIFT(level + 1); |
| for (i = 0U; i < Ln_XLAT_NUM_ENTRIES; i++) { |
| table[i] = desc | (i << stride_shift); |
| } |
| table_usage(table, Ln_XLAT_NUM_ENTRIES); |
| } else { |
| /* |
| * Adjust usage count for parent table's entry |
| * that will no longer be free. |
| */ |
| table_usage(pte, 1); |
| } |
| |
| /* Link the new table in place of the pte it replaces */ |
| set_pte_table_desc(pte, table, level); |
| table_usage(table, 1); |
| |
| return table; |
| } |
| |
| static int set_mapping(struct arm_mmu_ptables *ptables, |
| uintptr_t virt, size_t size, |
| uint64_t desc, bool may_overwrite) |
| { |
| uint64_t *pte, *ptes[XLAT_LAST_LEVEL + 1]; |
| uint64_t level_size; |
| uint64_t *table = ptables->base_xlat_table; |
| unsigned int level = BASE_XLAT_LEVEL; |
| int ret = 0; |
| |
| while (size) { |
| __ASSERT(level <= XLAT_LAST_LEVEL, |
| "max translation table level exceeded\n"); |
| |
| /* Locate PTE for given virtual address and page table level */ |
| pte = &table[XLAT_TABLE_VA_IDX(virt, level)]; |
| ptes[level] = pte; |
| |
| if (is_table_desc(*pte, level)) { |
| /* Move to the next translation table level */ |
| level++; |
| table = pte_desc_table(*pte); |
| continue; |
| } |
| |
| if (!may_overwrite && !is_free_desc(*pte)) { |
| /* the entry is already allocated */ |
| LOG_ERR("entry already in use: " |
| "level %d pte %p *pte 0x%016llx", |
| level, pte, *pte); |
| ret = -EBUSY; |
| break; |
| } |
| |
| level_size = 1ULL << LEVEL_TO_VA_SIZE_SHIFT(level); |
| |
| if (is_desc_superset(*pte, desc, level)) { |
| /* This block already covers our range */ |
| level_size -= (virt & (level_size - 1)); |
| if (level_size > size) { |
| level_size = size; |
| } |
| goto move_on; |
| } |
| |
| if ((size < level_size) || (virt & (level_size - 1)) || |
| !is_desc_block_aligned(desc, level_size)) { |
| /* Range doesn't fit, create subtable */ |
| table = expand_to_table(pte, level); |
| if (!table) { |
| ret = -ENOMEM; |
| break; |
| } |
| level++; |
| continue; |
| } |
| |
| /* Adjust usage count for corresponding table */ |
| if (is_free_desc(*pte)) { |
| table_usage(pte, 1); |
| } |
| if (!desc) { |
| table_usage(pte, -1); |
| } |
| /* Create (or erase) block/page descriptor */ |
| set_pte_block_desc(pte, desc, level); |
| |
| /* recursively free unused tables if any */ |
| while (level != BASE_XLAT_LEVEL && |
| is_table_unused(pte)) { |
| free_table(pte); |
| pte = ptes[--level]; |
| set_pte_block_desc(pte, 0, level); |
| table_usage(pte, -1); |
| } |
| |
| move_on: |
| virt += level_size; |
| desc += desc ? level_size : 0; |
| size -= level_size; |
| |
| /* Range is mapped, start again for next range */ |
| table = ptables->base_xlat_table; |
| level = BASE_XLAT_LEVEL; |
| } |
| |
| return ret; |
| } |
| |
| #ifdef CONFIG_USERSPACE |
| |
| static uint64_t *dup_table(uint64_t *src_table, unsigned int level) |
| { |
| uint64_t *dst_table = new_table(); |
| int i; |
| |
| if (!dst_table) { |
| return NULL; |
| } |
| |
| MMU_DEBUG("dup (level %d) [%d]%p to [%d]%p\n", level, |
| table_index(src_table), src_table, |
| table_index(dst_table), dst_table); |
| |
| for (i = 0; i < Ln_XLAT_NUM_ENTRIES; i++) { |
| /* |
| * After the table duplication, each table can be independently |
| * updated. Thus, entries may become non-global. |
| * To keep the invariants very simple, we thus force the non-global |
| * bit on duplication. Moreover, there is no process to revert this |
| * (e.g. in `globalize_table`). Could be improved in future work. |
| */ |
| if (!is_free_desc(src_table[i]) && !is_table_desc(src_table[i], level)) { |
| src_table[i] |= PTE_BLOCK_DESC_NG; |
| } |
| |
| dst_table[i] = src_table[i]; |
| if (is_table_desc(src_table[i], level)) { |
| table_usage(pte_desc_table(src_table[i]), 1); |
| } |
| if (!is_free_desc(dst_table[i])) { |
| table_usage(dst_table, 1); |
| } |
| } |
| |
| return dst_table; |
| } |
| |
| static int privatize_table(uint64_t *dst_table, uint64_t *src_table, |
| uintptr_t virt, size_t size, unsigned int level) |
| { |
| size_t step, level_size = 1ULL << LEVEL_TO_VA_SIZE_SHIFT(level); |
| unsigned int i; |
| int ret; |
| |
| for ( ; size; virt += step, size -= step) { |
| step = level_size - (virt & (level_size - 1)); |
| if (step > size) { |
| step = size; |
| } |
| i = XLAT_TABLE_VA_IDX(virt, level); |
| |
| if (!is_table_desc(dst_table[i], level) || |
| !is_table_desc(src_table[i], level)) { |
| /* this entry is already private */ |
| continue; |
| } |
| |
| uint64_t *dst_subtable = pte_desc_table(dst_table[i]); |
| uint64_t *src_subtable = pte_desc_table(src_table[i]); |
| |
| if (dst_subtable == src_subtable) { |
| /* need to make a private copy of this table */ |
| dst_subtable = dup_table(src_subtable, level + 1); |
| if (!dst_subtable) { |
| return -ENOMEM; |
| } |
| set_pte_table_desc(&dst_table[i], dst_subtable, level); |
| table_usage(dst_subtable, 1); |
| table_usage(src_subtable, -1); |
| } |
| |
| ret = privatize_table(dst_subtable, src_subtable, |
| virt, step, level + 1); |
| if (ret) { |
| return ret; |
| } |
| } |
| |
| return 0; |
| } |
| |
| /* |
| * Make the given virtual address range private in dst_pt with regards to |
| * src_pt. By "private" this means that corresponding page tables in dst_pt |
| * will be duplicated so not to share the same table(s) with src_pt. |
| * If corresponding page tables in dst_pt are already distinct from src_pt |
| * then nothing is done. This allows for subsequent mapping changes in that |
| * range to affect only dst_pt. |
| */ |
| static int privatize_page_range(struct arm_mmu_ptables *dst_pt, |
| struct arm_mmu_ptables *src_pt, |
| uintptr_t virt_start, size_t size, |
| const char *name) |
| { |
| k_spinlock_key_t key; |
| int ret; |
| |
| MMU_DEBUG("privatize [%s]: virt %lx size %lx\n", |
| name, virt_start, size); |
| |
| key = k_spin_lock(&xlat_lock); |
| |
| ret = privatize_table(dst_pt->base_xlat_table, src_pt->base_xlat_table, |
| virt_start, size, BASE_XLAT_LEVEL); |
| |
| k_spin_unlock(&xlat_lock, key); |
| return ret; |
| } |
| |
| static void discard_table(uint64_t *table, unsigned int level) |
| { |
| unsigned int i; |
| |
| for (i = 0U; i < Ln_XLAT_NUM_ENTRIES; i++) { |
| if (is_table_desc(table[i], level)) { |
| table_usage(pte_desc_table(table[i]), -1); |
| discard_table(pte_desc_table(table[i]), level + 1); |
| } |
| if (!is_free_desc(table[i])) { |
| table[i] = 0U; |
| table_usage(table, -1); |
| } |
| } |
| free_table(table); |
| } |
| |
| static int globalize_table(uint64_t *dst_table, uint64_t *src_table, |
| uintptr_t virt, size_t size, unsigned int level) |
| { |
| size_t step, level_size = 1ULL << LEVEL_TO_VA_SIZE_SHIFT(level); |
| unsigned int i; |
| int ret; |
| |
| for ( ; size; virt += step, size -= step) { |
| step = level_size - (virt & (level_size - 1)); |
| if (step > size) { |
| step = size; |
| } |
| i = XLAT_TABLE_VA_IDX(virt, level); |
| |
| if (dst_table[i] == src_table[i]) { |
| /* already identical to global table */ |
| continue; |
| } |
| |
| if (step != level_size) { |
| /* boundary falls in the middle of this pte */ |
| __ASSERT(is_table_desc(src_table[i], level), |
| "can't have partial block pte here"); |
| if (!is_table_desc(dst_table[i], level)) { |
| /* we need more fine grained boundaries */ |
| if (!expand_to_table(&dst_table[i], level)) { |
| return -ENOMEM; |
| } |
| } |
| ret = globalize_table(pte_desc_table(dst_table[i]), |
| pte_desc_table(src_table[i]), |
| virt, step, level + 1); |
| if (ret) { |
| return ret; |
| } |
| continue; |
| } |
| |
| /* we discard current pte and replace with global one */ |
| |
| uint64_t *old_table = is_table_desc(dst_table[i], level) ? |
| pte_desc_table(dst_table[i]) : NULL; |
| |
| if (is_free_desc(dst_table[i])) { |
| table_usage(dst_table, 1); |
| } |
| if (is_free_desc(src_table[i])) { |
| table_usage(dst_table, -1); |
| } |
| if (is_table_desc(src_table[i], level)) { |
| table_usage(pte_desc_table(src_table[i]), 1); |
| } |
| dst_table[i] = src_table[i]; |
| debug_show_pte(&dst_table[i], level); |
| |
| if (old_table) { |
| /* we can discard the whole branch */ |
| table_usage(old_table, -1); |
| discard_table(old_table, level + 1); |
| } |
| } |
| |
| return 0; |
| } |
| |
| /* |
| * Globalize the given virtual address range in dst_pt from src_pt. We make |
| * it global by sharing as much page table content from src_pt as possible, |
| * including page tables themselves, and corresponding private tables in |
| * dst_pt are then discarded. If page tables in the given range are already |
| * shared then nothing is done. If page table sharing is not possible then |
| * page table entries in dst_pt are synchronized with those from src_pt. |
| */ |
| static int globalize_page_range(struct arm_mmu_ptables *dst_pt, |
| struct arm_mmu_ptables *src_pt, |
| uintptr_t virt_start, size_t size, |
| const char *name) |
| { |
| k_spinlock_key_t key; |
| int ret; |
| |
| MMU_DEBUG("globalize [%s]: virt %lx size %lx\n", |
| name, virt_start, size); |
| |
| key = k_spin_lock(&xlat_lock); |
| |
| ret = globalize_table(dst_pt->base_xlat_table, src_pt->base_xlat_table, |
| virt_start, size, BASE_XLAT_LEVEL); |
| |
| k_spin_unlock(&xlat_lock, key); |
| return ret; |
| } |
| |
| #endif /* CONFIG_USERSPACE */ |
| |
| static uint64_t get_region_desc(uint32_t attrs) |
| { |
| unsigned int mem_type; |
| uint64_t desc = 0U; |
| |
| /* NS bit for security memory access from secure state */ |
| desc |= (attrs & MT_NS) ? PTE_BLOCK_DESC_NS : 0; |
| |
| /* |
| * AP bits for EL0 / ELh Data access permission |
| * |
| * AP[2:1] ELh EL0 |
| * +--------------------+ |
| * 00 RW NA |
| * 01 RW RW |
| * 10 RO NA |
| * 11 RO RO |
| */ |
| |
| /* AP bits for Data access permission */ |
| desc |= (attrs & MT_RW) ? PTE_BLOCK_DESC_AP_RW : PTE_BLOCK_DESC_AP_RO; |
| |
| /* Mirror permissions to EL0 */ |
| desc |= (attrs & MT_RW_AP_ELx) ? |
| PTE_BLOCK_DESC_AP_ELx : PTE_BLOCK_DESC_AP_EL_HIGHER; |
| |
| /* the access flag */ |
| desc |= PTE_BLOCK_DESC_AF; |
| |
| /* memory attribute index field */ |
| mem_type = MT_TYPE(attrs); |
| desc |= PTE_BLOCK_DESC_MEMTYPE(mem_type); |
| |
| switch (mem_type) { |
| case MT_DEVICE_nGnRnE: |
| case MT_DEVICE_nGnRE: |
| case MT_DEVICE_GRE: |
| /* Access to Device memory and non-cacheable memory are coherent |
| * for all observers in the system and are treated as |
| * Outer shareable, so, for these 2 types of memory, |
| * it is not strictly needed to set shareability field |
| */ |
| desc |= PTE_BLOCK_DESC_OUTER_SHARE; |
| /* Map device memory as execute-never */ |
| desc |= PTE_BLOCK_DESC_PXN; |
| desc |= PTE_BLOCK_DESC_UXN; |
| break; |
| case MT_NORMAL_NC: |
| case MT_NORMAL: |
| /* Make Normal RW memory as execute never */ |
| if ((attrs & MT_RW) || (attrs & MT_P_EXECUTE_NEVER)) |
| desc |= PTE_BLOCK_DESC_PXN; |
| |
| if (((attrs & MT_RW) && (attrs & MT_RW_AP_ELx)) || |
| (attrs & MT_U_EXECUTE_NEVER)) |
| desc |= PTE_BLOCK_DESC_UXN; |
| |
| if (mem_type == MT_NORMAL) |
| desc |= PTE_BLOCK_DESC_INNER_SHARE; |
| else |
| desc |= PTE_BLOCK_DESC_OUTER_SHARE; |
| } |
| |
| /* non-Global bit */ |
| if (attrs & MT_NG) { |
| desc |= PTE_BLOCK_DESC_NG; |
| } |
| |
| return desc; |
| } |
| |
| static int __add_map(struct arm_mmu_ptables *ptables, const char *name, |
| uintptr_t phys, uintptr_t virt, size_t size, uint32_t attrs) |
| { |
| uint64_t desc = get_region_desc(attrs); |
| bool may_overwrite = !(attrs & MT_NO_OVERWRITE); |
| |
| MMU_DEBUG("mmap [%s]: virt %lx phys %lx size %lx attr %llx %s overwrite\n", |
| name, virt, phys, size, desc, |
| may_overwrite ? "may" : "no"); |
| __ASSERT(((virt | phys | size) & (CONFIG_MMU_PAGE_SIZE - 1)) == 0, |
| "address/size are not page aligned\n"); |
| desc |= phys; |
| return set_mapping(ptables, virt, size, desc, may_overwrite); |
| } |
| |
| static int add_map(struct arm_mmu_ptables *ptables, const char *name, |
| uintptr_t phys, uintptr_t virt, size_t size, uint32_t attrs) |
| { |
| k_spinlock_key_t key; |
| int ret; |
| |
| key = k_spin_lock(&xlat_lock); |
| ret = __add_map(ptables, name, phys, virt, size, attrs); |
| k_spin_unlock(&xlat_lock, key); |
| return ret; |
| } |
| |
| static int remove_map(struct arm_mmu_ptables *ptables, const char *name, |
| uintptr_t virt, size_t size) |
| { |
| k_spinlock_key_t key; |
| int ret; |
| |
| MMU_DEBUG("unmmap [%s]: virt %lx size %lx\n", name, virt, size); |
| __ASSERT(((virt | size) & (CONFIG_MMU_PAGE_SIZE - 1)) == 0, |
| "address/size are not page aligned\n"); |
| |
| key = k_spin_lock(&xlat_lock); |
| ret = set_mapping(ptables, virt, size, 0, true); |
| k_spin_unlock(&xlat_lock, key); |
| return ret; |
| } |
| |
| static void invalidate_tlb_all(void) |
| { |
| __asm__ volatile ( |
| "dsb ishst; tlbi vmalle1; dsb ish; isb" |
| : : : "memory"); |
| } |
| |
| /* zephyr execution regions with appropriate attributes */ |
| |
| struct arm_mmu_flat_range { |
| char *name; |
| void *start; |
| void *end; |
| uint32_t attrs; |
| }; |
| |
| static const struct arm_mmu_flat_range mmu_zephyr_ranges[] = { |
| |
| /* Mark the zephyr execution regions (data, bss, noinit, etc.) |
| * cacheable, read-write |
| * Note: read-write region is marked execute-never internally |
| */ |
| { .name = "zephyr_data", |
| .start = _image_ram_start, |
| .end = _image_ram_end, |
| .attrs = MT_NORMAL | MT_P_RW_U_NA | MT_DEFAULT_SECURE_STATE }, |
| |
| /* Mark text segment cacheable,read only and executable */ |
| { .name = "zephyr_code", |
| .start = __text_region_start, |
| .end = __text_region_end, |
| .attrs = MT_NORMAL | MT_P_RX_U_RX | MT_DEFAULT_SECURE_STATE }, |
| |
| /* Mark rodata segment cacheable, read only and execute-never */ |
| { .name = "zephyr_rodata", |
| .start = __rodata_region_start, |
| .end = __rodata_region_end, |
| .attrs = MT_NORMAL | MT_P_RO_U_RO | MT_DEFAULT_SECURE_STATE }, |
| |
| #ifdef CONFIG_NOCACHE_MEMORY |
| /* Mark nocache segment noncachable, read-write and execute-never */ |
| { .name = "nocache_data", |
| .start = _nocache_ram_start, |
| .end = _nocache_ram_end, |
| .attrs = MT_NORMAL_NC | MT_P_RW_U_RW | MT_DEFAULT_SECURE_STATE }, |
| #endif |
| }; |
| |
| static inline void add_arm_mmu_flat_range(struct arm_mmu_ptables *ptables, |
| const struct arm_mmu_flat_range *range, |
| uint32_t extra_flags) |
| { |
| uintptr_t address = (uintptr_t)range->start; |
| size_t size = (uintptr_t)range->end - address; |
| |
| if (size) { |
| /* MMU not yet active: must use unlocked version */ |
| __add_map(ptables, range->name, address, address, |
| size, range->attrs | extra_flags); |
| } |
| } |
| |
| static inline void add_arm_mmu_region(struct arm_mmu_ptables *ptables, |
| const struct arm_mmu_region *region, |
| uint32_t extra_flags) |
| { |
| if (region->size || region->attrs) { |
| /* MMU not yet active: must use unlocked version */ |
| __add_map(ptables, region->name, region->base_pa, region->base_va, |
| region->size, region->attrs | extra_flags); |
| } |
| } |
| |
| static inline void inv_dcache_after_map_helper(void *virt, size_t size, uint32_t attrs) |
| { |
| /* |
| * DC IVAC instruction requires write access permission to the VA, |
| * otherwise it can generate a permission fault |
| */ |
| if ((attrs & MT_RW) != MT_RW) { |
| return; |
| } |
| |
| if (MT_TYPE(attrs) == MT_NORMAL || MT_TYPE(attrs) == MT_NORMAL_WT) { |
| sys_cache_data_invd_range(virt, size); |
| } |
| } |
| |
| static void setup_page_tables(struct arm_mmu_ptables *ptables) |
| { |
| unsigned int index; |
| const struct arm_mmu_flat_range *range; |
| const struct arm_mmu_region *region; |
| uintptr_t max_va = 0, max_pa = 0; |
| |
| MMU_DEBUG("xlat tables:\n"); |
| for (index = 0U; index < CONFIG_MAX_XLAT_TABLES; index++) |
| MMU_DEBUG("%d: %p\n", index, xlat_tables + index * Ln_XLAT_NUM_ENTRIES); |
| |
| for (index = 0U; index < mmu_config.num_regions; index++) { |
| region = &mmu_config.mmu_regions[index]; |
| max_va = MAX(max_va, region->base_va + region->size); |
| max_pa = MAX(max_pa, region->base_pa + region->size); |
| } |
| |
| __ASSERT(max_va <= (1ULL << CONFIG_ARM64_VA_BITS), |
| "Maximum VA not supported\n"); |
| __ASSERT(max_pa <= (1ULL << CONFIG_ARM64_PA_BITS), |
| "Maximum PA not supported\n"); |
| |
| /* setup translation table for zephyr execution regions */ |
| for (index = 0U; index < ARRAY_SIZE(mmu_zephyr_ranges); index++) { |
| range = &mmu_zephyr_ranges[index]; |
| add_arm_mmu_flat_range(ptables, range, 0); |
| } |
| |
| /* |
| * Create translation tables for user provided platform regions. |
| * Those must not conflict with our default mapping. |
| */ |
| for (index = 0U; index < mmu_config.num_regions; index++) { |
| region = &mmu_config.mmu_regions[index]; |
| add_arm_mmu_region(ptables, region, MT_NO_OVERWRITE); |
| } |
| |
| invalidate_tlb_all(); |
| |
| for (index = 0U; index < ARRAY_SIZE(mmu_zephyr_ranges); index++) { |
| size_t size; |
| |
| range = &mmu_zephyr_ranges[index]; |
| size = POINTER_TO_UINT(range->end) - POINTER_TO_UINT(range->start); |
| inv_dcache_after_map_helper(range->start, size, range->attrs); |
| } |
| |
| for (index = 0U; index < mmu_config.num_regions; index++) { |
| region = &mmu_config.mmu_regions[index]; |
| inv_dcache_after_map_helper(UINT_TO_POINTER(region->base_va), region->size, |
| region->attrs); |
| } |
| } |
| |
| /* Translation table control register settings */ |
| static uint64_t get_tcr(int el) |
| { |
| uint64_t tcr; |
| uint64_t va_bits = CONFIG_ARM64_VA_BITS; |
| uint64_t tcr_ps_bits; |
| |
| tcr_ps_bits = TCR_PS_BITS; |
| |
| if (el == 1) { |
| tcr = (tcr_ps_bits << TCR_EL1_IPS_SHIFT); |
| /* |
| * TCR_EL1.EPD1: Disable translation table walk for addresses |
| * that are translated using TTBR1_EL1. |
| */ |
| tcr |= TCR_EPD1_DISABLE; |
| } else { |
| tcr = (tcr_ps_bits << TCR_EL3_PS_SHIFT); |
| } |
| |
| tcr |= TCR_T0SZ(va_bits); |
| |
| /* |
| * Translation table walk is cacheable, inner/outer WBWA and |
| * inner shareable. Due to Cortex-A57 erratum #822227 we must |
| * set TG1[1] = 4KB. |
| */ |
| tcr |= TCR_TG1_4K | TCR_TG0_4K | TCR_SHARED_INNER | |
| TCR_ORGN_WBWA | TCR_IRGN_WBWA; |
| |
| return tcr; |
| } |
| |
| static void enable_mmu_el1(struct arm_mmu_ptables *ptables, unsigned int flags) |
| { |
| ARG_UNUSED(flags); |
| uint64_t val; |
| |
| /* Set MAIR, TCR and TBBR registers */ |
| write_mair_el1(MEMORY_ATTRIBUTES); |
| write_tcr_el1(get_tcr(1)); |
| write_ttbr0_el1((uint64_t)ptables->base_xlat_table); |
| |
| /* Ensure these changes are seen before MMU is enabled */ |
| barrier_isync_fence_full(); |
| |
| /* Enable the MMU and data cache */ |
| val = read_sctlr_el1(); |
| write_sctlr_el1(val | SCTLR_M_BIT | SCTLR_C_BIT); |
| |
| /* Ensure the MMU enable takes effect immediately */ |
| barrier_isync_fence_full(); |
| |
| MMU_DEBUG("MMU enabled with dcache\n"); |
| } |
| |
| /* ARM MMU Driver Initial Setup */ |
| |
| static struct arm_mmu_ptables kernel_ptables; |
| #ifdef CONFIG_USERSPACE |
| static sys_slist_t domain_list; |
| #endif |
| |
| /* |
| * @brief MMU default configuration |
| * |
| * This function provides the default configuration mechanism for the Memory |
| * Management Unit (MMU). |
| */ |
| void z_arm64_mm_init(bool is_primary_core) |
| { |
| unsigned int flags = 0U; |
| |
| __ASSERT(CONFIG_MMU_PAGE_SIZE == KB(4), |
| "Only 4K page size is supported\n"); |
| |
| __ASSERT(GET_EL(read_currentel()) == MODE_EL1, |
| "Exception level not EL1, MMU not enabled!\n"); |
| |
| /* Ensure that MMU is already not enabled */ |
| __ASSERT((read_sctlr_el1() & SCTLR_M_BIT) == 0, "MMU is already enabled\n"); |
| |
| /* |
| * Only booting core setup up the page tables. |
| */ |
| if (is_primary_core) { |
| kernel_ptables.base_xlat_table = new_table(); |
| setup_page_tables(&kernel_ptables); |
| } |
| |
| /* currently only EL1 is supported */ |
| enable_mmu_el1(&kernel_ptables, flags); |
| } |
| |
| static void sync_domains(uintptr_t virt, size_t size) |
| { |
| #ifdef CONFIG_USERSPACE |
| sys_snode_t *node; |
| struct arch_mem_domain *domain; |
| struct arm_mmu_ptables *domain_ptables; |
| k_spinlock_key_t key; |
| int ret; |
| |
| key = k_spin_lock(&z_mem_domain_lock); |
| SYS_SLIST_FOR_EACH_NODE(&domain_list, node) { |
| domain = CONTAINER_OF(node, struct arch_mem_domain, node); |
| domain_ptables = &domain->ptables; |
| ret = globalize_page_range(domain_ptables, &kernel_ptables, |
| virt, size, "generic"); |
| if (ret) { |
| LOG_ERR("globalize_page_range() returned %d", ret); |
| } |
| } |
| k_spin_unlock(&z_mem_domain_lock, key); |
| #endif |
| } |
| |
| static int __arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) |
| { |
| struct arm_mmu_ptables *ptables; |
| uint32_t entry_flags = MT_DEFAULT_SECURE_STATE | MT_P_RX_U_NA | MT_NO_OVERWRITE; |
| |
| /* Always map in the kernel page tables */ |
| ptables = &kernel_ptables; |
| |
| /* Translate flags argument into HW-recognized entry flags. */ |
| switch (flags & K_MEM_CACHE_MASK) { |
| /* |
| * K_MEM_CACHE_NONE, K_MEM_ARM_DEVICE_nGnRnE => MT_DEVICE_nGnRnE |
| * (Device memory nGnRnE) |
| * K_MEM_ARM_DEVICE_nGnRE => MT_DEVICE_nGnRE |
| * (Device memory nGnRE) |
| * K_MEM_ARM_DEVICE_GRE => MT_DEVICE_GRE |
| * (Device memory GRE) |
| * K_MEM_ARM_NORMAL_NC => MT_NORMAL_NC |
| * (Normal memory Non-cacheable) |
| * K_MEM_CACHE_WB => MT_NORMAL |
| * (Normal memory Outer WB + Inner WB) |
| * K_MEM_CACHE_WT => MT_NORMAL_WT |
| * (Normal memory Outer WT + Inner WT) |
| */ |
| case K_MEM_CACHE_NONE: |
| /* K_MEM_CACHE_NONE equal to K_MEM_ARM_DEVICE_nGnRnE */ |
| /* case K_MEM_ARM_DEVICE_nGnRnE: */ |
| entry_flags |= MT_DEVICE_nGnRnE; |
| break; |
| case K_MEM_ARM_DEVICE_nGnRE: |
| entry_flags |= MT_DEVICE_nGnRE; |
| break; |
| case K_MEM_ARM_DEVICE_GRE: |
| entry_flags |= MT_DEVICE_GRE; |
| break; |
| case K_MEM_ARM_NORMAL_NC: |
| entry_flags |= MT_NORMAL_NC; |
| break; |
| case K_MEM_CACHE_WT: |
| entry_flags |= MT_NORMAL_WT; |
| break; |
| case K_MEM_CACHE_WB: |
| entry_flags |= MT_NORMAL; |
| break; |
| default: |
| return -ENOTSUP; |
| } |
| |
| if ((flags & K_MEM_PERM_RW) != 0U) { |
| entry_flags |= MT_RW; |
| } |
| |
| if ((flags & K_MEM_PERM_EXEC) == 0U) { |
| entry_flags |= MT_P_EXECUTE_NEVER; |
| } |
| |
| if ((flags & K_MEM_PERM_USER) != 0U) { |
| entry_flags |= MT_RW_AP_ELx; |
| } |
| |
| return add_map(ptables, "generic", phys, (uintptr_t)virt, size, entry_flags); |
| } |
| |
| void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) |
| { |
| int ret = __arch_mem_map(virt, phys, size, flags); |
| |
| if (ret) { |
| LOG_ERR("__arch_mem_map() returned %d", ret); |
| k_panic(); |
| } else { |
| uint32_t mem_flags = flags & K_MEM_CACHE_MASK; |
| |
| sync_domains((uintptr_t)virt, size); |
| invalidate_tlb_all(); |
| |
| switch (mem_flags) { |
| case K_MEM_CACHE_WB: |
| case K_MEM_CACHE_WT: |
| mem_flags = (mem_flags == K_MEM_CACHE_WB) ? MT_NORMAL : MT_NORMAL_WT; |
| mem_flags |= (flags & K_MEM_PERM_RW) ? MT_RW : 0; |
| inv_dcache_after_map_helper(virt, size, mem_flags); |
| default: |
| break; |
| } |
| } |
| } |
| |
| void arch_mem_unmap(void *addr, size_t size) |
| { |
| int ret = remove_map(&kernel_ptables, "generic", (uintptr_t)addr, size); |
| |
| if (ret) { |
| LOG_ERR("remove_map() returned %d", ret); |
| } else { |
| sync_domains((uintptr_t)addr, size); |
| invalidate_tlb_all(); |
| } |
| } |
| |
| int arch_page_phys_get(void *virt, uintptr_t *phys) |
| { |
| uint64_t par; |
| int key; |
| |
| key = arch_irq_lock(); |
| __asm__ volatile ("at S1E1R, %0" : : "r" (virt)); |
| barrier_isync_fence_full(); |
| par = read_par_el1(); |
| arch_irq_unlock(key); |
| |
| if (par & BIT(0)) { |
| return -EFAULT; |
| } |
| |
| if (phys) { |
| *phys = par & GENMASK(47, 12); |
| } |
| return 0; |
| } |
| |
| size_t arch_virt_region_align(uintptr_t phys, size_t size) |
| { |
| size_t alignment = CONFIG_MMU_PAGE_SIZE; |
| size_t level_size; |
| int level; |
| |
| for (level = XLAT_LAST_LEVEL; level >= BASE_XLAT_LEVEL; level--) { |
| level_size = 1 << LEVEL_TO_VA_SIZE_SHIFT(level); |
| |
| if (size < level_size) { |
| break; |
| } |
| |
| if ((phys & (level_size - 1))) { |
| break; |
| } |
| |
| alignment = level_size; |
| } |
| |
| return alignment; |
| } |
| |
| #ifdef CONFIG_USERSPACE |
| |
| static uint16_t next_asid = 1; |
| |
| static uint16_t get_asid(uint64_t ttbr0) |
| { |
| return ttbr0 >> TTBR_ASID_SHIFT; |
| } |
| |
| static void z_arm64_swap_ptables(struct k_thread *incoming); |
| |
| int arch_mem_domain_max_partitions_get(void) |
| { |
| return CONFIG_MAX_DOMAIN_PARTITIONS; |
| } |
| |
| int arch_mem_domain_init(struct k_mem_domain *domain) |
| { |
| struct arm_mmu_ptables *domain_ptables = &domain->arch.ptables; |
| k_spinlock_key_t key; |
| uint16_t asid; |
| |
| MMU_DEBUG("%s\n", __func__); |
| |
| key = k_spin_lock(&xlat_lock); |
| |
| /* |
| * Pick a new ASID. We use round-robin |
| * Note: `next_asid` is an uint16_t and `VM_ASID_BITS` could |
| * be up to 16, hence `next_asid` might overflow to 0 below. |
| */ |
| asid = next_asid++; |
| if ((next_asid >= (1UL << VM_ASID_BITS)) || (next_asid == 0)) { |
| next_asid = 1; |
| } |
| |
| domain_ptables->base_xlat_table = |
| dup_table(kernel_ptables.base_xlat_table, BASE_XLAT_LEVEL); |
| k_spin_unlock(&xlat_lock, key); |
| if (!domain_ptables->base_xlat_table) { |
| return -ENOMEM; |
| } |
| |
| domain_ptables->ttbr0 = (((uint64_t)asid) << TTBR_ASID_SHIFT) | |
| ((uint64_t)(uintptr_t)domain_ptables->base_xlat_table); |
| |
| sys_slist_append(&domain_list, &domain->arch.node); |
| return 0; |
| } |
| |
| static int private_map(struct arm_mmu_ptables *ptables, const char *name, |
| uintptr_t phys, uintptr_t virt, size_t size, uint32_t attrs) |
| { |
| int ret; |
| |
| ret = privatize_page_range(ptables, &kernel_ptables, virt, size, name); |
| __ASSERT(ret == 0, "privatize_page_range() returned %d", ret); |
| ret = add_map(ptables, name, phys, virt, size, attrs | MT_NG); |
| __ASSERT(ret == 0, "add_map() returned %d", ret); |
| invalidate_tlb_all(); |
| |
| inv_dcache_after_map_helper(UINT_TO_POINTER(virt), size, attrs); |
| return ret; |
| } |
| |
| static int reset_map(struct arm_mmu_ptables *ptables, const char *name, |
| uintptr_t addr, size_t size) |
| { |
| int ret; |
| |
| ret = globalize_page_range(ptables, &kernel_ptables, addr, size, name); |
| __ASSERT(ret == 0, "globalize_page_range() returned %d", ret); |
| invalidate_tlb_all(); |
| |
| return ret; |
| } |
| |
| int arch_mem_domain_partition_add(struct k_mem_domain *domain, |
| uint32_t partition_id) |
| { |
| struct arm_mmu_ptables *domain_ptables = &domain->arch.ptables; |
| struct k_mem_partition *ptn = &domain->partitions[partition_id]; |
| |
| return private_map(domain_ptables, "partition", ptn->start, ptn->start, |
| ptn->size, ptn->attr.attrs | MT_NORMAL); |
| } |
| |
| int arch_mem_domain_partition_remove(struct k_mem_domain *domain, |
| uint32_t partition_id) |
| { |
| struct arm_mmu_ptables *domain_ptables = &domain->arch.ptables; |
| struct k_mem_partition *ptn = &domain->partitions[partition_id]; |
| |
| return reset_map(domain_ptables, "partition removal", |
| ptn->start, ptn->size); |
| } |
| |
| static int map_thread_stack(struct k_thread *thread, |
| struct arm_mmu_ptables *ptables) |
| { |
| return private_map(ptables, "thread_stack", thread->stack_info.start, |
| thread->stack_info.start, thread->stack_info.size, |
| MT_P_RW_U_RW | MT_NORMAL); |
| } |
| |
| int arch_mem_domain_thread_add(struct k_thread *thread) |
| { |
| struct arm_mmu_ptables *old_ptables, *domain_ptables; |
| struct k_mem_domain *domain; |
| bool is_user, is_migration; |
| int ret = 0; |
| |
| domain = thread->mem_domain_info.mem_domain; |
| domain_ptables = &domain->arch.ptables; |
| old_ptables = thread->arch.ptables; |
| |
| is_user = (thread->base.user_options & K_USER) != 0; |
| is_migration = (old_ptables != NULL) && is_user; |
| |
| if (is_migration) { |
| ret = map_thread_stack(thread, domain_ptables); |
| } |
| |
| thread->arch.ptables = domain_ptables; |
| if (thread == _current) { |
| z_arm64_swap_ptables(thread); |
| } else { |
| #ifdef CONFIG_SMP |
| /* the thread could be running on another CPU right now */ |
| z_arm64_mem_cfg_ipi(); |
| #endif |
| } |
| |
| if (is_migration) { |
| ret = reset_map(old_ptables, __func__, thread->stack_info.start, |
| thread->stack_info.size); |
| } |
| |
| return ret; |
| } |
| |
| int arch_mem_domain_thread_remove(struct k_thread *thread) |
| { |
| struct arm_mmu_ptables *domain_ptables; |
| struct k_mem_domain *domain; |
| |
| domain = thread->mem_domain_info.mem_domain; |
| domain_ptables = &domain->arch.ptables; |
| |
| if ((thread->base.user_options & K_USER) == 0) { |
| return 0; |
| } |
| |
| if ((thread->base.thread_state & _THREAD_DEAD) == 0) { |
| return 0; |
| } |
| |
| return reset_map(domain_ptables, __func__, thread->stack_info.start, |
| thread->stack_info.size); |
| } |
| |
| static void z_arm64_swap_ptables(struct k_thread *incoming) |
| { |
| struct arm_mmu_ptables *ptables = incoming->arch.ptables; |
| uint64_t curr_ttbr0 = read_ttbr0_el1(); |
| uint64_t new_ttbr0 = ptables->ttbr0; |
| |
| if (curr_ttbr0 == new_ttbr0) { |
| return; /* Already the right tables */ |
| } |
| |
| z_arm64_set_ttbr0(new_ttbr0); |
| |
| if (get_asid(curr_ttbr0) == get_asid(new_ttbr0)) { |
| invalidate_tlb_all(); |
| } |
| } |
| |
| void z_arm64_thread_mem_domains_init(struct k_thread *incoming) |
| { |
| struct arm_mmu_ptables *ptables; |
| |
| if ((incoming->base.user_options & K_USER) == 0) |
| return; |
| |
| ptables = incoming->arch.ptables; |
| |
| /* Map the thread stack */ |
| map_thread_stack(incoming, ptables); |
| |
| z_arm64_swap_ptables(incoming); |
| } |
| |
| void z_arm64_swap_mem_domains(struct k_thread *incoming) |
| { |
| z_arm64_swap_ptables(incoming); |
| } |
| |
| #endif /* CONFIG_USERSPACE */ |