arch/x86/core/x86_mmu.c

/*
 * Copyright (c) 2011-2014 Wind River Systems, Inc.
 * Copyright (c) 2017 Intel Corporation
 *
 * SPDX-License-Identifier: Apache-2.0
 */
#include <kernel.h>
#include <mmustructs.h>
#include <linker/linker-defs.h>
#include <kernel_internal.h>
#include <init.h>

/* Common regions for all x86 processors.
 * Peripheral I/O ranges configured at the SOC level
 */

/* Mark text and rodata as read-only.
 * Userspace may read all text and rodata.
 */
MMU_BOOT_REGION((u32_t)&_image_text_start, (u32_t)&_image_text_size,
		MMU_ENTRY_READ | MMU_ENTRY_USER);

MMU_BOOT_REGION((u32_t)&_image_rodata_start, (u32_t)&_image_rodata_size,
		MMU_ENTRY_READ | MMU_ENTRY_USER |
		MMU_ENTRY_EXECUTE_DISABLE);

#ifdef CONFIG_USERSPACE
MMU_BOOT_REGION((u32_t)&_app_smem_start, (u32_t)&_app_smem_size,
		MMU_ENTRY_WRITE | MMU_ENTRY_RUNTIME_USER |
		MMU_ENTRY_EXECUTE_DISABLE);
#endif

#ifdef CONFIG_COVERAGE_GCOV
MMU_BOOT_REGION((u32_t)&__gcov_bss_start, (u32_t)&__gcov_bss_size,
		MMU_ENTRY_WRITE | MMU_ENTRY_USER | MMU_ENTRY_EXECUTE_DISABLE);
#endif

/* __kernel_ram_size includes all unused memory, which is used for heaps.
 * User threads cannot access this unless granted at runtime. This is done
 * automatically for stacks.
 */
MMU_BOOT_REGION((u32_t)&__kernel_ram_start, (u32_t)&__kernel_ram_size,
		MMU_ENTRY_WRITE |
		MMU_ENTRY_RUNTIME_USER |
		MMU_ENTRY_EXECUTE_DISABLE);


void z_x86_mmu_get_flags(struct x86_mmu_pdpt *pdpt, void *addr,
			x86_page_entry_data_t *pde_flags,
			x86_page_entry_data_t *pte_flags)
{
	*pde_flags =
		(x86_page_entry_data_t)(X86_MMU_GET_PDE(pdpt, addr)->value &
			~(x86_page_entry_data_t)MMU_PDE_PAGE_TABLE_MASK);

	if ((*pde_flags & MMU_ENTRY_PRESENT) != 0) {
		*pte_flags = (x86_page_entry_data_t)
			(X86_MMU_GET_PTE(pdpt, addr)->value &
			 ~(x86_page_entry_data_t)MMU_PTE_PAGE_MASK);
	} else {
		*pte_flags = 0;
	}
}


int z_arch_buffer_validate(void *addr, size_t size, int write)
{
	u32_t start_pde_num;
	u32_t end_pde_num;
	u32_t starting_pte_num;
	u32_t ending_pte_num;
	u32_t pde;
	u32_t pte;
	union x86_mmu_pte pte_value;
	u32_t start_pdpte_num = MMU_PDPTE_NUM(addr);
	u32_t end_pdpte_num = MMU_PDPTE_NUM((char *)addr + size - 1);
	u32_t pdpte;
	struct x86_mmu_pt *pte_address;
	int ret = -EPERM;

	start_pde_num = MMU_PDE_NUM(addr);
	end_pde_num = MMU_PDE_NUM((char *)addr + size - 1);
	starting_pte_num = MMU_PAGE_NUM((char *)addr);

	for (pdpte = start_pdpte_num; pdpte <= end_pdpte_num; pdpte++) {
		if (pdpte != start_pdpte_num) {
			start_pde_num = 0U;
		}

		if (pdpte != end_pdpte_num) {
			end_pde_num = 0U;
		} else {
			end_pde_num = MMU_PDE_NUM((char *)addr + size - 1);
		}

		/* Ensure page directory pointer table entry is present */
		if (X86_MMU_GET_PDPTE_INDEX(&USER_PDPT, pdpte)->p == 0) {
			goto out;
		}

		struct x86_mmu_pd *pd_address =
			X86_MMU_GET_PD_ADDR_INDEX(&USER_PDPT, pdpte);

		/* Iterate for all the pde's the buffer might take up.
		 * (depends on the size of the buffer and start address
		 * of the buff)
		 */
		for (pde = start_pde_num; pde <= end_pde_num; pde++) {
			union x86_mmu_pde_pt pde_value =
				pd_address->entry[pde].pt;

			if ((pde_value.p) == 0 ||
			    (pde_value.us) == 0 ||
			    ((write != 0) && (pde_value.rw == 0))) {
				goto out;
			}

			pte_address = (struct x86_mmu_pt *)
				(pde_value.pt << MMU_PAGE_SHIFT);

			/* loop over all the possible page tables for the
			 * required size. If the pde is not the last one
			 * then the last pte would be 511. So each pde
			 * will be using all the page table entries except
			 * for the last pde. For the last pde, pte is
			 * calculated using the last memory address
			 * of the buffer.
			 */
			if (pde != end_pde_num) {
				ending_pte_num = 511U;
			} else {
				ending_pte_num =
					MMU_PAGE_NUM((char *)addr + size - 1);
			}

			/* For all the pde's appart from the starting pde,
			 * will have the start pte number as zero.
			 */
			if (pde != start_pde_num) {
				starting_pte_num = 0U;
			}

			pte_value.value = 0xFFFFFFFFU;

			/* Bitwise AND all the pte values.
			 * An optimization done to make sure a compare is
			 * done only once.
			 */
			for (pte = starting_pte_num;
			     pte <= ending_pte_num;
			     pte++) {
				pte_value.value &=
					pte_address->entry[pte].value;
			}

			if ((pte_value.p) == 0 ||
			    (pte_value.us) == 0 ||
			    ((write != 0) && (pte_value.rw == 0))) {
				goto out;
			}
		}
	}
	ret = 0;
out:
#ifdef CONFIG_BOUNDS_CHECK_BYPASS_MITIGATION
	__asm__ volatile ("lfence" : : : "memory");
#endif

	return ret;
}

static inline void tlb_flush_page(void *addr)
{
	/* Invalidate TLB entries corresponding to the page containing the
	 * specified address
	 */
	char *page = (char *)addr;

	__asm__ ("invlpg %0" :: "m" (*page));
}


void z_x86_mmu_set_flags(struct x86_mmu_pdpt *pdpt, void *ptr,
			size_t size,
			x86_page_entry_data_t flags,
			x86_page_entry_data_t mask)
{
	union x86_mmu_pte *pte;

	u32_t addr = (u32_t)ptr;

	__ASSERT((addr & MMU_PAGE_MASK) == 0U, "unaligned address provided");
	__ASSERT((size & MMU_PAGE_MASK) == 0U, "unaligned size provided");

	/* L1TF mitigation: non-present PTEs will have address fields
	 * zeroed. Expand the mask to include address bits if we are changing
	 * the present bit.
	 */
	if ((mask & MMU_PTE_P_MASK) != 0) {
		mask |= MMU_PTE_PAGE_MASK;
	}

	while (size != 0) {
		x86_page_entry_data_t cur_flags = flags;

		/* TODO we're not generating 2MB entries at the moment */
		__ASSERT(X86_MMU_GET_PDE(pdpt, addr)->ps != 1, "2MB PDE found");
		pte = X86_MMU_GET_PTE(pdpt, addr);

		/* If we're setting the present bit, restore the address
		 * field. If we're clearing it, then the address field
		 * will be zeroed instead, mapping the PTE to the NULL page.
		 */
		if (((mask & MMU_PTE_P_MASK) != 0) &&
		    ((flags & MMU_ENTRY_PRESENT) != 0)) {
			cur_flags |= addr;
		}

		pte->value = (pte->value & ~mask) | cur_flags;
		tlb_flush_page((void *)addr);

		size -= MMU_PAGE_SIZE;
		addr += MMU_PAGE_SIZE;
	}
}

#ifdef CONFIG_X86_USERSPACE
void z_x86_reset_pages(void *start, size_t size)
{
#ifdef CONFIG_X86_KPTI
	/* Clear both present bit and access flags. Only applies
	 * to threads running in user mode.
	 */
	z_x86_mmu_set_flags(&z_x86_user_pdpt, start, size,
			   MMU_ENTRY_NOT_PRESENT,
			   K_MEM_PARTITION_PERM_MASK | MMU_PTE_P_MASK);
#else
	/* Mark as supervisor read-write, user mode no access */
	z_x86_mmu_set_flags(&z_x86_kernel_pdpt, start, size,
			   K_MEM_PARTITION_P_RW_U_NA,
			   K_MEM_PARTITION_PERM_MASK);
#endif /* CONFIG_X86_KPTI */
}

static inline void activate_partition(struct k_mem_partition *partition)
{
	/* Set the partition attributes */
	u64_t attr, mask;

#if CONFIG_X86_KPTI
	attr = partition->attr | MMU_ENTRY_PRESENT;
	mask = K_MEM_PARTITION_PERM_MASK | MMU_PTE_P_MASK;
#else
	attr = partition->attr;
	mask = K_MEM_PARTITION_PERM_MASK;
#endif /* CONFIG_X86_KPTI */

	z_x86_mmu_set_flags(&USER_PDPT,
			    (void *)partition->start,
			    partition->size, attr, mask);
}

/* Helper macros needed to be passed to x86_update_mem_domain_pages */
#define X86_MEM_DOMAIN_SET_PAGES   (0U)
#define X86_MEM_DOMAIN_RESET_PAGES (1U)
/* Pass 1 to page_conf if reset of mem domain pages is needed else pass a 0*/
static inline void x86_mem_domain_pages_update(struct k_mem_domain *mem_domain,
						u32_t page_conf)
{
	u32_t partition_index;
	u32_t total_partitions;
	struct k_mem_partition *partition;
	u32_t partitions_count;

	/* If mem_domain doesn't point to a valid location return.*/
	if (mem_domain == NULL) {
		goto out;
	}

	/* Get the total number of partitions*/
	total_partitions = mem_domain->num_partitions;

	/* Iterate over all the partitions for the given mem_domain
	 * For x86: iterate over all the partitions and set the
	 * required flags in the correct MMU page tables.
	 */
	partitions_count = 0U;
	for (partition_index = 0U;
	     partitions_count < total_partitions;
	     partition_index++) {

		/* Get the partition info */
		partition = &mem_domain->partitions[partition_index];
		if (partition->size == 0U) {
			continue;
		}
		partitions_count++;
		if (page_conf == X86_MEM_DOMAIN_SET_PAGES) {
			activate_partition(partition);
		} else {
			z_x86_reset_pages((void *)partition->start,
					  partition->size);
		}
	}
out:
	return;
}

/* Load the partitions of the thread. */
void z_arch_mem_domain_configure(struct k_thread *thread)
{
	x86_mem_domain_pages_update(thread->mem_domain_info.mem_domain,
				     X86_MEM_DOMAIN_SET_PAGES);
}

/* Destroy or reset the mmu page tables when necessary.
 * Needed when either swap takes place or k_mem_domain_destroy is called.
 */
void z_arch_mem_domain_destroy(struct k_mem_domain *domain)
{
	x86_mem_domain_pages_update(domain, X86_MEM_DOMAIN_RESET_PAGES);
}

/* Reset/destroy one partition specified in the argument of the API. */
void z_arch_mem_domain_partition_remove(struct k_mem_domain *domain,
					u32_t partition_id)
{
	struct k_mem_partition *partition;

	__ASSERT_NO_MSG(domain != NULL);
	__ASSERT(partition_id <= domain->num_partitions,
		 "invalid partitions");

	partition = &domain->partitions[partition_id];
	z_x86_reset_pages((void *)partition->start, partition->size);
}

/* Add one partition specified in the argument of the API. */
void z_arch_mem_domain_partition_add(struct k_mem_domain *domain,
				    u32_t partition_id)
{
	struct k_mem_partition *partition;

	__ASSERT_NO_MSG(domain != NULL);
	__ASSERT(partition_id <= domain->num_partitions,
		 "invalid partitions");

	partition = &domain->partitions[partition_id];
	activate_partition(partition);
}

int z_arch_mem_domain_max_partitions_get(void)
{
	return CONFIG_MAX_DOMAIN_PARTITIONS;
}
#endif	/* CONFIG_X86_USERSPACE*/