include/arch/x86/mmustructs.h

/*
 * Copyright (c) 2011-2014 Wind River Systems, Inc.
 * Copyright (c) 2017 Intel Corporation
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#ifndef ZEPHYR_INCLUDE_ARCH_X86_MMUSTRUCTS_H_
#define ZEPHYR_INCLUDE_ARCH_X86_MMUSTRUCTS_H_

#include <sys/util.h>

#define MMU_PAGE_SIZE 4096UL
#define MMU_PAGE_MASK 0xfffU
#define MMU_PAGE_SHIFT 12U
#define PAGES(x) ((x) << (MMU_PAGE_SHIFT))
#define MMU_ARE_IN_SAME_PAGE(a, b) \
	(((u32_t)(a) & ~MMU_PAGE_MASK) == ((u32_t)(b) & ~MMU_PAGE_MASK))
#define MMU_IS_ON_PAGE_BOUNDARY(a) (!((u32_t)(a) & MMU_PAGE_MASK))

/*
 * Common flags in the same bit position regardless of which structure level,
 * although not every flag is supported at every level, and some may be
 * ignored depending on the state of other bits (such as P or PS)
 *
 * These flags indicate bit position, and can be used for setting flags or
 * masks as needed.
 */

#define Z_X86_MMU_P		BIT64(0)	/** Present */
#define Z_X86_MMU_RW		BIT64(1)	/** Read-Write */
#define Z_X86_MMU_US		BIT64(2)	/** User-Supervisor */
#define Z_X86_MMU_PWT		BIT64(3)	/** Page Write Through */
#define Z_X86_MMU_PCD		BIT64(4)	/** Page Cache Disable */
#define Z_X86_MMU_A		BIT64(5)	/** Accessed */
#define Z_X86_MMU_D		BIT64(6)	/** Dirty */
#define Z_X86_MMU_PS		BIT64(7)	/** Page Size */
#define Z_X86_MMU_G		BIT64(8)	/** Global */
#define Z_X86_MMU_XD		BIT64(63)	/** Execute Disable */

#ifdef CONFIG_X86_64
#define Z_X86_MMU_PROT_KEY_MASK		0x7800000000000000ULL
#endif

/*
 * Structure-specific flags / masks
 */
#define Z_X86_MMU_PDPTE_PAT	BIT64(12)
#define Z_X86_MMU_PDE_PAT	BIT64(12)
#define Z_X86_MMU_PTE_PAT	BIT64(7)	/** Page Attribute Table */

/* The true size of the mask depends on MAXADDR, which is found at run-time.
 * As a simplification, roll the area for the memory address, and the
 * reserved or ignored regions immediately above it, into a single area.
 * This will work as expected if valid memory addresses are written.
 */
#ifdef CONFIG_X86_64
#define Z_X86_MMU_PML4E_PDPT_MASK	0x7FFFFFFFFFFFF000ULL
#endif
#define Z_X86_MMU_PDPTE_PD_MASK		0x7FFFFFFFFFFFF000ULL
#ifdef CONFIG_X86_64
#define Z_X86_MMU_PDPTE_1G_MASK		0x07FFFFFFC0000000ULL
#endif
#define Z_X86_MMU_PDE_PT_MASK		0x7FFFFFFFFFFFF000ULL
#define Z_X86_MMU_PDE_2MB_MASK		0x07FFFFFFFFC00000ULL
#define Z_X86_MMU_PTE_ADDR_MASK		0x07FFFFFFFFFFF000ULL

/*
 * These flags indicate intention when setting access properties.
 */

#define MMU_ENTRY_NOT_PRESENT       0ULL
#define MMU_ENTRY_PRESENT           Z_X86_MMU_P

#define MMU_ENTRY_READ              0ULL
#define MMU_ENTRY_WRITE             Z_X86_MMU_RW

#define MMU_ENTRY_SUPERVISOR        0ULL
#define MMU_ENTRY_USER              Z_X86_MMU_US

#define MMU_ENTRY_WRITE_BACK        0ULL
#define MMU_ENTRY_WRITE_THROUGH     Z_X86_MMU_PWT

#define MMU_ENTRY_CACHING_ENABLE    0ULL
#define MMU_ENTRY_CACHING_DISABLE   Z_X86_MMU_PCD

#define MMU_ENTRY_NOT_ACCESSED      0ULL
#define MMU_ENTRY_ACCESSED          Z_X86_MMU_A

#define MMU_ENTRY_NOT_DIRTY         0ULL
#define MMU_ENTRY_DIRTY             Z_X86_MMU_D

#define MMU_ENTRY_NOT_GLOBAL        0ULL
#define MMU_ENTRY_GLOBAL            Z_X86_MMU_G

#define MMU_ENTRY_EXECUTE_DISABLE   Z_X86_MMU_XD
#define MMU_ENTRY_EXECUTE_ENABLE    0ULL

/* memory partition arch/soc independent attribute */
#define K_MEM_PARTITION_P_RW_U_RW   (MMU_ENTRY_WRITE | \
				     MMU_ENTRY_USER  | \
				     MMU_ENTRY_EXECUTE_DISABLE)

#define K_MEM_PARTITION_P_RW_U_NA   (MMU_ENTRY_WRITE | \
				     MMU_ENTRY_SUPERVISOR | \
				     MMU_ENTRY_EXECUTE_DISABLE)

#define K_MEM_PARTITION_P_RO_U_RO   (MMU_ENTRY_READ | \
				     MMU_ENTRY_USER | \
				     MMU_ENTRY_EXECUTE_DISABLE)

#define K_MEM_PARTITION_P_RO_U_NA   (MMU_ENTRY_READ  | \
				     MMU_ENTRY_SUPERVISOR | \
				     MMU_ENTRY_EXECUTE_DISABLE)

/* Execution-allowed attributes */
#define K_MEM_PARTITION_P_RWX_U_RWX (MMU_ENTRY_WRITE | MMU_ENTRY_USER)

#define K_MEM_PARTITION_P_RWX_U_NA  (MMU_ENTRY_WRITE | MMU_ENTRY_SUPERVISOR)

#define K_MEM_PARTITION_P_RX_U_RX   (MMU_ENTRY_READ | MMU_ENTRY_USER)

#define K_MEM_PARTITION_P_RX_U_NA   (MMU_ENTRY_READ | MMU_ENTRY_SUPERVISOR)


 /* memory partition access permission mask */
#define K_MEM_PARTITION_PERM_MASK   (Z_X86_MMU_RW | Z_X86_MMU_US | \
				     Z_X86_MMU_XD)

#ifndef _ASMLANGUAGE
#include <sys/__assert.h>
#include <zephyr/types.h>

/* Structure used by gen_mmu.py to create page directories and page tables.
 * In order to populate this structure use macro MMU_BOOT_REGION.
 */
struct mmu_region {
	uintptr_t address; /*Start address of the memory region */
	size_t size; /* Size of the memory region*/
	u64_t flags; /* Permissions needed for this region*/
};

/* permission_flags are calculated using the macros
 * region_size has to be provided in bytes
 * for read write access = MMU_ENTRY_READ/MMU_ENTRY_WRITE
 * for supervisor/user mode access = MMU_ENTRY_SUPERVISOR/MMU_ENTRY_USER
 *
 * Preprocessor indirection layers used to ensure __COUNTER__ is expanded
 * properly.
 */

#define __MMU_BOOT_REGION(id, addr, region_size, permission_flags)	\
	static const Z_STRUCT_SECTION_ITERABLE(mmu_region, region_##id) =	\
	{								\
		.address = (uintptr_t)(addr),				\
		.size = (size_t)(region_size),				\
		.flags = (permission_flags),				\
	}

#define Z_MMU_BOOT_REGION(id, addr, region_size, permission_flags)	\
	__MMU_BOOT_REGION(id, addr, region_size, permission_flags)

#define MMU_BOOT_REGION(addr, region_size, permission_flags)		\
	Z_MMU_BOOT_REGION(__COUNTER__, addr, region_size, permission_flags)

#ifdef CONFIG_X86_64
#define Z_X86_NUM_PML4_ENTRIES	512U
#define Z_X86_NUM_PDPT_ENTRIES	512U
#else
#define Z_X86_NUM_PDPT_ENTRIES	4U
#endif
#define Z_X86_NUM_PD_ENTRIES	512U
#define Z_X86_NUM_PT_ENTRIES	512U

/* Memory range covered by an instance of various table types */
#define Z_X86_PT_AREA	(MMU_PAGE_SIZE * Z_X86_NUM_PT_ENTRIES)
#define Z_X86_PD_AREA	(Z_X86_PT_AREA * Z_X86_NUM_PD_ENTRIES)
#define Z_X86_PDPT_AREA (Z_X86_PD_AREA * Z_X86_NUM_PDPT_ENTRIES)

typedef u64_t k_mem_partition_attr_t;

#ifdef CONFIG_X86_64
struct x86_mmu_pml4 {
	u64_t entry[Z_X86_NUM_PML4_ENTRIES];
};
#endif

struct x86_mmu_pdpt {
	u64_t entry[Z_X86_NUM_PDPT_ENTRIES];
};

struct x86_mmu_pd {
	u64_t entry[Z_X86_NUM_PD_ENTRIES];
};

struct x86_mmu_pt {
	u64_t entry[Z_X86_NUM_PT_ENTRIES];
};

struct x86_page_tables {
#ifdef CONFIG_X86_64
	struct x86_mmu_pml4 pml4;
#else
	struct x86_mmu_pdpt pdpt;
#endif
};

/*
 * Inline functions for getting the next linked structure
 */
#ifdef CONFIG_X86_64
static inline u64_t *z_x86_pml4_get_pml4e(struct x86_mmu_pml4 *pml4,
					  uintptr_t addr)
{
	int index = (addr >> 39U) & (Z_X86_NUM_PML4_ENTRIES - 1);

	return &pml4->entry[index];
}

static inline struct x86_mmu_pdpt *z_x86_pml4e_get_pdpt(u64_t pml4e)
{
	uintptr_t addr = pml4e & Z_X86_MMU_PML4E_PDPT_MASK;

	return (struct x86_mmu_pdpt *)addr;
}
#endif

static inline u64_t *z_x86_pdpt_get_pdpte(struct x86_mmu_pdpt *pdpt,
					  uintptr_t addr)
{
	int index = (addr >> 30U) & (Z_X86_NUM_PDPT_ENTRIES - 1);

	return &pdpt->entry[index];
}

static inline struct x86_mmu_pd *z_x86_pdpte_get_pd(u64_t pdpte)
{
	uintptr_t addr = pdpte & Z_X86_MMU_PDPTE_PD_MASK;

#ifdef CONFIG_X86_64
	__ASSERT((pdpte & Z_X86_MMU_PS) == 0, "PDPT is for 1GB page");
#endif
	return (struct x86_mmu_pd *)addr;
}

static inline u64_t *z_x86_pd_get_pde(struct x86_mmu_pd *pd, uintptr_t addr)
{
	int index = (addr >> 21U) & (Z_X86_NUM_PD_ENTRIES - 1);

	return &pd->entry[index];
}

static inline struct x86_mmu_pt *z_x86_pde_get_pt(u64_t pde)
{
	uintptr_t addr = pde & Z_X86_MMU_PDE_PT_MASK;

	__ASSERT((pde & Z_X86_MMU_PS) == 0, "pde is for 2MB page");

	return (struct x86_mmu_pt *)addr;
}

static inline u64_t *z_x86_pt_get_pte(struct x86_mmu_pt *pt, uintptr_t addr)
{
	int index = (addr >> 12U) & (Z_X86_NUM_PT_ENTRIES - 1);

	return &pt->entry[index];
}

/*
 * Inline functions for obtaining page table structures from the top-level
 */

#ifdef CONFIG_X86_64
static inline struct x86_mmu_pml4 *
z_x86_get_pml4(struct x86_page_tables *ptables)
{
	return &ptables->pml4;
}

static inline u64_t *z_x86_get_pml4e(struct x86_page_tables *ptables,
				     uintptr_t addr)
{
	return z_x86_pml4_get_pml4e(z_x86_get_pml4(ptables), addr);
}

static inline struct x86_mmu_pdpt *
z_x86_get_pdpt(struct x86_page_tables *ptables, uintptr_t addr)
{
	return z_x86_pml4e_get_pdpt(*z_x86_get_pml4e(ptables, addr));
}
#else
static inline struct x86_mmu_pdpt *
z_x86_get_pdpt(struct x86_page_tables *ptables, uintptr_t addr)
{
	ARG_UNUSED(addr);

	return &ptables->pdpt;
}
#endif /* CONFIG_X86_64 */

static inline u64_t *z_x86_get_pdpte(struct x86_page_tables *ptables,
				       uintptr_t addr)
{
	return z_x86_pdpt_get_pdpte(z_x86_get_pdpt(ptables, addr), addr);
}

static inline struct x86_mmu_pd *
z_x86_get_pd(struct x86_page_tables *ptables, uintptr_t addr)
{
	return z_x86_pdpte_get_pd(*z_x86_get_pdpte(ptables, addr));
}

static inline u64_t *z_x86_get_pde(struct x86_page_tables *ptables,
				     uintptr_t addr)
{
	return z_x86_pd_get_pde(z_x86_get_pd(ptables, addr), addr);
}

static inline struct x86_mmu_pt *
z_x86_get_pt(struct x86_page_tables *ptables, uintptr_t addr)
{
	return z_x86_pde_get_pt(*z_x86_get_pde(ptables, addr));
}

static inline u64_t *z_x86_get_pte(struct x86_page_tables *ptables,
				     uintptr_t addr)
{
	return z_x86_pt_get_pte(z_x86_get_pt(ptables, addr), addr);
}

/**
 * Debug function for dumping out page tables
 *
 * Iterates through the entire linked set of page table structures,
 * dumping out codes for the configuration of each table entry.
 *
 * Entry codes:
 *
 *   . - not present
 *   w - present, writable, not executable
 *   a - present, writable, executable
 *   r - present, read-only, not executable
 *   x - present, read-only, executable
 *
 * Entry codes in uppercase indicate that user mode may access.
 *
 * @param ptables Top-level pointer to the page tables, as programmed in CR3
 */
void z_x86_dump_page_tables(struct x86_page_tables *ptables);

static inline struct x86_page_tables *z_x86_page_tables_get(void)
{
	struct x86_page_tables *ret;

	__asm__ volatile("movl %%cr3, %0\n\t" : "=r" (ret));

	return ret;
}

/* Kernel's page table. Always active when threads are running in supervisor
 * mode, or handling an interrupt.
 *
 * If KPTI is not enabled, this is used as a template to create per-thread
 * page tables for when threads run in user mode.
 */
extern struct x86_page_tables z_x86_kernel_ptables;
#ifdef CONFIG_X86_KPTI
/* Separate page tables for user mode threads. This is never installed into the
 * CPU; instead it is used as a template for creating per-thread page tables.
 */
extern struct x86_page_tables z_x86_user_ptables;
#define USER_PTABLES	z_x86_user_ptables
#else
#define USER_PTABLES	z_x86_kernel_ptables
#endif
/**
 * @brief Fetch page table flags for a particular page
 *
 * Given a memory address, return the flags for the containing page's
 * PDE and PTE entries. Intended for debugging.
 *
 * @param ptables Which set of page tables to use
 * @param addr Memory address to example
 * @param pde_flags Output parameter for page directory entry flags
 * @param pte_flags Output parameter for page table entry flags
 */
void z_x86_mmu_get_flags(struct x86_page_tables *ptables, void *addr,
			 u64_t *pde_flags, u64_t *pte_flags);

/**
 * @brief set flags in the MMU page tables
 *
 * Modify bits in the existing page tables for a particular memory
 * range, which must be page-aligned
 *
 * @param ptables Which set of page tables to use
 * @param ptr Starting memory address which must be page-aligned
 * @param size Size of the region, must be page size multiple
 * @param flags Value of bits to set in the page table entries
 * @param mask Mask indicating which particular bits in the page table entries
 *             to modify
 * @param flush Whether to flush the TLB for the modified pages, only needed
 *              when modifying the active page tables
 */
void z_x86_mmu_set_flags(struct x86_page_tables *ptables, void *ptr,
			 size_t size, u64_t flags, u64_t mask, bool flush);

int z_x86_mmu_validate(struct x86_page_tables *ptables, void *addr, size_t size,
		       bool write);
#endif /* _ASMLANGUAGE */

#endif /* ZEPHYR_INCLUDE_ARCH_X86_MMUSTRUCTS_H_ */