arch/x86/include/x86_mmu.h

/*
 * Copyright (c) 2011-2014 Wind River Systems, Inc.
 * Copyright (c) 2017-2020 Intel Corporation
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * Internal memory management interfaces implemented in x86_mmu.c.
 * None of these are application-facing, use only if you know what you are
 * doing!
 */

#ifndef ZEPHYR_ARCH_X86_INCLUDE_X86_MMU_H
#define ZEPHYR_ARCH_X86_INCLUDE_X86_MMU_H

#include <zephyr/kernel.h>
#include <zephyr/arch/x86/mmustructs.h>
#include <zephyr/kernel/mm.h>

#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
#define XD_SUPPORTED
#define BITL		BIT64
#define PRI_ENTRY	"0x%016llx"
#else
#define BITL		BIT
#define PRI_ENTRY	"0x%08x"
#endif

/*
 * Common flags in the same bit position regardless of which structure level,
 * although not every flag is supported at every level, and some may be
 * ignored depending on the state of other bits (such as P or PS)
 *
 * These flags indicate bit position, and can be used for setting flags or
 * masks as needed.
 */

#define MMU_P		BITL(0)		/** Present */
#define MMU_RW		BITL(1)		/** Read-Write */
#define MMU_US		BITL(2)		/** User-Supervisor */
#define MMU_PWT		BITL(3)		/** Page Write Through */
#define MMU_PCD		BITL(4)		/** Page Cache Disable */
#define MMU_A		BITL(5)		/** Accessed */
#define MMU_D		BITL(6)		/** Dirty */
#define MMU_PS		BITL(7)		/** Page Size (non PTE)*/
#define MMU_PAT		BITL(7)		/** Page Attribute (PTE) */
#define MMU_G		BITL(8)		/** Global */
#ifdef XD_SUPPORTED
#define MMU_XD		BITL(63)	/** Execute Disable */
#else
#define MMU_XD		0
#endif

/* Unused PTE bits ignored by the CPU, which we use for our own OS purposes.
 * These bits ignored for all paging modes.
 */
#define MMU_IGNORED0	BITL(9)
#define MMU_IGNORED1	BITL(10)
#define MMU_IGNORED2	BITL(11)

/* Page fault error code flags. See Chapter 4.7 of the Intel SDM vol. 3A. */
#define PF_P		BIT(0)	/* 0 Non-present page  1 Protection violation */
#define PF_WR		BIT(1)  /* 0 Read              1 Write */
#define PF_US		BIT(2)  /* 0 Supervisor mode   1 User mode */
#define PF_RSVD		BIT(3)  /* 1 reserved bit set */
#define PF_ID		BIT(4)  /* 1 instruction fetch */
#define PF_PK		BIT(5)  /* 1 protection-key violation */
#define PF_SGX		BIT(15) /* 1 SGX-specific access control requirements */

#ifndef _ASMLANGUAGE

#ifdef CONFIG_EXCEPTION_DEBUG
/**
 * Dump out page table entries for a particular virtual memory address
 *
 * For the provided memory address, dump out interesting information about
 * its mapping to the error log
 *
 * @param ptables Page tables to walk
 * @param virt Virtual address to inspect
 */
void z_x86_dump_mmu_flags(pentry_t *ptables, void *virt);

/**
 * Fetch the page table entry for a virtual memory address
 *
 * @param paging_level [out] what paging level the entry was found at.
 *                     0=toplevel
 * @param val Value stored in page table entry, with address and flags
 * @param ptables Toplevel pointer to page tables
 * @param virt Virtual address to lookup
 */
void z_x86_pentry_get(int *paging_level, pentry_t *val, pentry_t *ptables,
		      void *virt);

/**
 * Debug function for dumping out page tables
 *
 * Iterates through the entire linked set of page table structures,
 * dumping out codes for the configuration of each table entry.
 *
 * Entry codes:
 *
 *   . - not present
 *   w - present, writable, not executable
 *   a - present, writable, executable
 *   r - present, read-only, not executable
 *   x - present, read-only, executable
 *
 * Entry codes in uppercase indicate that user mode may access.
 *
 * Color is used to indicate the physical mapping characteristics:
 *
 *   yellow - Identity mapping (virt = phys)
 *    green - Fixed virtual memory mapping (virt = phys + constant)
 *  magenta - entry is child page table
 *     cyan - General mapped memory
 *
 * @param ptables Top-level pointer to the page tables, as programmed in CR3
 */
void z_x86_dump_page_tables(pentry_t *ptables);
#endif /* CONFIG_EXCEPTION_DEBUG */

#ifdef CONFIG_X86_STACK_PROTECTION
/* Legacy function - set identity-mapped MMU stack guard page to RO in the
 * kernel's page tables to prevent writes and generate an exception
 */
void z_x86_set_stack_guard(k_thread_stack_t *stack);
#endif

#ifdef CONFIG_USERSPACE
#ifdef CONFIG_X86_KPTI
/* Defined in linker script. Contains all the data that must be mapped
 * in a KPTI table even though US bit is not set (trampoline stack, GDT,
 * IDT, etc)
 */
extern uint8_t z_shared_kernel_page_start;

#ifdef CONFIG_DEMAND_PAGING
/* Called from page fault handler. ptables here is the ptage tables for the
 * faulting user thread and not the current set of page tables
 */
extern bool z_x86_kpti_is_access_ok(void *virt, pentry_t *ptables)
#endif /* CONFIG_DEMAND_PAGING */
#endif /* CONFIG_X86_KPTI */
#endif /* CONFIG_USERSPACE */

#ifdef CONFIG_X86_PAE
#define PTABLES_ALIGN	0x1fU
#else
#define PTABLES_ALIGN	0xfffU
#endif

/* Set CR3 to a physical address. There must be a valid top-level paging
 * structure here or the CPU will triple fault. The incoming page tables must
 * have the same kernel mappings wrt supervisor mode. Don't use this function
 * unless you know exactly what you are doing.
 */
static inline void z_x86_cr3_set(uintptr_t phys)
{
	__ASSERT((phys & PTABLES_ALIGN) == 0U, "unaligned page tables");
#ifdef CONFIG_X86_64
	__asm__ volatile("movq %0, %%cr3\n\t" : : "r" (phys) : "memory");
#else
	__asm__ volatile("movl %0, %%cr3\n\t" : : "r" (phys) : "memory");
#endif
}

/* Return cr3 value, which is the physical (not virtual) address of the
 * current set of page tables
 */
static inline uintptr_t z_x86_cr3_get(void)
{
	uintptr_t cr3;
#ifdef CONFIG_X86_64
	__asm__ volatile("movq %%cr3, %0\n\t" : "=r" (cr3));
#else
	__asm__ volatile("movl %%cr3, %0\n\t" : "=r" (cr3));
#endif
	return cr3;
}

/* Return the virtual address of the page tables installed in this CPU in CR3 */
static inline pentry_t *z_x86_page_tables_get(void)
{
	return k_mem_virt_addr(z_x86_cr3_get());
}

/* Return cr2 value, which contains the page fault linear address.
 * See Section 6.15 of the IA32 Software Developer's Manual vol 3.
 * Used by page fault handling code.
 */
static inline void *z_x86_cr2_get(void)
{
	void *cr2;
#ifdef CONFIG_X86_64
	__asm__ volatile("movq %%cr2, %0\n\t" : "=r" (cr2));
#else
	__asm__ volatile("movl %%cr2, %0\n\t" : "=r" (cr2));
#endif
	return cr2;
}

/* Kernel's page table. This is in CR3 for all supervisor threads.
 * if KPTI is enabled, we switch to this when handling exceptions or syscalls
 */
extern pentry_t z_x86_kernel_ptables[];

/* Get the page tables used by this thread during normal execution */
static inline pentry_t *z_x86_thread_page_tables_get(struct k_thread *thread)
{
#if defined(CONFIG_USERSPACE) && !defined(CONFIG_X86_COMMON_PAGE_TABLE)
	if (!IS_ENABLED(CONFIG_X86_KPTI) ||
	    (thread->base.user_options & K_USER) != 0U) {
		/* If KPTI is enabled, supervisor threads always use
		 * the kernel's page tables and not the page tables associated
		 * with their memory domain.
		 */
		return k_mem_virt_addr(thread->arch.ptables);
	}
#else
	ARG_UNUSED(thread);
#endif
	return z_x86_kernel_ptables;
}

#ifdef CONFIG_SMP
/* Handling function for TLB shootdown inter-processor interrupts. */
void z_x86_tlb_ipi(const void *arg);
#endif

#ifdef CONFIG_X86_COMMON_PAGE_TABLE
void z_x86_swap_update_common_page_table(struct k_thread *incoming);
#endif

/* Early-boot paging setup tasks, called from prep_c */
void z_x86_mmu_init(void);
#endif /* _ASMLANGUAGE */
#endif /* ZEPHYR_ARCH_X86_INCLUDE_X86_MMU_H */