arch/xtensa/core/include/xtensa_mmu_priv.h

/*
 * Xtensa MMU support
 *
 * Private data declarations
 *
 * Copyright (c) 2022 Intel Corporation
 * SPDX-License-Identifier: Apache-2.0
 */

#ifndef ZEPHYR_ARCH_XTENSA_XTENSA_MMU_PRIV_H_
#define ZEPHYR_ARCH_XTENSA_XTENSA_MMU_PRIV_H_

#include <stdint.h>
#include <xtensa/config/core-isa.h>
#include <zephyr/toolchain.h>
#include <zephyr/sys/util_macro.h>

#define Z_XTENSA_PTE_VPN_MASK 0xFFFFF000U
#define Z_XTENSA_PTE_PPN_MASK 0xFFFFF000U
#define Z_XTENSA_PTE_ATTR_MASK 0x0000000FU
#define Z_XTENSA_L1_MASK 0x3FF00000U
#define Z_XTENSA_L2_MASK 0x3FFFFFU

#define Z_XTENSA_PPN_SHIFT 12U

#define Z_XTENSA_PTE_RING_MASK 0x00000030U

#define Z_XTENSA_PTE(paddr, ring, attr) \
	(((paddr) & Z_XTENSA_PTE_PPN_MASK) | \
	(((ring) << 4) & Z_XTENSA_PTE_RING_MASK) | \
	((attr) & Z_XTENSA_PTE_ATTR_MASK))

#define Z_XTENSA_TLB_ENTRY(vaddr, way) \
	(((vaddr) & Z_XTENSA_PTE_PPN_MASK) | (way))

#define Z_XTENSA_AUTOFILL_TLB_ENTRY(vaddr) \
	(((vaddr) & Z_XTENSA_PTE_PPN_MASK) | \
	 (((vaddr) >> Z_XTENSA_PPN_SHIFT) & 0x03U))

#define Z_XTENSA_L2_POS(vaddr) \
	(((vaddr) & Z_XTENSA_L2_MASK) >> Z_XTENSA_PPN_SHIFT)


/* Number of data TLB ways [0-9] */
#define Z_XTENSA_DTLB_WAYS 10

/* Number of instruction TLB ways [0-6] */
#define Z_XTENSA_ITLB_WAYS 7

/* Number of auto-refill ways */
#define Z_XTENSA_TLB_AUTOREFILL_WAYS 4


/* PITLB HIT bit. For more information see
 * Xtensa Instruction Set Architecture (ISA) Reference Manual
 * 4.6.5.7 Formats for Probing MMU Option TLB Entries
 */
#define Z_XTENSA_PITLB_HIT BIT(3)

/* PDTLB HIT bit. For more information see
 * Xtensa Instruction Set Architecture (ISA) Reference Manual
 * 4.6.5.7 Formats for Probing MMU Option TLB Entries
 */
#define Z_XTENSA_PDTLB_HIT BIT(4)

/*
 * Virtual address where the page table is mapped
 */
#define Z_XTENSA_PTEVADDR CONFIG_XTENSA_MMU_PTEVADDR

/*
 * Find the pte entry address of a given vaddr.
 *
 * For example, assuming PTEVADDR in 0xE0000000,
 * the page spans from 0xE0000000 - 0xE03FFFFF

 *
 * address 0x00 is in 0xE0000000
 * address 0x1000 is in 0xE0000004
 * .....
 * address 0xE0000000 (where the page is) is in 0xE0380000
 *
 * Generalizing it, any PTE virtual address can be calculated this way:
 *
 * PTE_ENTRY_ADDRESS = PTEVADDR + ((VADDR / 4096) * 4)
 */
#define Z_XTENSA_PTE_ENTRY_VADDR(vaddr) \
	(Z_XTENSA_PTEVADDR + (((vaddr) / KB(4)) * 4))

/*
 * The address of the top level page where the page
 * is located in the virtual address.
 */
#define Z_XTENSA_PAGE_TABLE_VADDR \
	Z_XTENSA_PTE_ENTRY_VADDR(Z_XTENSA_PTEVADDR)

static ALWAYS_INLINE void xtensa_rasid_set(uint32_t rasid)
{
	__asm__ volatile("wsr %0, rasid\n\t"
			"isync\n" : : "a"(rasid));
}

static ALWAYS_INLINE uint32_t xtensa_rasid_get(void)
{
	uint32_t rasid;

	__asm__ volatile("rsr %0, rasid" : "=a"(rasid));
	return rasid;
}

static ALWAYS_INLINE void xtensa_itlb_entry_invalidate(uint32_t entry)
{
	__asm__ volatile("iitlb %0\n\t"
			: : "a" (entry));
}

static ALWAYS_INLINE void xtensa_itlb_entry_invalidate_sync(uint32_t entry)
{
	__asm__ volatile("iitlb %0\n\t"
			"isync\n\t"
			: : "a" (entry));
}

static ALWAYS_INLINE void xtensa_dtlb_entry_invalidate_sync(uint32_t entry)
{
	__asm__ volatile("idtlb %0\n\t"
			"dsync\n\t"
			: : "a" (entry));
}

static ALWAYS_INLINE void xtensa_dtlb_entry_invalidate(uint32_t entry)
{
	__asm__ volatile("idtlb %0\n\t"
			: : "a" (entry));
}

static ALWAYS_INLINE void xtensa_dtlb_entry_write_sync(uint32_t pte, uint32_t entry)
{
	__asm__ volatile("wdtlb %0, %1\n\t"
			"dsync\n\t"
			: : "a" (pte), "a"(entry));
}

static ALWAYS_INLINE void xtensa_dtlb_entry_write(uint32_t pte, uint32_t entry)
{
	__asm__ volatile("wdtlb %0, %1\n\t"
			: : "a" (pte), "a"(entry));
}

static ALWAYS_INLINE void xtensa_itlb_entry_write(uint32_t pte, uint32_t entry)
{
	__asm__ volatile("witlb %0, %1\n\t"
			: : "a" (pte), "a"(entry));
}

static ALWAYS_INLINE void xtensa_itlb_entry_write_sync(uint32_t pte, uint32_t entry)
{
	__asm__ volatile("witlb %0, %1\n\t"
			"isync\n\t"
			: : "a" (pte), "a"(entry));
}

/**
 * @brief Invalidate all ITLB entries.
 *
 * This should be used carefully since all entries in the instruction TLB
 * will be erased and the only way to find lookup a physical address will be
 * through the page tables.
 */
static inline void xtensa_itlb_invalidate_sync(void)
{
	uint8_t way, i;

	for (way = 0; way < Z_XTENSA_ITLB_WAYS; way++) {
		for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) {
			uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT);

			xtensa_itlb_entry_invalidate(entry);
		}
	}
	__asm__ volatile("isync");
}

/**
 * @brief Invalidate all DTLB entries.
 *
 * This should be used carefully since all entries in the data TLB will be
 * erased and the only way to find lookup a physical address will be through
 * the page tables.
 */
static inline void xtensa_dtlb_invalidate_sync(void)
{
	uint8_t way, i;

	for (way = 0; way < Z_XTENSA_DTLB_WAYS; way++) {
		for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) {
			uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT);

			xtensa_dtlb_entry_invalidate(entry);
		}
	}
	__asm__ volatile("isync");
}

/**
 * @brief Invalidates an autorefill DTLB entry.
 *
 * Invalidates the page table enrty that maps a given virtual address.
 */
static inline void xtensa_dtlb_autorefill_invalidate_sync(void *vaddr)
{
	uint8_t way;

	for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) {
		xtensa_dtlb_entry_invalidate(Z_XTENSA_TLB_ENTRY((uint32_t)vaddr, way));
	}
	__asm__ volatile("dsync");
}

/**
 * @brief Invalidates an autorefill ITLB entry.
 *
 * Invalidates the page table enrty that maps a given virtual address.
 */
static inline void xtensa_itlb_autorefill_invalidate_sync(void *vaddr)
{
	uint8_t way;

	for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) {
		xtensa_itlb_entry_invalidate(Z_XTENSA_TLB_ENTRY((uint32_t)vaddr, way));
	}
	__asm__ volatile("isync");
}
/**
 * @brief Invalidate all autorefill ITLB entries.
 *
 * This should be used carefully since all entries in the instruction TLB
 * will be erased and the only way to find lookup a physical address will be
 * through the page tables.
 */
static inline void xtensa_itlb_autorefill_invalidate_all_sync(void)
{
	uint8_t way, i;

	for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) {
		for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) {
			uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT);

			xtensa_itlb_entry_invalidate(entry);
		}
	}
	__asm__ volatile("isync");
}

/**
 * @brief Invalidate all autorefill DTLB entries.
 *
 * This should be used carefully since all entries in the data TLB will be
 * erased and the only way to find lookup a physical address will be through
 * the page tables.
 */
static inline void xtensa_dtlb_autorefill_invalidate_all_sync(void)
{
	uint8_t way, i;

	for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) {
		for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) {
			uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT);

			xtensa_dtlb_entry_invalidate(entry);
		}
	}
	__asm__ volatile("isync");
}


/**
 * @brief Set the page tables.
 *
 * The page tables is set writing ptevaddr address.
 *
 * @param ptables The page tables address (virtual address)
 */
static ALWAYS_INLINE void xtensa_ptevaddr_set(void *ptables)
{
	__asm__ volatile("wsr.ptevaddr %0" : : "a"((uint32_t)ptables));
}

/*
 * The following functions are helpful when debugging.
 */
static ALWAYS_INLINE void *xtensa_dtlb_vaddr_read(uint32_t entry)
{
	uint32_t vaddr;

	__asm__ volatile("rdtlb0  %0, %1\n\t" : "=a" (vaddr) : "a" (entry));
	return (void *)(vaddr & Z_XTENSA_PTE_VPN_MASK);
}

static ALWAYS_INLINE uint32_t xtensa_dtlb_paddr_read(uint32_t entry)
{
	uint32_t paddr;

	__asm__ volatile("rdtlb1  %0, %1\n\t" : "=a" (paddr) : "a" (entry));
	return (paddr & Z_XTENSA_PTE_PPN_MASK);
}

static ALWAYS_INLINE void *xtensa_itlb_vaddr_read(uint32_t entry)
{
	uint32_t vaddr;

	__asm__ volatile("ritlb0  %0, %1\n\t" : "=a" (vaddr), "+a" (entry));
	return (void *)(vaddr & Z_XTENSA_PTE_VPN_MASK);
}

static ALWAYS_INLINE uint32_t xtensa_itlb_paddr_read(uint32_t entry)
{
	uint32_t paddr;

	__asm__ volatile("ritlb1  %0, %1\n\t" : "=a" (paddr), "+a" (entry));
	return (paddr & Z_XTENSA_PTE_PPN_MASK);
}

static ALWAYS_INLINE uint32_t xtensa_itlb_probe(void *vaddr)
{
	uint32_t ret;

	__asm__ __volatile__("pitlb  %0, %1\n\t" : "=a" (ret) : "a" ((uint32_t)vaddr));
	return ret;
}

static ALWAYS_INLINE uint32_t xtensa_dtlb_probe(void *vaddr)
{
	uint32_t ret;

	__asm__ __volatile__("pdtlb  %0, %1\n\t" : "=a" (ret) : "a" ((uint32_t)vaddr));
	return ret;
}

static inline void xtensa_itlb_vaddr_invalidate(void *vaddr)
{
	uint32_t entry = xtensa_itlb_probe(vaddr);

	if (entry & Z_XTENSA_PITLB_HIT) {
		xtensa_itlb_entry_invalidate_sync(entry);
	}
}

static inline void xtensa_dtlb_vaddr_invalidate(void *vaddr)
{
	uint32_t entry = xtensa_dtlb_probe(vaddr);

	if (entry & Z_XTENSA_PDTLB_HIT) {
		xtensa_dtlb_entry_invalidate_sync(entry);
	}
}

#endif /* ZEPHYR_ARCH_XTENSA_XTENSA_MMU_PRIV_H_ */