/*
 * Copyright (c) 2016 Jean-Paul Etienne <fractalclone@gmail.com>
 * Copyright (c) 2018 Foundries.io Ltd
 * Copyright (c) 2020 BayLibre, SAS
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#include <zephyr/toolchain.h>
#include <zephyr/linker/sections.h>
#include <offsets_short.h>
#include <zephyr/arch/cpu.h>
#include <zephyr/sys/util.h>
#include <zephyr/kernel.h>
#include <zephyr/syscall.h>
#include <zephyr/arch/riscv/csr.h>
#include <zephyr/arch/riscv/syscall.h>
#include "asm_macros.inc"

#ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING
#include <soc_isr_stacking.h>
#endif

/* Convenience macro for loading/storing register states. */
#define DO_CALLER_SAVED(op) \
	RV_E(	op t0, __z_arch_esf_t_t0_OFFSET(sp)	);\
	RV_E(	op t1, __z_arch_esf_t_t1_OFFSET(sp)	);\
	RV_E(	op t2, __z_arch_esf_t_t2_OFFSET(sp)	);\
	RV_I(	op t3, __z_arch_esf_t_t3_OFFSET(sp)	);\
	RV_I(	op t4, __z_arch_esf_t_t4_OFFSET(sp)	);\
	RV_I(	op t5, __z_arch_esf_t_t5_OFFSET(sp)	);\
	RV_I(	op t6, __z_arch_esf_t_t6_OFFSET(sp)	);\
	RV_E(	op a0, __z_arch_esf_t_a0_OFFSET(sp)	);\
	RV_E(	op a1, __z_arch_esf_t_a1_OFFSET(sp)	);\
	RV_E(	op a2, __z_arch_esf_t_a2_OFFSET(sp)	);\
	RV_E(	op a3, __z_arch_esf_t_a3_OFFSET(sp)	);\
	RV_E(	op a4, __z_arch_esf_t_a4_OFFSET(sp)	);\
	RV_E(	op a5, __z_arch_esf_t_a5_OFFSET(sp)	);\
	RV_I(	op a6, __z_arch_esf_t_a6_OFFSET(sp)	);\
	RV_I(	op a7, __z_arch_esf_t_a7_OFFSET(sp)	);\
	RV_E(	op ra, __z_arch_esf_t_ra_OFFSET(sp)	)

	.macro get_current_cpu dst
#if defined(CONFIG_SMP) || defined(CONFIG_USERSPACE)
	csrr \dst, mscratch
#else
	la \dst, _kernel + ___kernel_t_cpus_OFFSET
#endif
	.endm

/* imports */
GDATA(_sw_isr_table)
GTEXT(__soc_is_irq)
GTEXT(__soc_handle_irq)
GTEXT(_Fault)
#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE
GTEXT(__soc_save_context)
GTEXT(__soc_restore_context)
#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */

GTEXT(z_riscv_fatal_error)
GTEXT(z_get_next_switch_handle)
GTEXT(z_riscv_switch)
GTEXT(z_riscv_thread_start)

#ifdef CONFIG_TRACING
GTEXT(sys_trace_isr_enter)
GTEXT(sys_trace_isr_exit)
#endif

#ifdef CONFIG_USERSPACE
GDATA(_k_syscall_table)
#endif

/* exports */
GTEXT(_isr_wrapper)

/* use ABI name of registers for the sake of simplicity */

/*
 * Generic architecture-level IRQ handling, along with callouts to
 * SoC-specific routines.
 *
 * Architecture level IRQ handling includes basic context save/restore
 * of standard registers and calling ISRs registered at Zephyr's driver
 * level.
 *
 * Since RISC-V does not completely prescribe IRQ handling behavior,
 * implementations vary (some implementations also deviate from
 * what standard behavior is defined). Hence, the arch level code expects
 * the following functions to be provided at the SOC level:
 *
 *     - __soc_is_irq: decide if we're handling an interrupt or an exception
 *     - __soc_handle_irq: handle SoC-specific details for a pending IRQ
 *       (e.g. clear a pending bit in a SoC-specific register)
 *
 * If CONFIG_RISCV_SOC_CONTEXT_SAVE=y, calls to SoC-level context save/restore
 * routines are also made here. For details, see the Kconfig help text.
 */

/*
 * Handler called upon each exception/interrupt/fault
 */
SECTION_FUNC(exception.entry, _isr_wrapper)

#ifdef CONFIG_USERSPACE
	/* retrieve address of _current_cpu preserving s0 */
	csrrw s0, mscratch, s0

	/* preserve t0 and t1 temporarily */
	sr t0, _curr_cpu_arch_user_exc_tmp0(s0)
	sr t1, _curr_cpu_arch_user_exc_tmp1(s0)

	/* determine if we come from user space */
	csrr t0, mstatus
	li t1, MSTATUS_MPP
	and t0, t0, t1
	bnez t0, 1f

	/* in user space we were: switch to our privileged stack */
	mv t0, sp
	lr sp, _curr_cpu_arch_user_exc_sp(s0)

	/* Save user stack value. Coming from user space, we know this
	 * can't overflow the privileged stack. The esf will be allocated
	 * later but it is safe to store our saved user sp here. */
	sr t0, (-__z_arch_esf_t_SIZEOF + __z_arch_esf_t_sp_OFFSET)(sp)

	/* Make sure tls pointer is sane */
	lr t0, ___cpu_t_current_OFFSET(s0)
	lr tp, _thread_offset_to_tls(t0)

	/* Clear our per-thread usermode flag */
	lui t0, %tprel_hi(is_user_mode)
	add t0, t0, tp, %tprel_add(is_user_mode)
	sb zero, %tprel_lo(is_user_mode)(t0)
1:
	/* retrieve original t0/t1 values */
	lr t0, _curr_cpu_arch_user_exc_tmp0(s0)
	lr t1, _curr_cpu_arch_user_exc_tmp1(s0)

	/* retrieve original s0 and restore _current_cpu in mscratch */
	csrrw s0, mscratch, s0
#endif

#ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING
	SOC_ISR_SW_STACKING
#else
	/* Save caller-saved registers on current thread stack. */
	addi sp, sp, -__z_arch_esf_t_SIZEOF
	DO_CALLER_SAVED(sr)		;
#endif /* CONFIG_RISCV_SOC_HAS_ISR_STACKING */

	/* Save s0 in the esf and load it with &_current_cpu. */
	sr s0, __z_arch_esf_t_s0_OFFSET(sp)
	get_current_cpu s0

	/* Save MEPC register */
	csrr t0, mepc
	sr t0, __z_arch_esf_t_mepc_OFFSET(sp)

	/* Save MSTATUS register */
	csrr t2, mstatus
	sr t2, __z_arch_esf_t_mstatus_OFFSET(sp)

#if defined(CONFIG_FPU_SHARING)
	/* determine if this is an Illegal Instruction exception */
	csrr t0, mcause
	li t1, 2		/* 2 = illegal instruction */
	bne t0, t1, no_fp
	/* determine if FPU access was disabled */
	csrr t0, mstatus
	li t1, MSTATUS_FS
	and t0, t0, t1
	bnez t0, no_fp
	/* determine if we trapped on an FP instruction. */
	csrr t2, mtval		/* get faulting instruction */
	andi t0, t2, 0x7f	/* keep only the opcode bits */
	xori t1, t0, 0b1010011	/* OP-FP */
	beqz t1, is_fp
	ori  t0, t0, 0b0100000
	xori t1, t0, 0b0100111	/* LOAD-FP / STORE-FP */
#if !defined(CONFIG_RISCV_ISA_EXT_C)
	bnez t1, no_fp
#else
	beqz t1, is_fp
	/* remaining non RVC (0b11) and RVC with 0b01 are not FP instructions */
	andi t1, t0, 1
	bnez t1, no_fp
	/*
	 * 001...........00 = C.FLD    RV32/64  (RV128 = C.LQ)
	 * 001...........10 = C.FLDSP  RV32/64  (RV128 = C.LQSP)
	 * 011...........00 = C.FLW    RV32     (RV64/128 = C.LD)
	 * 011...........10 = C.FLWSPP RV32     (RV64/128 = C.LDSP)
	 * 101...........00 = C.FSD    RV32/64  (RV128 = C.SQ)
	 * 101...........10 = C.FSDSP  RV32/64  (RV128 = C.SQSP)
	 * 111...........00 = C.FSW    RV32     (RV64/128 = C.SD)
	 * 111...........10 = C.FSWSP  RV32     (RV64/128 = C.SDSP)
	 *
	 * so must be .01............. on RV64 and ..1............. on RV32.
	 */
	srli t0, t2, 8
#if defined(CONFIG_64BIT)
	andi t1, t0, 0b01100000
	xori t1, t1, 0b00100000
	bnez t1, no_fp
#else
	andi t1, t0, 0b00100000
	beqz t1, no_fp
#endif
#endif /* CONFIG_RISCV_ISA_EXT_C */

is_fp:	/* Process the FP trap and quickly return from exception */
	la ra, fp_trap_exit
	mv a0, sp
	tail z_riscv_fpu_trap

no_fp:	/* increment _current->arch.exception_depth */
	lr t0, ___cpu_t_current_OFFSET(s0)
	lb t1, _thread_offset_to_exception_depth(t0)
	add t1, t1, 1
	sb t1, _thread_offset_to_exception_depth(t0)

	/* configure the FPU for exception mode */
	call z_riscv_fpu_enter_exc
#endif

#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE
	/* Handle context saving at SOC level. */
	addi a0, sp, __z_arch_esf_t_soc_context_OFFSET
	jal ra, __soc_save_context
#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */

	/*
	 * Check if exception is the result of an interrupt or not.
	 * (SOC dependent). Following the RISC-V architecture spec, the MSB
	 * of the mcause register is used to indicate whether an exception
	 * is the result of an interrupt or an exception/fault. But for some
	 * SOCs (like pulpino or riscv-qemu), the MSB is never set to indicate
	 * interrupt. Hence, check for interrupt/exception via the __soc_is_irq
	 * function (that needs to be implemented by each SOC). The result is
	 * returned via register a0 (1: interrupt, 0 exception)
	 */
	jal ra, __soc_is_irq

	/* If a0 != 0, jump to is_interrupt */
	bnez a0, is_interrupt

	/*
	 * If the exception is the result of an ECALL, check whether to
	 * perform a context-switch or an IRQ offload. Otherwise call _Fault
	 * to report the exception.
	 */
	csrr t0, mcause
	li t2, SOC_MCAUSE_EXP_MASK
	and t0, t0, t2

	/*
	 * If mcause == SOC_MCAUSE_ECALL_EXP, handle system call from
	 * kernel thread.
	 */
	li t1, SOC_MCAUSE_ECALL_EXP
	beq t0, t1, is_kernel_syscall

#ifdef CONFIG_USERSPACE
	/*
	 * If mcause == SOC_MCAUSE_USER_ECALL_EXP, handle system call
	 * for user mode thread.
	 */
	li t1, SOC_MCAUSE_USER_ECALL_EXP
	beq t0, t1, is_user_syscall
#endif /* CONFIG_USERSPACE */

	/*
	 * Call _Fault to handle exception.
	 * Stack pointer is pointing to a z_arch_esf_t structure, pass it
	 * to _Fault (via register a0).
	 * If _Fault shall return, set return address to
	 * no_reschedule to restore stack.
	 */
	mv a0, sp
	la ra, no_reschedule
	tail _Fault

is_kernel_syscall:
	/*
	 * A syscall is the result of an ecall instruction, in which case the
	 * MEPC will contain the address of the ecall instruction.
	 * Increment saved MEPC by 4 to prevent triggering the same ecall
	 * again upon exiting the ISR.
	 *
	 * It's safe to always increment by 4, even with compressed
	 * instructions, because the ecall instruction is always 4 bytes.
	 */
	lr t0, __z_arch_esf_t_mepc_OFFSET(sp)
	addi t0, t0, 4
	sr t0, __z_arch_esf_t_mepc_OFFSET(sp)

#ifdef CONFIG_PMP_STACK_GUARD
	/* Re-activate PMP for m-mode */
	li t1, MSTATUS_MPP
	csrc mstatus, t1
	li t1, MSTATUS_MPRV
	csrs mstatus, t1
#endif

	/* Determine what to do. Operation code is in t0. */
	lr t0, __z_arch_esf_t_t0_OFFSET(sp)

	.if RV_ECALL_RUNTIME_EXCEPT != 0; .err; .endif
	beqz t0, do_fault

#if defined(CONFIG_IRQ_OFFLOAD)
	li t1, RV_ECALL_IRQ_OFFLOAD
	beq t0, t1, do_irq_offload
#endif

#ifdef CONFIG_RISCV_ALWAYS_SWITCH_THROUGH_ECALL
	li t1, RV_ECALL_SCHEDULE
	bne t0, t1, skip_schedule
	lr a0, __z_arch_esf_t_a0_OFFSET(sp)
	lr a1, __z_arch_esf_t_a1_OFFSET(sp)
	j reschedule
skip_schedule:
#endif

	/* default fault code is K_ERR_KERNEL_OOPS */
	li a0, 3
	j 1f

do_fault:
	/* Handle RV_ECALL_RUNTIME_EXCEPT. Retrieve reason in a0, esf in A1. */
	lr a0, __z_arch_esf_t_a0_OFFSET(sp)
1:	mv a1, sp
	tail z_riscv_fatal_error

#if defined(CONFIG_IRQ_OFFLOAD)
do_irq_offload:
	/*
	 * Retrieve provided routine and argument from the stack.
	 * Routine pointer is in saved a0, argument in saved a1
	 * so we load them with a1/a0 (reversed).
	 */
	lr a1, __z_arch_esf_t_a0_OFFSET(sp)
	lr a0, __z_arch_esf_t_a1_OFFSET(sp)

	/* Increment _current_cpu->nested */
	lw t1, ___cpu_t_nested_OFFSET(s0)
	addi t2, t1, 1
	sw t2, ___cpu_t_nested_OFFSET(s0)
	bnez t1, 1f

	/* Switch to interrupt stack */
	mv t0, sp
	lr sp, ___cpu_t_irq_stack_OFFSET(s0)

	/* Save thread stack pointer on interrupt stack */
	addi sp, sp, -16
	sr t0, 0(sp)
1:
	/* Execute provided routine (argument is in a0 already). */
	jalr ra, a1, 0

	/* Leave through the regular IRQ exit path */
	j irq_done
#endif /* CONFIG_IRQ_OFFLOAD */

#ifdef CONFIG_USERSPACE
is_user_syscall:

#ifdef CONFIG_PMP_STACK_GUARD
	/*
	 * We came from userspace and need to reconfigure the
	 * PMP for kernel mode stack guard.
	 */
	lr a0, ___cpu_t_current_OFFSET(s0)
	call z_riscv_pmp_stackguard_enable
#endif

	/* It is safe to re-enable IRQs now */
	csrs mstatus, MSTATUS_IEN

	/*
	 * Same as for is_kernel_syscall: increment saved MEPC by 4 to
	 * prevent triggering the same ecall again upon exiting the ISR.
	 */
	lr t1, __z_arch_esf_t_mepc_OFFSET(sp)
	addi t1, t1, 4
	sr t1, __z_arch_esf_t_mepc_OFFSET(sp)

	/* Restore argument registers from user stack */
	lr a0, __z_arch_esf_t_a0_OFFSET(sp)
	lr a1, __z_arch_esf_t_a1_OFFSET(sp)
	lr a2, __z_arch_esf_t_a2_OFFSET(sp)
	lr a3, __z_arch_esf_t_a3_OFFSET(sp)
	lr a4, __z_arch_esf_t_a4_OFFSET(sp)
	lr a5, __z_arch_esf_t_a5_OFFSET(sp)
	lr t0, __z_arch_esf_t_t0_OFFSET(sp)
#if defined(CONFIG_RISCV_ISA_RV32E)
	/* Stack alignment for RV32E is 4 bytes */
	addi sp, sp, -4
	mv t1, sp
	sw t1, 0(sp)
#else
	mv a6, sp
#endif /* CONFIG_RISCV_ISA_RV32E */

	/* validate syscall limit */
	li t1, K_SYSCALL_LIMIT
	bltu t0, t1, valid_syscall_id

	/* bad syscall id.  Set arg1 to bad id and set call_id to SYSCALL_BAD */
	mv a0, t0
	li t0, K_SYSCALL_BAD

valid_syscall_id:

	la t2, _k_syscall_table

	slli t1, t0, RV_REGSHIFT	# Determine offset from indice value
	add t2, t2, t1			# Table addr + offset = function addr
	lr t2, 0(t2)			# Load function address

	/* Execute syscall function */
	jalr ra, t2, 0

#if defined(CONFIG_RISCV_ISA_RV32E)
	addi sp, sp, 4
#endif /* CONFIG_RISCV_ISA_RV32E */

	/* Update a0 (return value) on the stack */
	sr a0, __z_arch_esf_t_a0_OFFSET(sp)

	/* Disable IRQs again before leaving */
	csrc mstatus, MSTATUS_IEN
	j might_have_rescheduled
#endif /* CONFIG_USERSPACE */

is_interrupt:

#ifdef CONFIG_PMP_STACK_GUARD
#ifdef CONFIG_USERSPACE
	/*
	 * If we came from userspace then we need to reconfigure the
	 * PMP for kernel mode stack guard.
	 */
	lr t0, __z_arch_esf_t_mstatus_OFFSET(sp)
	li t1, MSTATUS_MPP
	and t0, t0, t1
	bnez t0, 1f
	lr a0, ___cpu_t_current_OFFSET(s0)
	call z_riscv_pmp_stackguard_enable
	j 2f
#endif /* CONFIG_USERSPACE */
1:	/* Re-activate PMP for m-mode */
	li t1, MSTATUS_MPP
	csrc mstatus, t1
	li t1, MSTATUS_MPRV
	csrs mstatus, t1
2:
#endif

	/* Increment _current_cpu->nested */
	lw t1, ___cpu_t_nested_OFFSET(s0)
	addi t2, t1, 1
	sw t2, ___cpu_t_nested_OFFSET(s0)
	bnez t1, on_irq_stack

	/* Switch to interrupt stack */
	mv t0, sp
	lr sp, ___cpu_t_irq_stack_OFFSET(s0)

	/*
	 * Save thread stack pointer on interrupt stack
	 * In RISC-V, stack pointer needs to be 16-byte aligned
	 */
	addi sp, sp, -16
	sr t0, 0(sp)

on_irq_stack:

#ifdef CONFIG_TRACING_ISR
	call sys_trace_isr_enter
#endif

	/* Get IRQ causing interrupt */
	csrr a0, mcause
	li t0, SOC_MCAUSE_EXP_MASK
	and a0, a0, t0

	/*
	 * Clear pending IRQ generating the interrupt at SOC level
	 * Pass IRQ number to __soc_handle_irq via register a0
	 */
	jal ra, __soc_handle_irq

	/*
	 * Call corresponding registered function in _sw_isr_table.
	 * (table is 2-word wide, we should shift index accordingly)
	 */
	la t0, _sw_isr_table
	slli a0, a0, (RV_REGSHIFT + 1)
	add t0, t0, a0

	/* Load argument in a0 register */
	lr a0, 0(t0)

	/* Load ISR function address in register t1 */
	lr t1, RV_REGSIZE(t0)

	/* Call ISR function */
	jalr ra, t1, 0

#ifdef CONFIG_TRACING_ISR
    call sys_trace_isr_exit
#endif

irq_done:
	/* Decrement _current_cpu->nested */
	lw t2, ___cpu_t_nested_OFFSET(s0)
	addi t2, t2, -1
	sw t2, ___cpu_t_nested_OFFSET(s0)
	bnez t2, no_reschedule

	/* nested count is back to 0: Return to thread stack */
	lr sp, 0(sp)

#ifdef CONFIG_STACK_SENTINEL
	call z_check_stack_sentinel
#endif

check_reschedule:

	/* Get pointer to current thread on this CPU */
	lr a1, ___cpu_t_current_OFFSET(s0)

	/*
	 * Get next thread to schedule with z_get_next_switch_handle().
	 * We pass it a NULL as we didn't save the whole thread context yet.
	 * If no scheduling is necessary then NULL will be returned.
	 */
	addi sp, sp, -16
	sr a1, 0(sp)
	mv a0, zero
	call z_get_next_switch_handle
	lr a1, 0(sp)
	addi sp, sp, 16
	beqz a0, no_reschedule

reschedule:

	/*
	 * Perform context switch:
	 * a0 = new thread
	 * a1 = old thread
	 */
	call z_riscv_switch

z_riscv_thread_start:
might_have_rescheduled:
	/* reload s0 with &_current_cpu as it might have changed or be unset */
	get_current_cpu s0

no_reschedule:

#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE
	/* Restore context at SOC level */
	addi a0, sp, __z_arch_esf_t_soc_context_OFFSET
	jal ra, __soc_restore_context
#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */

#if defined(CONFIG_FPU_SHARING)
	/* FPU handling upon exception mode exit */
	mv a0, sp
	call z_riscv_fpu_exit_exc

	/* decrement _current->arch.exception_depth */
	lr t0, ___cpu_t_current_OFFSET(s0)
	lb t1, _thread_offset_to_exception_depth(t0)
	add t1, t1, -1
	sb t1, _thread_offset_to_exception_depth(t0)
fp_trap_exit:
#endif

	/* Restore MEPC and MSTATUS registers */
	lr t0, __z_arch_esf_t_mepc_OFFSET(sp)
	lr t2, __z_arch_esf_t_mstatus_OFFSET(sp)
	csrw mepc, t0
	csrw mstatus, t2

#ifdef CONFIG_USERSPACE
	/*
	 * Check if we are returning to user mode. If so then we must
	 * set is_user_mode to true and preserve our kernel mode stack for
	 * the next exception to come.
	 */
	li t1, MSTATUS_MPP
	and t0, t2, t1
	bnez t0, 1f

#ifdef CONFIG_PMP_STACK_GUARD
	/* Remove kernel stack guard and Reconfigure PMP for user mode */
	lr a0, ___cpu_t_current_OFFSET(s0)
	call z_riscv_pmp_usermode_enable
#endif

	/* Set our per-thread usermode flag */
	li t1, 1
	lui t0, %tprel_hi(is_user_mode)
	add t0, t0, tp, %tprel_add(is_user_mode)
	sb t1, %tprel_lo(is_user_mode)(t0)

	/* preserve stack pointer for next exception entry */
	add t0, sp, __z_arch_esf_t_SIZEOF
	sr t0, _curr_cpu_arch_user_exc_sp(s0)

	j 2f
1:
	/*
	 * We are returning to kernel mode. Store the stack pointer to
	 * be re-loaded further down.
	 */
	addi t0, sp, __z_arch_esf_t_SIZEOF
	sr t0, __z_arch_esf_t_sp_OFFSET(sp)
2:
#endif

	/* Restore s0 (it is no longer ours) */
	lr s0, __z_arch_esf_t_s0_OFFSET(sp)

#ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING
	SOC_ISR_SW_UNSTACKING
#else
	/* Restore caller-saved registers from thread stack */
	DO_CALLER_SAVED(lr)
#endif /* CONFIG_RISCV_SOC_HAS_ISR_STACKING */

#ifdef CONFIG_USERSPACE
	/* retrieve saved stack pointer */
	lr sp, __z_arch_esf_t_sp_OFFSET(sp)
#else
	/* remove esf from the stack */
	addi sp, sp, __z_arch_esf_t_SIZEOF
#endif

	mret