arch/arm/core/cortex_a_r/swap_helper.S

/*
 * Copyright (c) 2013-2014 Wind River Systems, Inc.
 * Copyright (c) 2017-2019 Nordic Semiconductor ASA.
 * Copyright (c) 2020 Stephanos Ioannidis <root@stephanos.io>
 *
 * SPDX-License-Identifier: Apache-2.0
 */

/**
 * @file
 * @brief Thread context switching for ARM Cortex-A and Cortex-R
 *
 * This module implements the routines necessary for thread context switching
 * on ARM Cortex-A and Cortex-R CPUs.
 */

#include <zephyr/toolchain.h>
#include <zephyr/linker/sections.h>
#include <offsets_short.h>
#include <zephyr/arch/cpu.h>
#include <zephyr/syscall.h>
#include <zephyr/kernel.h>
#include "macro_priv.inc"

_ASM_FILE_PROLOGUE

GTEXT(z_arm_svc)
GTEXT(z_arm_do_swap)
GTEXT(z_do_kernel_oops)
#if defined(CONFIG_USERSPACE)
GTEXT(z_arm_do_syscall)
#endif

GDATA(_kernel)

/**
 *
 * @brief Routine to handle context switches
 *
 * For Cortex-R, this function is directly called either by z_arm_{exc,int}_exit
 * in case of preemption, or z_arm_svc in case of cooperative switching.
 */

SECTION_FUNC(TEXT, z_arm_do_swap)

#ifdef CONFIG_INSTRUMENT_THREAD_SWITCHING
    /* Register the context switch */
    push {r0, lr}
    bl z_thread_mark_switched_out
    pop {r0, lr}
#endif /* CONFIG_INSTRUMENT_THREAD_SWITCHING */

    /* load current _cpu into r1 and current k_thread into r2 */
    get_cpu r1
    ldr r2, [r1, #___cpu_t_current_OFFSET]

#if defined(CONFIG_ARM_STORE_EXC_RETURN)
    /* Store LSB of LR (EXC_RETURN) to the thread's 'mode' word. */
    strb lr, [r2, #_thread_offset_to_mode_exc_return]
#endif

    /* addr of callee-saved regs in thread in r0 */
    ldr r0, =_thread_offset_to_callee_saved
    add r0, r2

    /* Store rest of process context */
    cps #MODE_SYS
    stm r0, {r4-r11, sp}
    cps #MODE_SVC

#if defined(CONFIG_FPU_SHARING)
    ldrb r0, [r2, #_thread_offset_to_user_options]
    tst r0, #K_FP_REGS /* _current->base.user_options & K_FP_REGS */
    beq out_fp_inactive

    mov ip, #FPEXC_EN
    vmsr fpexc, ip

    /*
     * If the float context pointer is not null, then the VFP has not been
     * used since this thread has used it.  Consequently, the caller-saved
     * float registers have not been saved away, so write them to the
     * exception stack frame.
     */
    ldr r0, [r1, #___cpu_t_fp_ctx_OFFSET]
    cmp r0, #0
    beq out_store_thread_context

    vstmia r0!, {s0-s15}
#ifdef CONFIG_VFP_FEATURE_REGS_S64_D32
    vstmia r0!, {d16-d31}
#endif
    vmrs r3, fpscr
    stm r0, {r3, ip}

out_store_thread_context:
    /* Store s16-s31 to thread context */
    add r0, r2, #_thread_offset_to_preempt_float
    vstmia r0, {s16-s31}

    mov ip, #0
    vmsr fpexc, ip

out_fp_inactive:
    /*
     * The floating context has now been saved to the exception stack
     * frame, so zero out the global pointer to note this.
     */
    mov r0, #0
    str r0, [r1, #___cpu_t_fp_ctx_OFFSET]
#endif /* CONFIG_FPU_SHARING */

    /* fetch the thread to run from the ready queue cache */
    ldr r3, =_kernel
    ldr r2, [r3, #_kernel_offset_to_ready_q_cache]

    str r2, [r1, #___cpu_t_current_OFFSET]

#if defined(CONFIG_THREAD_LOCAL_STORAGE)
    /* Grab the TLS pointer */
    ldr r4, =_thread_offset_to_tls
    adds r4, r2, r4
    ldr r0, [r4]

    /* Store TLS pointer in the "Process ID" register.
     * TPIDRURW is used as a base pointer to all
     * thread variables with offsets added by toolchain.
     */
    mcr 15, 0, r0, c13, c0, 2
#endif

#if defined(CONFIG_ARM_STORE_EXC_RETURN)
    /* Restore EXC_RETURN value. */
    ldrsb lr, [r2, #_thread_offset_to_mode_exc_return]
#endif

    /* Restore previous interrupt disable state (irq_lock key)
     * (We clear the arch.basepri field after restoring state)
     */
    ldr r0, [r2, #_thread_offset_to_basepri]
    movs r3, #0
    str r3, [r2, #_thread_offset_to_basepri]

    /* addr of callee-saved regs in thread in r0 */
    ldr r0, =_thread_offset_to_callee_saved
    add r0, r2

    /* restore r4-r11 and sp for incoming thread */
    cps #MODE_SYS
    ldm r0, {r4-r11, sp}
    cps #MODE_SVC

#if defined(CONFIG_FPU_SHARING)
    ldrb r0, [r2, #_thread_offset_to_user_options]
    tst r0, #K_FP_REGS /* _current->base.user_options & K_FP_REGS */
    beq in_fp_inactive

    mov r3, #FPEXC_EN
    vmsr fpexc, r3

    /* Restore s16-s31 from thread context */
    add r0, r2, #_thread_offset_to_preempt_float
    vldmia r0, {s16-s31}

    mov r3, #0
    vmsr fpexc, r3

in_fp_inactive:
#endif /* CONFIG_FPU_SHARING */

#if defined (CONFIG_ARM_MPU)
    /* r2 contains k_thread */
    mov r0, r2
    /* Re-program dynamic memory map */
    push {r0, lr}
    bl z_arm_configure_dynamic_mpu_regions
    pop {r0, lr}
#endif

#ifdef CONFIG_INSTRUMENT_THREAD_SWITCHING
    /* Register the context switch */
    push {r0, lr}
    bl z_thread_mark_switched_in
    pop {r0, lr}
#endif /* CONFIG_INSTRUMENT_THREAD_SWITCHING */

    /*
     * Cortex-R: return to the caller (z_arm_{exc,int}_exit, or z_arm_svc)
     */
    bx lr

#if defined(CONFIG_FPU_SHARING)
#define FPU_SF_SIZE ___fpu_t_SIZEOF
#else
#define FPU_SF_SIZE 0
#endif

/**
 *
 * @brief Service call handler
 *
 * The service call (svc) is used in the following occasions:
 * - Cooperative context switching
 * - IRQ offloading
 * - Kernel run-time exceptions
 *
 */
SECTION_FUNC(TEXT, z_arm_svc)
#if defined(CONFIG_USERSPACE)
    /* Determine if incoming thread was in user context */
    push {r0}
    mrs r0, spsr
    and r0, #MODE_MASK
    cmp r0, #MODE_USR
    bne svc_system_thread

    get_cpu r0
    ldr r0, [r0, #___cpu_t_current_OFFSET]

    /* Save away user stack pointer */
    cps #MODE_SYS
    str sp, [r0, #_thread_offset_to_sp_usr] /* sp_usr */

    /* Switch to privileged stack */
    ldr sp, [r0, #_thread_offset_to_priv_stack_end] /* priv stack end */
    cps #MODE_SVC

svc_system_thread:
    pop {r0}
#endif

    /*
     * Switch to system mode to store r0-r3 to the process stack pointer.
     * Save r12 and the lr as we could be swapping in another process and
     * returning to a different location.
     */
    srsdb #MODE_SYS!
    cps #MODE_SYS
    push {r0-r3, r12, lr}

#if defined(CONFIG_FPU_SHARING)
    sub sp, sp, #___fpu_t_SIZEOF

    /*
     * Note that this handler was entered with the VFP unit enabled.
     * The undefined instruction handler uses this to know that it
     * needs to save the current floating context.
     */
    vmrs r0, fpexc
    str r0, [sp, #___fpu_t_SIZEOF - 4]
    tst r0, #FPEXC_EN
    beq _vfp_not_enabled
    vmrs r0, fpscr
    str r0, [sp, #___fpu_t_SIZEOF - 8]

    /* Disable VFP */
    mov r0, #0
    vmsr fpexc, r0

_vfp_not_enabled:
    /*
     * Mark where to store the floating context for the undefined
     * instruction handler
     */
    get_cpu r2
    ldr r0, [r2, #___cpu_t_fp_ctx_OFFSET]
    cmp r0, #0
    streq sp, [r2, #___cpu_t_fp_ctx_OFFSET]
#endif /* CONFIG_FPU_SHARING */

    mov ip, sp

    cps #MODE_SVC

    /*
     * Store lr_svc to the SVC mode stack. This value will be restored prior to
     * exiting the SVC call in z_arm_int_exit.
     */
    push {lr}

    /* Align stack at double-word boundary */
    /* TODO: Question, why push {r2, r3} here */
    and r3, sp, #4
    sub sp, sp, r3
    push {r2, r3}

    /* Increment interrupt nesting count */
    get_cpu r2
    ldr r0, [r2, #___cpu_t_nested_OFFSET]
    add r0, r0, #1
    str r0, [r2, #___cpu_t_nested_OFFSET]

    /* Get SVC number */
    mrs r0, spsr
    tst r0, #0x20

    ldreq r1, [lr, #-4]
    biceq r1, #0xff000000
    beq demux

    ldr r1, [lr, #-2]
    and r1, #0xff

   /*
    * grab service call number:
    * 0: context switch
    * 1: irq_offload (if configured)
    * 2: kernel panic or oops (software generated fatal exception)
    * 3: system calls for memory protection
    */
demux:

#if defined(CONFIG_USERSPACE)
    cmp r1, #_SVC_CALL_SYSTEM_CALL
    beq _do_syscall
#endif

    cmp r1, #_SVC_CALL_CONTEXT_SWITCH
    beq _context_switch

    cmp r1, #_SVC_CALL_RUNTIME_EXCEPT
    beq _oops

#if CONFIG_IRQ_OFFLOAD
    blx z_irq_do_offload  /* call C routine which executes the offload */

    /* exception return is done in z_arm_int_exit() */
    b z_arm_int_exit
#endif

_context_switch:
    /* handler mode exit, to PendSV */
    bl z_arm_do_swap

    b z_arm_int_exit

_oops:
    /*
     * Pass the exception frame to z_do_kernel_oops.  r0 contains the
     * exception reason.
     */
    cps #MODE_SYS
    mov r0, sp
    cps #MODE_SVC
    bl z_do_kernel_oops
    b z_arm_int_exit

#if defined(CONFIG_USERSPACE)
    /*
     * System call will setup a jump to the _do_arm_syscall function
     * running in system mode when returning from the exception.
     *
     * There is some trickery involved here because we have to preserve
     * the original PC value so that we can return back to the caller of
     * the SVC.
     *
     * On SVC exception, the USER/SYSTEM stack looks like the following:
     * { possible FPU space } - r0 - r1 - r2 - r3 - r12 - LR - PC - SPSR
     *
     * Registers look like:
     * r0 - arg1
     * r1 - arg2
     * r2 - arg3
     * r3 - arg4
     * r4 - arg5
     * r5 - arg6
     * r6 - call_id
     * r8 - saved link register
     */
_do_syscall:
    /* grab address of LR from stack frame */
    ldr r8, [ip, #(FPU_SF_SIZE + ___basic_sf_t_pc_OFFSET)]

    /* Make the exception return to system state */
    ldr r1, [ip, #(FPU_SF_SIZE + ___basic_sf_t_xpsr_OFFSET)]

    /* If leaving thumb mode, set the return address to thumb mode */
    tst r1, #T_BIT
    orrne r8, #1

    bic r1, #(MODE_MASK | T_BIT)
    orr r1, r1, #MODE_SYS
    str r1, [ip, #(FPU_SF_SIZE + ___basic_sf_t_xpsr_OFFSET)]

    /*
     * Store the address of z_arm_do_syscall for the exit so the exception
     * return goes there in system state.
     */
    ldr r1, =z_arm_do_syscall
    str r1, [ip, #(FPU_SF_SIZE + ___basic_sf_t_pc_OFFSET)]

    /* validate syscall limit, only set priv mode if valid */
    ldr ip, =K_SYSCALL_LIMIT
    cmp r6, ip
    blo valid_syscall_id

    /* bad syscall id.  Set arg0 to bad id and set call_id to SYSCALL_BAD */
    cps #MODE_SYS
    str r6, [sp]
    cps #MODE_SVC
    ldr r6, =K_SYSCALL_BAD

valid_syscall_id:
    get_cpu r0
    ldr r0, [r0, #___cpu_t_current_OFFSET]
    ldr r1, [r0, #_thread_offset_to_mode]
    bic r1, #1
    /* Store (privileged) mode in thread's mode state variable */
    str r1, [r0, #_thread_offset_to_mode]
    dsb

    /* ISB is not strictly necessary here (stack pointer is not being
     * touched), but it's recommended to avoid executing pre-fetched
     * instructions with the previous privilege.
     */
    isb

    /* Return to _arm_do_syscall in system state. */
    b z_arm_int_exit
#endif

GTEXT(z_arm_cortex_r_svc)
SECTION_FUNC(TEXT, z_arm_cortex_r_svc)
    svc #_SVC_CALL_CONTEXT_SWITCH
    bx lr