| /* |
| * Copyright (c) 2017 Intel Corporation |
| * |
| * SPDX-License-Identifier: Apache-2.0 |
| */ |
| |
| #include <zephyr/toolchain.h> |
| #include <zephyr/arch/cpu.h> |
| #include <offsets_short.h> |
| #include <zephyr/syscall.h> |
| #include <zephyr/sys/mem_manage.h> |
| |
| #ifdef CONFIG_X86_KPTI |
| /* Copy interrupt return stack context to the trampoline stack, switch back |
| * to the user page table, and only then 'iret'. We jump to this instead |
| * of calling 'iret' if KPTI is turned on. This must be invoked with interrupts |
| * locked. |
| * |
| * Stack layout is expected to be what 'iretq' expects, which is as follows: |
| * |
| * 32 SS |
| * 24 RSP |
| * 16 RFLAGS |
| * 8 CS |
| * 0 RIP |
| */ |
| .global z_x86_trampoline_to_user |
| z_x86_trampoline_to_user: |
| /* Stash EDI, need a free register */ |
| pushq %rdi |
| |
| /* Store old stack pointer and switch to trampoline stack */ |
| movq %rsp, %rdi |
| movq %gs:__x86_tss64_t_ist2_OFFSET, %rsp |
| |
| /* Copy context */ |
| pushq 40(%rdi) /* SS */ |
| pushq 32(%rdi) /* RSP */ |
| pushq 24(%rdi) /* RFLAGS */ |
| pushq 16(%rdi) /* CS */ |
| pushq 8(%rdi) /* RIP */ |
| xchgq %rdi, (%rdi) /* Exchange old rdi to restore it and put |
| trampoline stack address in its old storage |
| area */ |
| |
| /* Switch to thread's page table */ |
| pushq %rax |
| movq %gs:__x86_tss64_t_cpu_OFFSET, %rax |
| movq ___cpu_t_current_OFFSET(%rax), %rax |
| movq _thread_offset_to_ptables(%rax), %rax |
| movq %rax, %cr3 |
| popq %rax |
| movq $0, -8(%rsp) /* Delete stashed RAX data */ |
| |
| /* Trampoline stack should have nothing sensitive in it at this point */ |
| swapgs |
| iretq |
| #endif /* CONFIG_X86_KPTI */ |
| |
| |
| /* Landing site for 'syscall' instruction |
| * |
| * Call id is in RAX |
| * Arguments are in RDI, RSI, RDX, R10, R8, R9 |
| * Return address stored by CPU in RCX |
| * User RFLAGS store by CPU in R11 |
| * Current RFLAGS has been masked with ~X86_FMASK_MSR |
| */ |
| .global z_x86_syscall_entry_stub |
| z_x86_syscall_entry_stub: |
| swapgs |
| |
| /* Save original stack pointer from user mode in memory, at the |
| * moment we have no free registers or stack to save it to. This |
| * eventually gets put on the stack before we re-enable interrupts |
| * as this is a per-cpu and not per-thread area. |
| */ |
| movq %rsp, %gs:__x86_tss64_t_usp_OFFSET |
| |
| #ifdef CONFIG_X86_KPTI |
| /* We need to switch to the trampoline stack so that we can |
| * switch to the kernel's page table |
| */ |
| movq %gs:__x86_tss64_t_ist2_OFFSET, %rsp |
| |
| /* Load kernel's page table */ |
| pushq %rax |
| |
| /* NOTE: Presumes phys=virt */ |
| movq $Z_MEM_PHYS_ADDR(z_x86_kernel_ptables), %rax |
| movq %rax, %cr3 |
| popq %rax |
| movq $0, -8(%rsp) /* Delete stashed RAX data */ |
| #endif /* CONFIG_X86_KPTI */ |
| |
| /* Switch to the privilege mode stack pointer stored in |
| * x86_tss64.psp |
| */ |
| movq %gs:__x86_tss64_t_psp_OFFSET, %rsp |
| |
| /* We're now on the privilege mode stack; push the old user stack |
| * pointer onto it |
| */ |
| pushq %gs:__x86_tss64_t_usp_OFFSET |
| #ifdef CONFIG_X86_KPTI |
| movq $0, %gs:__x86_tss64_t_usp_OFFSET |
| #endif |
| |
| sti /* re-enable interrupts */ |
| |
| /* call_id is in RAX. bounds-check it, must be less than |
| * K_SYSCALL_LIMIT. |
| */ |
| cmp $K_SYSCALL_LIMIT, %rax |
| jae _bad_syscall |
| |
| _id_ok: |
| #ifdef CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION |
| /* Prevent speculation with bogus system call IDs */ |
| lfence |
| #endif |
| |
| /* Remaining registers not involved in the syscall operation are |
| * RBX, RBP, R12-R15, plus floating point / SIMD registers. |
| * |
| * We save caller-saved registers so we can restore to original values |
| * when we call 'sysretq' at the end. |
| */ |
| pushq %rdi |
| subq $X86_FXSAVE_SIZE, %rsp |
| fxsave (%rsp) |
| pushq %rsi |
| pushq %rdx |
| pushq %r8 |
| pushq %r9 |
| pushq %r10 |
| pushq %r11 /* RFLAGS */ |
| pushq %rcx /* Return address stored by 'syscall' */ |
| pushq %rsp /* SSF parameter */ |
| |
| /* All other args are in the right registers, except arg4 which |
| * we had to put in r10 instead of RCX |
| */ |
| movq %r10, %rcx |
| |
| /* from the call ID in RAX, load R10 with the actual function pointer |
| * to call by looking it up in the system call dispatch table |
| */ |
| xorq %r11, %r11 |
| movq _k_syscall_table(%r11, %rax, 8), %r10 |
| |
| /* Run the marshal function, which is some entry in _k_syscall_table */ |
| call *%r10 |
| |
| /* RAX now contains the return value |
| * |
| * Callee-saved registers are un-touched from original values per C |
| * calling convention, but sensitive data may lurk in caller-saved regs |
| * RDI, RSI, RDX, R8, R9, R10, XMM* after we have serviced the system |
| * call. We saved them earlier, restore their original values when |
| * the syscall was made. This also preserves these registers if they |
| * were not used as arguments. |
| * |
| * We also can't have RCX and R11 clobbered as we need the original |
| * values to successfully 'sysretq'. |
| */ |
| addq $8, %rsp /* Discard ssf */ |
| popq %rcx /* Restore return address for 'sysretq' */ |
| popq %r11 /* Restore RFLAGS for 'sysretq' */ |
| popq %r10 |
| popq %r9 |
| popq %r8 |
| popq %rdx |
| popq %rsi |
| fxrstor (%rsp) |
| addq $X86_FXSAVE_SIZE, %rsp |
| popq %rdi |
| |
| #ifdef CONFIG_X86_KPTI |
| /* Lock IRQs as we are using per-cpu memory areas and the |
| * trampoline stack |
| */ |
| cli |
| |
| /* Stash user stack pointer and switch to trampoline stack */ |
| popq %gs:__x86_tss64_t_usp_OFFSET |
| movq %gs:__x86_tss64_t_ist2_OFFSET, %rsp |
| |
| /* Switch to thread's page table */ |
| pushq %rax |
| movq %gs:__x86_tss64_t_cpu_OFFSET, %rax |
| movq ___cpu_t_current_OFFSET(%rax), %rax |
| movq _thread_offset_to_ptables(%rax), %rax |
| movq %rax, %cr3 |
| popq %rax |
| movq $0, -8(%rsp) /* Delete stashed RAX data */ |
| |
| /* Restore saved user stack pointer */ |
| movq %gs:__x86_tss64_t_usp_OFFSET, %rsp |
| movq $0, %gs:__x86_tss64_t_usp_OFFSET |
| #else |
| /* Restore user stack pointer */ |
| popq %rsp |
| |
| /* Return to user mode, locking interrupts as the normal interrupt |
| * handling path will get very confused if it occurs between |
| * 'swapgs' and 'sysretq' |
| */ |
| cli |
| #endif /* CONFIG_X86_KPTI */ |
| |
| swapgs |
| sysretq |
| |
| _bad_syscall: |
| /* RAX had a bogus syscall value in it, replace with the bad syscall |
| * handler's ID, and put the bad ID as its first argument. |
| * |
| * TODO: On this and all other arches, simply immediately return |
| * with -ENOSYS, once all syscalls have a return value |
| */ |
| movq %rax, %rdi |
| movq $K_SYSCALL_BAD, %rax |
| jmp _id_ok |
| |
| /* |
| * size_t arch_user_string_nlen(const char *s, size_t maxsize, int *err_arg) |
| * ^ RDI ^ RSI ^ RDX |
| */ |
| .global arch_user_string_nlen |
| arch_user_string_nlen: |
| /* Initial error value, strlen_done adjusts this if we succeed */ |
| movl $-1, %r8d |
| |
| /* use RAX as our length count (this function's return value) */ |
| xor %rax, %rax |
| |
| /* This code might page fault */ |
| strlen_loop: |
| .global z_x86_user_string_nlen_fault_start |
| z_x86_user_string_nlen_fault_start: |
| cmpb $0x0, (%rdi, %rax, 1) /* *(RDI + RAX) == 0? Could fault. */ |
| |
| .global z_x86_user_string_nlen_fault_end |
| z_x86_user_string_nlen_fault_end: |
| je strlen_done |
| cmp %rsi, %rax /* Max length reached? */ |
| je strlen_done |
| inc %rax /* EAX++ and loop again */ |
| jmp strlen_loop |
| |
| strlen_done: |
| /* Set error value to 0 since we succeeded */ |
| xorl %r8d, %r8d |
| |
| .global z_x86_user_string_nlen_fixup |
| z_x86_user_string_nlen_fixup: |
| /* Write error value to 32-bit integer err pointer parameter */ |
| movl %r8d, (%rdx) |
| retq |
| |
| /* |
| * Trampoline function to put the p3 parameter in the register expected |
| * by the calling convention, we couldn't use RCX when we called 'sysret' |
| */ |
| z_x86_userspace_landing_site: |
| /* Place argument 4 in the correct position */ |
| movq %r10, %rcx |
| call z_thread_entry |
| |
| /* FUNC_NORETURN void z_x86_userspace_enter( |
| * k_thread_entry_t user_entry, <- RDI |
| * void *p1, void *p2, void *p3, <- RSI, RDX, RCX |
| * uintptr_t stack_end, <- R8 |
| * uintptr_t stack_start) <- R9 |
| * |
| * A one-way trip to userspace. |
| */ |
| .global z_x86_userspace_enter |
| z_x86_userspace_enter: |
| /* RCX is sysret return address, pass along p3 in r10, |
| * z_x86_userspace_landing_site will fix this up |
| */ |
| movq %rcx, %r10 |
| |
| /* switch to privilege mode stack so we can erase thread stack buffer, |
| * the buffer is the page immediately before the thread stack |
| */ |
| movq %r9, %rsp |
| |
| /* Push callee-saved regs and go back into C code to erase the stack |
| * buffer and set US bit in page tables for it |
| */ |
| pushq %rdx |
| pushq %rsi |
| pushq %rdi |
| pushq %r8 |
| pushq %r10 |
| callq z_x86_current_stack_perms |
| popq %r10 |
| popq %r8 |
| popq %rdi |
| popq %rsi |
| popq %rdx |
| |
| /* Reset to the beginning of the user stack */ |
| movq %r8, %rsp |
| |
| /* set sysret entry point */ |
| movq $z_x86_userspace_landing_site, %rcx |
| |
| /* Copy RFLAGS into r11, required by sysret */ |
| pushfq |
| movq (%rsp), %r11 |
| movq $0, (%rsp) /* Now a debugger-friendly return address */ |
| |
| /* cleanse other registers */ |
| xorq %rbx, %rbx |
| xorq %rbp, %rbp |
| xorq %r12, %r12 |
| xorq %r13, %r13 |
| xorq %r14, %r14 |
| xorq %r15, %r15 |
| |
| cli |
| |
| #ifdef CONFIG_X86_KPTI |
| /* Switch to thread's page table. We have free registers so no need |
| * to involve the trampoline stack. |
| */ |
| movq %gs:__x86_tss64_t_cpu_OFFSET, %rax |
| movq ___cpu_t_current_OFFSET(%rax), %rax |
| movq _thread_offset_to_ptables(%rax), %rax |
| movq %rax, %cr3 |
| #endif |
| swapgs |
| sysretq |
