| /* |
| * Copyright (c) 2010-2014 Wind River Systems, Inc. |
| * |
| * SPDX-License-Identifier: Apache-2.0 |
| */ |
| |
| /** |
| * @file |
| * @brief Interrupt management support for IA-32 architecture |
| * |
| * This module implements assembly routines to manage interrupts on |
| * the Intel IA-32 architecture. More specifically, the interrupt (asynchronous |
| * exception) stubs are implemented in this module. The stubs are invoked when |
| * entering and exiting a C interrupt handler. |
| */ |
| |
| #include <kernel_structs.h> |
| #include <arch/x86/asm.h> |
| #include <offsets_short.h> |
| #include <arch/cpu.h> /* _NANO_ERR_SPURIOUS_INT */ |
| #include <arch/x86/irq_controller.h> |
| |
| /* exports (internal APIs) */ |
| |
| GTEXT(_interrupt_enter) |
| GTEXT(_SpuriousIntNoErrCodeHandler) |
| GTEXT(_SpuriousIntHandler) |
| GTEXT(_irq_sw_handler) |
| |
| /* externs */ |
| |
| GTEXT(__swap) |
| #if defined(CONFIG_TIMESLICING) |
| GTEXT(_update_time_slice_before_swap) |
| #endif |
| |
| #ifdef CONFIG_SYS_POWER_MANAGEMENT |
| GTEXT(_sys_power_save_idle_exit) |
| #endif |
| |
| |
| #ifdef CONFIG_INT_LATENCY_BENCHMARK |
| GTEXT(_int_latency_start) |
| GTEXT(_int_latency_stop) |
| #endif |
| /** |
| * |
| * @brief Inform the kernel of an interrupt |
| * |
| * This function is called from the interrupt stub created by IRQ_CONNECT() |
| * to inform the kernel of an interrupt. This routine increments |
| * _kernel.nested (to support interrupt nesting), switches to the |
| * base of the interrupt stack, if not already on the interrupt stack, and then |
| * saves the volatile integer registers onto the stack. Finally, control is |
| * returned back to the interrupt stub code (which will then invoke the |
| * "application" interrupt service routine). |
| * |
| * Only the volatile integer registers are saved since ISRs are assumed not to |
| * utilize floating point (or SSE) instructions. |
| * |
| * WARNINGS |
| * |
| * Host-based tools and the target-based GDB agent depend on the stack frame |
| * created by this routine to determine the locations of volatile registers. |
| * These tools must be updated to reflect any changes to the stack frame. |
| * |
| * @return N/A |
| * |
| * C function prototype: |
| * |
| * void _interrupt_enter(void *isr, void *isr_param); |
| */ |
| SECTION_FUNC(TEXT, _interrupt_enter) |
| |
| #ifdef CONFIG_EXECUTION_BENCHMARKING |
| pushl %eax |
| pushl %edx |
| rdtsc |
| mov %eax, __start_intr_time |
| mov %edx, __start_intr_time+4 |
| pop %edx |
| pop %eax |
| #endif |
| /* |
| * The gen_idt tool creates an interrupt-gate descriptor for |
| * all connections. The processor will automatically clear the IF |
| * bit in the EFLAGS register upon execution of the handler, hence |
| * this need not issue an 'cli' as the first instruction. |
| * |
| * Clear the direction flag. It is automatically restored when the |
| * interrupt exits via the IRET instruction. |
| */ |
| |
| cld |
| |
| /* |
| * Note that the processor has pushed both the EFLAGS register |
| * and the logical return address (cs:eip) onto the stack prior |
| * to invoking the handler specified in the IDT. The stack looks |
| * like this: |
| * |
| * EFLAGS |
| * CS |
| * EIP |
| * isr_param |
| * isr <-- stack pointer |
| */ |
| |
| |
| /* |
| * Swap EAX with isr_param and EDX with isr. |
| * Push ECX onto the stack |
| */ |
| xchgl %eax, 4(%esp) |
| xchgl %edx, (%esp) |
| pushl %ecx |
| |
| /* Now the stack looks like: |
| * |
| * EFLAGS |
| * CS |
| * EIP |
| * saved EAX |
| * saved EDX |
| * saved ECX |
| * |
| * EAX = isr_param, EDX = isr |
| */ |
| |
| /* Push EDI as we will use it for scratch space. |
| * Rest of the callee-saved regs get saved by invocation of C |
| * functions (isr handler, __swap(), etc) |
| */ |
| pushl %edi |
| |
| |
| #if defined(CONFIG_INT_LATENCY_BENCHMARK) || \ |
| defined(CONFIG_KERNEL_EVENT_LOGGER_INTERRUPT) || \ |
| defined(CONFIG_KERNEL_EVENT_LOGGER_SLEEP) |
| |
| /* Save these as we are using to keep track of isr and isr_param */ |
| pushl %eax |
| pushl %edx |
| |
| #ifdef CONFIG_INT_LATENCY_BENCHMARK |
| /* |
| * Volatile registers are now saved it is safe to start measuring |
| * how long interrupt are disabled. |
| * The interrupt gate created by IRQ_CONNECT disables the |
| * interrupt. |
| */ |
| |
| call _int_latency_start |
| #endif |
| |
| #ifdef CONFIG_KERNEL_EVENT_LOGGER_INTERRUPT |
| call _sys_k_event_logger_interrupt |
| #endif |
| |
| #ifdef CONFIG_KERNEL_EVENT_LOGGER_SLEEP |
| call _sys_k_event_logger_exit_sleep |
| #endif |
| |
| popl %edx |
| popl %eax |
| #endif |
| |
| /* load %ecx with &_kernel */ |
| |
| movl $_kernel, %ecx |
| |
| /* switch to the interrupt stack for the non-nested case */ |
| |
| incl _kernel_offset_to_nested(%ecx) |
| |
| /* use interrupt stack if not nested */ |
| cmpl $1, _kernel_offset_to_nested(%ecx) |
| jne alreadyOnIntStack |
| |
| /* |
| * switch to base of the interrupt stack: save esp in edi, then load |
| * irq_stack pointer |
| */ |
| |
| movl %esp, %edi |
| movl _kernel_offset_to_irq_stack(%ecx), %esp |
| |
| |
| /* save thread's stack pointer onto base of interrupt stack */ |
| |
| pushl %edi /* Save stack pointer */ |
| |
| #ifdef CONFIG_SYS_POWER_MANAGEMENT |
| cmpl $0, _kernel_offset_to_idle(%ecx) |
| jne handle_idle |
| /* fast path is !idle, in the pipeline */ |
| #endif /* CONFIG_SYS_POWER_MANAGEMENT */ |
| |
| /* fall through to nested case */ |
| |
| alreadyOnIntStack: |
| #ifdef CONFIG_INT_LATENCY_BENCHMARK |
| pushl %eax |
| pushl %edx |
| call _int_latency_stop |
| popl %edx |
| popl %eax |
| #endif |
| |
| #ifndef CONFIG_X86_IAMCU |
| /* EAX has the interrupt handler argument, needs to go on |
| * stack for sys V calling convention |
| */ |
| push %eax |
| #endif |
| #ifdef CONFIG_EXECUTION_BENCHMARKING |
| /* Save the eax and edx registers before reading the time stamp |
| * once done pop the values |
| */ |
| pushl %eax |
| pushl %edx |
| rdtsc |
| mov %eax,__end_intr_time |
| mov %edx,__end_intr_time+4 |
| pop %edx |
| pop %eax |
| #endif |
| |
| #ifdef CONFIG_NESTED_INTERRUPTS |
| sti /* re-enable interrupts */ |
| #endif |
| /* Now call the interrupt handler */ |
| call *%edx |
| #ifndef CONFIG_X86_IAMCU |
| /* Discard ISR argument */ |
| addl $0x4, %esp |
| #endif |
| #ifdef CONFIG_NESTED_INTERRUPTS |
| cli /* disable interrupts again */ |
| #endif |
| |
| /* irq_controller.h interface */ |
| _irq_controller_eoi_macro |
| |
| #ifdef CONFIG_INT_LATENCY_BENCHMARK |
| call _int_latency_start |
| #endif |
| |
| /* determine whether exiting from a nested interrupt */ |
| movl $_kernel, %ecx |
| decl _kernel_offset_to_nested(%ecx) /* dec interrupt nest count */ |
| jne nestedInterrupt /* 'iret' if nested case */ |
| |
| |
| #ifdef CONFIG_PREEMPT_ENABLED |
| movl _kernel_offset_to_current(%ecx), %edx |
| |
| /* |
| * Non-preemptible thread ? Do not schedule (see explanation of |
| * preempt field in kernel_struct.h). |
| */ |
| cmpw $_NON_PREEMPT_THRESHOLD, _thread_offset_to_preempt(%edx) |
| jae noReschedule |
| |
| |
| /* reschedule only if the scheduler says that we must do so */ |
| cmpl %edx, _kernel_offset_to_ready_q_cache(%ecx) |
| je noReschedule |
| |
| /* |
| * Set the _INT_ACTIVE bit in the k_thread to allow the upcoming call to |
| * __swap() to determine whether non-floating registers need to be |
| * preserved using the lazy save/restore algorithm, or to indicate to |
| * debug tools that a preemptive context switch has occurred. |
| */ |
| |
| #if defined(CONFIG_FP_SHARING) |
| orb $_INT_ACTIVE, _thread_offset_to_thread_state(%edx) |
| #endif |
| |
| /* |
| * A context reschedule is required: keep the volatile registers of |
| * the interrupted thread on the context's stack. Utilize |
| * the existing __swap() primitive to save the remaining |
| * thread's registers (including floating point) and perform |
| * a switch to the new thread. |
| */ |
| |
| popl %esp /* switch back to outgoing thread's stack */ |
| |
| #if defined(CONFIG_TIMESLICING) |
| call _update_time_slice_before_swap |
| #endif |
| #ifdef CONFIG_STACK_SENTINEL |
| call _check_stack_sentinel |
| #endif |
| pushfl /* push KERNEL_LOCK_KEY argument */ |
| #ifdef CONFIG_X86_IAMCU |
| /* IAMCU first argument goes into a register, not the stack. |
| */ |
| popl %eax |
| #endif |
| call __swap |
| |
| #ifndef CONFIG_X86_IAMCU |
| addl $4, %esp /* pop KERNEL_LOCK_KEY argument */ |
| #endif |
| /* |
| * The interrupted thread has now been scheduled, |
| * as the result of a _later_ invocation of __swap(). |
| * |
| * Now need to restore the interrupted thread's environment before |
| * returning control to it at the point where it was interrupted ... |
| */ |
| |
| #if defined(CONFIG_FP_SHARING) |
| /* |
| * __swap() has restored the floating point registers, if needed. |
| * Clear the _INT_ACTIVE bit in the interrupted thread's state |
| * since it has served its purpose. |
| */ |
| |
| movl _kernel + _kernel_offset_to_current, %eax |
| andb $~_INT_ACTIVE, _thread_offset_to_thread_state(%eax) |
| #endif /* CONFIG_FP_SHARING */ |
| |
| /* Restore volatile registers and return to the interrupted thread */ |
| #ifdef CONFIG_INT_LATENCY_BENCHMARK |
| call _int_latency_stop |
| #endif |
| popl %edi |
| popl %ecx |
| popl %edx |
| popl %eax |
| |
| /* Pop of EFLAGS will re-enable interrupts and restore direction flag */ |
| iret |
| |
| #endif /* CONFIG_PREEMPT_ENABLED */ |
| |
| noReschedule: |
| |
| /* |
| * A thread reschedule is not required; switch back to the |
| * interrupted thread's stack and restore volatile registers |
| */ |
| |
| popl %esp /* pop thread stack pointer */ |
| |
| #ifdef CONFIG_STACK_SENTINEL |
| call _check_stack_sentinel |
| #endif |
| |
| /* fall through to 'nestedInterrupt' */ |
| |
| |
| /* |
| * For the nested interrupt case, the interrupt stack must still be |
| * utilized, and more importantly, a rescheduling decision must |
| * not be performed. |
| */ |
| |
| nestedInterrupt: |
| #ifdef CONFIG_INT_LATENCY_BENCHMARK |
| call _int_latency_stop |
| #endif |
| |
| popl %edi |
| popl %ecx /* pop volatile registers in reverse order */ |
| popl %edx |
| popl %eax |
| /* Pop of EFLAGS will re-enable interrupts and restore direction flag */ |
| iret |
| |
| |
| #ifdef CONFIG_SYS_POWER_MANAGEMENT |
| handle_idle: |
| pushl %eax |
| pushl %edx |
| /* Populate 'ticks' argument to _sys_power_save_idle_exit */ |
| #ifdef CONFIG_X86_IAMCU |
| movl _kernel_offset_to_idle(%ecx), %eax |
| #else |
| /* SYS V calling convention */ |
| push _kernel_offset_to_idle(%ecx) |
| #endif |
| /* Zero out _kernel.idle */ |
| movl $0, _kernel_offset_to_idle(%ecx) |
| |
| /* |
| * Beware that a timer driver's _sys_power_save_idle_exit() implementation might |
| * expect that interrupts are disabled when invoked. This ensures that |
| * the calculation and programming of the device for the next timer |
| * deadline is not interrupted. |
| */ |
| |
| call _sys_power_save_idle_exit |
| #ifndef CONFIG_X86_IAMCU |
| /* SYS V: discard 'ticks' argument passed on the stack */ |
| add $0x4, %esp |
| #endif |
| popl %edx |
| popl %eax |
| jmp alreadyOnIntStack |
| #endif /* CONFIG_SYS_POWER_MANAGEMENT */ |
| |
| /** |
| * |
| * _SpuriousIntHandler - |
| * @brief Spurious interrupt handler stubs |
| * |
| * Interrupt-gate descriptors are statically created for all slots in the IDT |
| * that point to _SpuriousIntHandler() or _SpuriousIntNoErrCodeHandler(). The |
| * former stub is connected to exception vectors where the processor pushes an |
| * error code onto the stack (or kernel stack) in addition to the EFLAGS/CS/EIP |
| * records. |
| * |
| * A spurious interrupt is considered a fatal condition, thus this routine |
| * merely sets up the 'reason' and 'pEsf' parameters to the routine |
| * _SysFatalHwErrorHandler(). In other words, there is no provision to return |
| * to the interrupted execution context and thus the volatile registers are not |
| * saved. |
| * |
| * @return Never returns |
| * |
| * C function prototype: |
| * |
| * void _SpuriousIntHandler (void); |
| * |
| * INTERNAL |
| * The gen_idt tool creates an interrupt-gate descriptor for all |
| * connections. The processor will automatically clear the IF bit |
| * in the EFLAGS register upon execution of the handler, |
| * thus _SpuriousIntNoErrCodeHandler()/_SpuriousIntHandler() shall be |
| * invoked with interrupts disabled. |
| */ |
| SECTION_FUNC(TEXT, _SpuriousIntNoErrCodeHandler) |
| |
| pushl $0 /* push dummy err code onto stk */ |
| |
| /* fall through to _SpuriousIntHandler */ |
| |
| |
| SECTION_FUNC(TEXT, _SpuriousIntHandler) |
| |
| cld /* Clear direction flag */ |
| |
| /* Create the ESF */ |
| |
| pushl %eax |
| pushl %ecx |
| pushl %edx |
| pushl %edi |
| pushl %esi |
| pushl %ebx |
| pushl %ebp |
| |
| leal 44(%esp), %ecx /* Calculate ESP before exception occurred */ |
| pushl %ecx /* Save calculated ESP */ |
| |
| #ifndef CONFIG_X86_IAMCU |
| pushl %esp /* push cur stack pointer: pEsf arg */ |
| #else |
| mov %esp, %edx |
| #endif |
| |
| /* re-enable interrupts */ |
| sti |
| |
| /* push the 'unsigned int reason' parameter */ |
| #ifndef CONFIG_X86_IAMCU |
| pushl $_NANO_ERR_SPURIOUS_INT |
| #else |
| movl $_NANO_ERR_SPURIOUS_INT, %eax |
| #endif |
| /* call the fatal error handler */ |
| call _NanoFatalErrorHandler |
| |
| /* handler doesn't return */ |
| |
| #if CONFIG_IRQ_OFFLOAD |
| SECTION_FUNC(TEXT, _irq_sw_handler) |
| push $0 |
| push $_irq_do_offload |
| jmp _interrupt_enter |
| |
| #endif |