arch/x86/core/ia32/float.c

/*
 * Copyright (c) 2010-2014 Wind River Systems, Inc.
 *
 * SPDX-License-Identifier: Apache-2.0
 */

/**
 * @file
 * @brief Floating point register sharing routines
 *
 * This module allows multiple preemptible threads to safely share the system's
 * floating point registers, by allowing the system to save FPU state info
 * in a thread's stack region when a preemptive context switch occurs.
 *
 * Note: If the kernel has been built without floating point register sharing
 * support (CONFIG_FPU_SHARING), the floating point registers can still be used
 * safely by one or more cooperative threads OR by a single preemptive thread,
 * but not by both.
 *
 * This code is not necessary for systems with CONFIG_EAGER_FPU_SHARING, as
 * the floating point context is unconditionally saved/restored with every
 * context switch.
 *
 * The floating point register sharing mechanism is designed for minimal
 * intrusiveness.  Floating point state saving is only performed for threads
 * that explicitly indicate they are using FPU registers, to avoid impacting
 * the stack size requirements of all other threads. Also, the SSE registers
 * are only saved for threads that actually used them. For those threads that
 * do require floating point state saving, a "lazy save/restore" mechanism
 * is employed so that the FPU's register sets are only switched in and out
 * when absolutely necessary; this avoids wasting effort preserving them when
 * there is no risk that they will be altered, or when there is no need to
 * preserve their contents.
 *
 * WARNING
 * The use of floating point instructions by ISRs is not supported by the
 * kernel.
 *
 * INTERNAL
 * The kernel sets CR0[TS] to 0 only for threads that require FP register
 * sharing. All other threads have CR0[TS] set to 1 so that an attempt
 * to perform an FP operation will cause an exception, allowing the kernel
 * to enable FP register sharing on its behalf.
 */

#include <zephyr/kernel.h>
#include <kernel_internal.h>

/* SSE control/status register default value (used by assembler code) */
extern uint32_t _sse_mxcsr_default_value;

/**
 * @brief Disallow use of floating point capabilities
 *
 * This routine sets CR0[TS] to 1, which disallows the use of FP instructions
 * by the currently executing thread.
 */
static inline void z_FpAccessDisable(void)
{
	void *tempReg;

	__asm__ volatile(
		"movl %%cr0, %0;\n\t"
		"orl $0x8, %0;\n\t"
		"movl %0, %%cr0;\n\t"
		: "=r"(tempReg)
		:
		: "memory");
}


/**
 * @brief Save non-integer context information
 *
 * This routine saves the system's "live" non-integer context into the
 * specified area.  If the specified thread supports SSE then
 * x87/MMX/SSEx thread info is saved, otherwise only x87/MMX thread is saved.
 * Function is invoked by FpCtxSave(struct k_thread *thread)
 */
static inline void z_do_fp_regs_save(void *preemp_float_reg)
{
	__asm__ volatile("fnsave (%0);\n\t"
			 :
			 : "r"(preemp_float_reg)
			 : "memory");
}

/**
 * @brief Save non-integer context information
 *
 * This routine saves the system's "live" non-integer context into the
 * specified area.  If the specified thread supports SSE then
 * x87/MMX/SSEx thread info is saved, otherwise only x87/MMX thread is saved.
 * Function is invoked by FpCtxSave(struct k_thread *thread)
 */
static inline void z_do_fp_and_sse_regs_save(void *preemp_float_reg)
{
	__asm__ volatile("fxsave (%0);\n\t"
			 :
			 : "r"(preemp_float_reg)
			 : "memory");
}

/**
 * @brief Initialize floating point register context information.
 *
 * This routine initializes the system's "live" floating point registers.
 */
static inline void z_do_fp_regs_init(void)
{
	__asm__ volatile("fninit\n\t");
}

/**
 * @brief Initialize SSE register context information.
 *
 * This routine initializes the system's "live" SSE registers.
 */
static inline void z_do_sse_regs_init(void)
{
	__asm__ volatile("ldmxcsr _sse_mxcsr_default_value\n\t");
}

/*
 * Save a thread's floating point context information.
 *
 * This routine saves the system's "live" floating point context into the
 * specified thread control block. The SSE registers are saved only if the
 * thread is actually using them.
 */
static void FpCtxSave(struct k_thread *thread)
{
#ifdef CONFIG_X86_SSE
	if ((thread->base.user_options & K_SSE_REGS) != 0) {
		z_do_fp_and_sse_regs_save(&thread->arch.preempFloatReg);
		return;
	}
#endif
	z_do_fp_regs_save(&thread->arch.preempFloatReg);
}

/*
 * Initialize a thread's floating point context information.
 *
 * This routine initializes the system's "live" floating point context.
 * The SSE registers are initialized only if the thread is actually using them.
 */
static inline void FpCtxInit(struct k_thread *thread)
{
	z_do_fp_regs_init();
#ifdef CONFIG_X86_SSE
	if ((thread->base.user_options & K_SSE_REGS) != 0) {
		z_do_sse_regs_init();
	}
#endif
}

/*
 * Enable preservation of floating point context information.
 *
 * The transition from "non-FP supporting" to "FP supporting" must be done
 * atomically to avoid confusing the floating point logic used by z_swap(), so
 * this routine locks interrupts to ensure that a context switch does not occur.
 * The locking isn't really needed when the routine is called by a cooperative
 * thread (since context switching can't occur), but it is harmless.
 */
void z_float_enable(struct k_thread *thread, unsigned int options)
{
	unsigned int imask;
	struct k_thread *fp_owner;

	/* Ensure a preemptive context switch does not occur */

	imask = irq_lock();

	/* Indicate thread requires floating point context saving */

	thread->base.user_options |= (uint8_t)options;

	/*
	 * The current thread might not allow FP instructions, so clear CR0[TS]
	 * so we can use them. (CR0[TS] gets restored later on, if necessary.)
	 */

	__asm__ volatile("clts\n\t");

	/*
	 * Save existing floating point context (since it is about to change),
	 * but only if the FPU is "owned" by an FP-capable task that is
	 * currently handling an interrupt or exception (meaning its FP context
	 * must be preserved).
	 */

	fp_owner = _kernel.current_fp;
	if (fp_owner != NULL) {
		if ((fp_owner->arch.flags & X86_THREAD_FLAG_ALL) != 0) {
			FpCtxSave(fp_owner);
		}
	}

	/* Now create a virgin FP context */

	FpCtxInit(thread);

	/* Associate the new FP context with the specified thread */

	if (thread == _current) {
		/*
		 * When enabling FP support for the current thread, just claim
		 * ownership of the FPU and leave CR0[TS] unset.
		 *
		 * (The FP context is "live" in hardware, not saved in TCS.)
		 */

		_kernel.current_fp = thread;
	} else {
		/*
		 * When enabling FP support for someone else, assign ownership
		 * of the FPU to them (unless we need it ourselves).
		 */

		if ((_current->base.user_options & _FP_USER_MASK) == 0) {
			/*
			 * We are not FP-capable, so mark FPU as owned by the
			 * thread we've just enabled FP support for, then
			 * disable our own FP access by setting CR0[TS] back
			 * to its original state.
			 */

			_kernel.current_fp = thread;
			z_FpAccessDisable();
		} else {
			/*
			 * We are FP-capable (and thus had FPU ownership on
			 * entry), so save the new FP context in their TCS,
			 * leave FPU ownership with self, and leave CR0[TS]
			 * unset.
			 *
			 * The saved FP context is needed in case the thread
			 * we enabled FP support for is currently pre-empted,
			 * since z_swap() uses it to restore FP context when
			 * the thread re-activates.
			 *
			 * Saving the FP context reinits the FPU, and thus
			 * our own FP context, but that's OK since it didn't
			 * need to be preserved. (i.e. We aren't currently
			 * handling an interrupt or exception.)
			 */

			FpCtxSave(thread);
		}
	}

	irq_unlock(imask);
}

/**
 * Disable preservation of floating point context information.
 *
 * The transition from "FP supporting" to "non-FP supporting" must be done
 * atomically to avoid confusing the floating point logic used by z_swap(), so
 * this routine locks interrupts to ensure that a context switch does not occur.
 * The locking isn't really needed when the routine is called by a cooperative
 * thread (since context switching can't occur), but it is harmless.
 */
int z_float_disable(struct k_thread *thread)
{
	unsigned int imask;

	/* Ensure a preemptive context switch does not occur */

	imask = irq_lock();

	/* Disable all floating point capabilities for the thread */

	thread->base.user_options &= ~_FP_USER_MASK;

	if (thread == _current) {
		z_FpAccessDisable();
		_kernel.current_fp = (struct k_thread *)0;
	} else {
		if (_kernel.current_fp == thread) {
			_kernel.current_fp = (struct k_thread *)0;
		}
	}

	irq_unlock(imask);

	return 0;
}

/*
 * Handler for "device not available" exception.
 *
 * This routine is registered to handle the "device not available" exception
 * (vector = 7).
 *
 * The processor will generate this exception if any x87 FPU, MMX, or SSEx
 * instruction is executed while CR0[TS]=1. The handler then enables the
 * current thread to use all supported floating point registers.
 */
void _FpNotAvailableExcHandler(z_arch_esf_t *pEsf)
{
	ARG_UNUSED(pEsf);

	/*
	 * Assume the exception did not occur in an ISR.
	 * (In other words, CPU cycles will not be consumed to perform
	 * error checking to ensure the exception was not generated in an ISR.)
	 */

	/* Enable highest level of FP capability configured into the kernel */

	k_float_enable(_current, _FP_USER_MASK);
}
_EXCEPTION_CONNECT_NOCODE(_FpNotAvailableExcHandler,
		IV_DEVICE_NOT_AVAILABLE, 0);