arch/x86/core/atomic.c

/*
 * Copyright (c) 2015 Intel Corporation
 * Copyright (c) 2011-2014 Wind River Systems, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * @file Atomic ops for x86
 *
 * This module provides the atomic operators for IA-32
 * architectures on platforms that support the LOCK prefix instruction.
 *
 * The atomic operations are guaranteed to be atomic with respect
 * to interrupt service routines, and to operations performed by peer
 * processors.
 */

#include <atomic.h>
#include <toolchain.h>

/**
 *
 * @brief Atomic compare-and-set primitive
 *
 * This routine provides the compare-and-set operator. If the original value at
 * <target> equals <oldValue>, then <newValue> is stored at <target> and the
 * function returns 1.
 *
 * If the original value at <target> does not equal <oldValue>, then the store
 * is not done and the function returns 0.
 *
 * The reading of the original value at <target>, the comparison,
 * and the write of the new value (if it occurs) all happen atomically with
 * respect to both interrupts and accesses of other processors to <target>.
 *
 * @param target address to be tested
 * @param oldValue value to compare against
 * @param newValue value to compare against
 * @return Returns 1 if <newValue> is written, 0 otherwise.
 */
FUNC_NO_FP int atomic_cas(atomic_t *target, atomic_val_t oldValue,
			  atomic_val_t newValue)
{
	int eax;

	__asm__ (
	    "movl %[oldValue], %%eax\n\t"
	    "lock cmpxchg %[newValue], (%[target])\n\t"
	    "sete %%al\n\t"
	    "movzbl %%al,%%eax"
	    : "=&a" (eax)
	    : [newValue] "r" (newValue), [oldValue] "m" (oldValue),
	      [target] "r" (target));

	return eax;
}

/**
 *
 * @brief Atomic addition primitive
 *
 * This routine provides the atomic addition operator. The <value> is
 * atomically added to the value at <target>, placing the result at <target>,
 * and the old value from <target> is returned.
 *
 * @param target memory location to add to
 * @param value the value to add
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_add(atomic_t *target, atomic_val_t value)
{
	__asm__ ("lock xadd %[value], (%[target])"
		 : [value] "+r" (value)
		 : [target] "r" (target)
		 : );
	return value;
}

/**
 *
 * @brief Atomic subtraction primitive
 *
 * This routine provides the atomic subtraction operator. The <value> is
 * atomically subtracted from the value at <target>, placing the result at
 * <target>, and the old value from <target> is returned.
 *
 * @param target the memory location to subtract from
 * @param value the value to subtract
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_sub(atomic_t *target, atomic_val_t value)
{
	__asm__ ("neg %[value]\n\t"
		 "lock xadd %[value], (%[target])"
		  : [value] "+r" (value)
		  : [target] "r" (target)
		  : );
	return value;
}

/**
 *
 * @brief Atomic increment primitive
 *
 * @param target memory location to increment
 *
 * This routine provides the atomic increment operator. The value at <target>
 * is atomically incremented by 1, and the old value from <target> is returned.
 *
 * @return The value from <target> before the increment
 */
FUNC_NO_FP atomic_val_t atomic_inc(atomic_t *target)
{
	atomic_t value = 1;

	__asm__ ("lock xadd %[value], (%[target])"
		 : [value] "+r" (value)
		 : [target] "r" (target));
	return value;
}

/**
 *
 * @brief Atomic decrement primitive
 *
 * @param target memory location to decrement
 *
 * This routine provides the atomic decrement operator. The value at <target>
 * is atomically decremented by 1, and the old value from <target> is returned.
 *
 * @return The value from <target> prior to the decrement
 */
FUNC_NO_FP atomic_val_t atomic_dec(atomic_t *target)
{
	atomic_t value = -1;

	__asm__ ("lock xadd %[value], (%[target])"
		 : [value] "+r" (value)
		 : [target] "r" (target));
	return value;
}

/**
 *
 * @brief Atomic get primitive
 *
 * @param target memory location to read from
 *
 * This routine provides the atomic get primitive to atomically read
 * a value from <target>. It simply does an ordinary load.  Note that <target>
 * is expected to be aligned to a 4-byte boundary.
 *
 * @return The value read from <target>
 */
FUNC_NO_FP atomic_val_t atomic_get(const atomic_t *target)
{
	return *target;
}

/**
 *
 * @brief Atomic get-and-set primitive
 *
 * This routine provides the atomic set operator. The <value> is atomically
 * written at <target> and the previous value at <target> is returned.
 *
 * @param target the memory location to write to
 * @param value the value to write
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_set(atomic_t *target, atomic_val_t value)
{
	/*
	 * The 'lock' prefix is not required with the 'xchg' instruction.
	 * According to the IA-32 instruction reference manual:
	 *
	 *   "If a memory operand is referenced, the processor's locking
	 *    protocol is automatically implemented for the duration of
	 *    the exchange operation, regardless of the presence
	 *    or absence of the LOCK prefix or of the value of the IOPL."
	 */
	__asm__ ("xchg %[value], (%[target])"
		 : [value] "+r" (value)
		 : [target] "r" (target));

	return value;
}

/**
 *
 * @brief Atomic clear primitive
 *
 * This routine provides the atomic clear operator. The value of 0 is atomically
 * written at <target> and the previous value at <target> is returned. (Hence,
 * atomic_clear(pAtomicVar) is equivalent to atomic_set(pAtomicVar, 0).)
 *
 * @param target the memory location to write
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_clear(atomic_t *target)
{
	atomic_t value = 0;
	/* See note in atomic_set about non-use of 'lock' here */
	__asm__ ("xchg %[value], (%[target])"
		 : [value] "+r" (value)
		 : [target] "r" (target));

	return value;
}

/**
 *
 * @brief Atomic bitwise inclusive OR primitive
 *
 * This routine provides the atomic bitwise inclusive OR operator. The <value>
 * is atomically bitwise OR'ed with the value at <target>, placing the result
 * at <target>, and the previous value at <target> is returned.
 *
 * @param target the memory location to be modified
 * @param value the value to OR
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_or(atomic_t *target, atomic_val_t value)
{
	atomic_val_t eax;

	__asm__ volatile (
	    "mov %[target], %%edx\n\t"
	    /*
	     * Dereference target pointer and store in EAX, we will
	     * use this later to ensure the value hasn't changed
	     */
	    "mov (%%edx), %%eax\n\t"
	    "1:\n\t"
	    /*
	     * Set ECX to be (value <op> *eax), use ECX so we don't lose
	     * the original value in case we need to do this again
	     */
	    "mov %[value], %%ecx\n\t"
	    "or %%eax, %%ecx\n\t"
	    /*
	     * Check if *EDX (which is *target) == EAX
	     * If they differ, *target was changed, EAX gets updated with
	     * the new value of *target, and we try again
	     * If they are the same, EAX now has ECX's value which is
	     * what we want to return to the caller.
	     */
	    "lock cmpxchg %%ecx, (%%edx)\n\t"
	    "jnz 1b"
	    : "=a" (eax)
	    : [target] "m" (target), [value] "m" (value)
	    : "ecx", "edx");

	return eax;
}

/**
 *
 * @brief Atomic bitwise exclusive OR (XOR) primitive
 *
 * This routine provides the atomic bitwise exclusive OR operator. The <value>
 * is atomically bitwise XOR'ed with the value at <target>, placing the result
 * at <target>, and the previous value at <target> is returned.
 *
 * @param target the memory location to be modified
 * @param value the value to XOR
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_xor(atomic_t *target, atomic_val_t value)
{
	/*
	 * See comments in atomic_or() for explanation on how
	 * this works
	 */
	atomic_val_t eax;

	__asm__ volatile (
	    "mov %[target], %%edx\n\t"
	    "mov (%%edx), %%eax\n\t"
	    "1:\n\t"
	    "mov %[value], %%ecx\n\t"
	    "xor %%eax, %%ecx\n\t"
	    "lock cmpxchg %%ecx, (%%edx)\n\t"
	    "jnz 1b"
	    : "=a" (eax)
	    : [target] "m" (target), [value] "m" (value)
	    : "ecx", "edx");

	return eax;
}

/**
 *
 * @brief Atomic bitwise AND primitive
 *
 * This routine provides the atomic bitwise AND operator. The <value> is
 * atomically bitwise AND'ed with the value at <target>, placing the result
 * at <target>, and the previous value at <target> is returned.
 *
 * @param target the memory location to be modified
 * @param value the value to AND
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_and(atomic_t *target, atomic_val_t value)
{
	/*
	 * See comments in atomic_or() for explanation on how
	 * this works
	 */
	atomic_val_t eax;

	__asm__ volatile (
	    "mov %[target], %%edx\n\t"
	    "mov (%%edx), %%eax\n\t"
	    "1:\n\t"
	    "mov %[value], %%ecx\n\t"
	    "and %%eax, %%ecx\n\t"
	    "lock cmpxchg %%ecx, (%%edx)\n\t"
	    "jnz 1b"
	    : "=a" (eax)
	    : [target] "m" (target), [value] "m" (value)
	    : "ecx", "edx");

	return eax;
}

/**
 *
 * @brief Atomic bitwise NAND primitive
 *
 * This routine provides the atomic bitwise NAND operator. The <value> is
 * atomically bitwise NAND'ed with the value at <target>, placing the result
 * at <target>, and the previous value at <target> is returned.
 *
 * @param target the memory location to be modified
 * @param value the value to NAND
 *
 * @return The previous value from <target>
 */
FUNC_NO_FP atomic_val_t atomic_nand(atomic_t *target, atomic_val_t value)
{
	/*
	 * See comments in atomic_or() for explanation on how
	 * this works
	 */
	atomic_val_t eax;

	__asm__ volatile (
	    "mov %[target], %%edx\n\t"
	    "mov (%%edx), %%eax\n\t"
	    "1:\n\t"
	    "mov %[value], %%ecx\n\t"
	    "and %%eax, %%ecx\n\t"
	    "not %%ecx\n\t"
	    "lock cmpxchg %%ecx, (%%edx)\n\t"
	    "jnz 1b"
	    : "=a" (eax)
	    : [target] "m" (target), [value] "m" (value)
	    : "ecx", "edx");

	return eax;
}