/* **********************************************************
 * Copyright 2008 - 2009 VMware, Inc.  All rights reserved.
 * **********************************************************/

/*
 * @VMKAPIMOD_LICENSE@
 */

/*
 ***********************************************************************
 * Atomics                                                        */ /**
 * \defgroup Atomics Atomic Operations
 *
 * Interfaces to CPU atomic operations.
 *
 * @{
 ***********************************************************************
 */

#ifndef _VMKAPI_ATOMIC_H_
#define _VMKAPI_ATOMIC_H_

/** \cond never */
#ifndef VMK_HEADER_INCLUDED_FROM_VMKAPI_H
#error This vmkapi file should never be included directly but only via vmkapi.h
#endif
/** \endcond never */

/***
 * \brief Atomic unsigned 64 bit integer
 *
 * \warning Operations on unaligned vmk_atomic64's are not guaranteed to
 *          be atomic.
 *
 * \note Be aware that some compilers will not obey a type's alignment
 *       if it is embedded in a packed structure.
 */
typedef volatile vmk_uint64 vmk_atomic64
   VMK_ATTRIBUTE_ALIGN(sizeof(vmk_uint64));

/**
 * \brief Read Fence Indicator for Atomic Memory Access
 *
 * If this global is set to true, then CPU atomic operations should
 * be followed by a read fence (i.e. "lfence" instruction for the
 * x64-64 architecture).
 *
 * \note This variable should only be used in assembly language routines
 *       or fragments. C routines should utilize vmk_AtomicPrologue()
 *       and vmk_AtomicEpilogue().
 */
extern vmk_Bool vmk_AtomicUseFence;

/*
 ***********************************************************************
 * vmk_AtomicPrologue --                                          */ /**
 *
 * \ingroup Atomics
 * \brief Setup for atomic operations
 *
 * This routine should be invoked before atomic operations to allow
 * for errata work-arounds.
 *
 ***********************************************************************
 */
static VMK_ALWAYS_INLINE void vmk_AtomicPrologue(
   void)
{
   /* Nothing for now */
}

/*
 ***********************************************************************
 * vmk_AtomicEpilogue --                                          */ /**
 *
 * \ingroup Atomics
 * \brief Finish atomic operations
 *
 * This routine should be invoked after atomic operations to allow
 * for errata work-arounds.
 *
 ***********************************************************************
 */
static VMK_ALWAYS_INLINE void vmk_AtomicEpilogue(
   void)
{
   if (VMK_UNLIKELY(vmk_AtomicUseFence)) {
      vmk_CPUMemFenceRead();
   }
}

/*
 ***********************************************************************
 * vmk_AtomicInc64 --                                             */ /**
 *
 * \ingroup Atomics
 * \brief Atomically increment 
 *
 * \param[out] var   Atomic to increment.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicInc64(
   vmk_atomic64 *var)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; incq %0"
      : "+m" (*var)
      :
      : "cc"
   );
   vmk_AtomicEpilogue();
}

/*
 ***********************************************************************
 * vmk_AtomicDec64 --                                             */ /**
 *
 * \ingroup Atomics
 * \brief Atomically decrement
 *
 * \param[out] var   Atomic to decrement.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicDec64(
   vmk_atomic64 *var)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; decq %0"
      : "+m" (*var)
      :
      : "cc"
   );
   vmk_AtomicEpilogue();
}

/*
 ***********************************************************************
 * vmk_AtomicAdd64 --                                             */ /**
 *
 * \ingroup Atomics
 * \brief Atomically add
 *
 * \param[out] var   Atomic to add to.
 * \param[in]  val   Value to add to atomic.
 * \return None.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicAdd64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; addq %1, %0"
      : "+m" (*var)
      : "re" (val)
      : "cc"
   );
   vmk_AtomicEpilogue();
}

/*
 ***********************************************************************
 * vmk_AtomicSub64 --                                             */ /**
 *
 * \ingroup Atomics
 * \brief Atomically subtract
 *
 * \param[out] var   Atomic to subtract from.
 * \param[in]  val   Value to subtract from atomic.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicSub64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; subq %1, %0"
      : "+m" (*var)
      : "re" (val)
      : "cc"
   );
   vmk_AtomicEpilogue();
}

/*
 ***********************************************************************
 * vmk_AtomicOr64 --                                              */ /**
 *
 * \ingroup Atomics
 * \brief Atomically bitwise OR
 *
 * \param[out] var   Atomic to OR
 * \param[in] val    Value to OR
 * \return None.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicOr64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; orq %1, %0"
      : "+m" (*var)
      : "re" (val)
      : "cc"
   );
   vmk_AtomicEpilogue();
}

/*
 ***********************************************************************
 * vmk_AtomicAnd64 --                                             */ /**
 *
 * \ingroup Atomics
 * \brief Atomically bitwise AND
 *
 * \param[out] var   Atomic to AND.
 * \param[in]  val   Value to AND.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicAnd64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; andq %1, %0"
      : "+m" (*var)
      : "re" (val)
      : "cc"
   );
   vmk_AtomicEpilogue();
}

/*
 ***********************************************************************
 * vmk_AtomicXor64 --                                             */ /**
 *
 * \ingroup Atomics
 * \brief Atomically bitwise XOR
 *
 * \param[out] var   Atomic to XOR.
 * \param[in] val    Value to XOR.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicXor64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; xorq %1, %0"
      : "+m" (*var)
      : "re" (val)
      : "cc"
   );
   vmk_AtomicEpilogue();
}

/*
 ***********************************************************************
 * vmk_AtomicRead64 --                                            */ /**
 *
 * \ingroup Atomics
 * \brief Atomically read
 *
 * \param[out] var   Atomic to read.
 *
 * \return Value of the atomic.
 *
 ***********************************************************************
 */
static inline vmk_uint64 vmk_AtomicRead64(
   vmk_atomic64 const *var)
{
   /* Ensure alignment otherwise this isn't atomic. */
   VMK_ASSERT(((vmk_uintptr_t)var & (vmk_uintptr_t)(sizeof(*var) - 1)) == 0);
   return *var;
}

/*
 ***********************************************************************
 * vmk_AtomicWrite64 --                                           */ /**
 *
 * \ingroup Atomics
 * \brief Atomically write
 *
 * \param[out] var   Atomic to write.
 * \param[in]  val   Value to write.
 *
 ***********************************************************************
 */
static inline void vmk_AtomicWrite64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   /* Ensure alignment otherwise this isn't atomic. */
   VMK_ASSERT(((vmk_uintptr_t)var & (vmk_uintptr_t)(sizeof(*var) - 1)) == 0);

   /*
    * Ensure that we do a single movq. Without this, the compiler
    * may do write with a constant as two movl operations.
    */
   __asm__ __volatile__(
      "movq %1, %0"
      : "=m" (*var)
      : "r" (val)
   );
}

/*
 ***********************************************************************
 * vmk_AtomicReadWrite64 --                                       */ /**
 *
 * \ingroup Atomics
 * \brief Atomically read and write
 *
 * \param[out] var   Atomic to read.
 * \param[in]  val   Value to write.
 *
 * \return Value of the atomic before written.
 *
 ***********************************************************************
 */
static inline vmk_uint64 vmk_AtomicReadWrite64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   vmk_AtomicPrologue();
   __asm__ __volatile__(
      /* The lock prefix is not strictly necesessary but it is imformative. */
      "lock; xchgq %0, %1"
      : "=r" (val),
	"+m" (*var)
      : "0" (val)
   );
   vmk_AtomicEpilogue();

   return val;
}

/*
 ***********************************************************************
 * vmk_AtomicReadIfEqualWrite64 --                                */ /**
 *
 * \ingroup Atomics
 * \brief Atomically read, compare, and conditionally write
 *
 * Atomically writes new to var if its current value is old.
 * The value returned is old if new was written, else
 * the current unmodified value of the atomic is returned
 *
 * \param[out] var   Atomic to read.
 * \param[in]  old   Value to compare.
 * \param[in]  new   Value to write.
 *
 * \return Value of the atomic before written.
 *
 ***********************************************************************
 */
static inline vmk_uint64 vmk_AtomicReadIfEqualWrite64(
   vmk_atomic64 *var,
   vmk_uint64 old,
   vmk_uint64 new)
{
   vmk_uint64 val;

   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; cmpxchgq %2, %1"
      : "=a" (val),
	"+m" (*var)
      : "r" (new),
	"0" (old)
      : "cc"
   );
   vmk_AtomicEpilogue();

   return val;
}

/*
 ***********************************************************************
 * vmk_AtomicReadAdd64 --                                         */ /**
 *
 * \ingroup Atomics
 * \brief Atomically read and add
 *
 * \param[out] var   Atomic to read.
 * \param[in]  val   Value to add.
 *
 * \return Value of the atomic before addition.
 *
 ***********************************************************************
 */
static inline vmk_uint64 vmk_AtomicReadAdd64(
  vmk_atomic64 *var,
  vmk_uint64 val)
{

   vmk_AtomicPrologue();
   __asm__ __volatile__(
      "lock; xaddq %0, %1"
      : "=r" (val),
	"+m" (*var)
      : "0" (val)
      : "cc"
   );
   vmk_AtomicEpilogue();

   return val;
}

/*
 ***********************************************************************
 * vmk_AtomicReadInc64 --                                         */ /**
 *
 * \ingroup Atomics
 * \brief Atomically read and increment
 *
 * \param[out] var   Atomic to read.
 *
 * \return Value of the atomic before incremented.
 *
 ***********************************************************************
 */
static inline vmk_uint64 vmk_AtomicReadInc64(
   vmk_atomic64 *var)
{
   return vmk_AtomicReadAdd64(var, 1);
}

/*
 ***********************************************************************
 * vmk_AtomicReadDec64 --                                         */ /**
 *
 * \ingroup Atomics
 * \brief Atomically read and decrement
 *
 * \param[out] var   Atomic to read.
 *
 * \return Value of the atomic before decremented.
 *
 ***********************************************************************
 */
static inline vmk_uint64 vmk_AtomicReadDec64(
   vmk_atomic64 *var)
{
   return vmk_AtomicReadAdd64(var, VMK_CONST64U(-1));
}

/*
 ***********************************************************************
 * vmk_AtomicReadOr64 --                                          */ /**
 *
 * \ingroup Atomics
 * \brief Atomically read and bitwise OR with a value
 *
 * \param[out] var   Atomic to read/OR.
 * \param[in]  val   Value to OR atomically.
 *
 * \return Value of the atomic before the operation.
 *
 ***********************************************************************
 */
static inline vmk_uint64 vmk_AtomicReadOr64(
   vmk_atomic64 *var,
   vmk_uint64 val)
{
   vmk_uint64 res;

   do {
      res = vmk_AtomicRead64(var);
   } while (res != vmk_AtomicReadIfEqualWrite64(var, res, res | val));

   return res;
}

#endif /* _VMKAPI_ATOMIC_H_ */
/** @} */