path: root/intern/cycles/util/ssef.h

/* SPDX-License-Identifier: Apache-2.0
 * Copyright 2011-2013 Intel Corporation
 * Modifications Copyright 2014-2022 Blender Foundation. */

#ifndef __UTIL_SSEF_H__
#define __UTIL_SSEF_H__

#include <math.h>

#include "util/ssei.h"

CCL_NAMESPACE_BEGIN

#ifdef __KERNEL_SSE2__

struct sseb;
struct ssef;

/*! 4-wide SSE float type. */
struct ssef {
  typedef sseb Mask;   // mask type
  typedef ssei Int;    // int type
  typedef ssef Float;  // float type

  enum { size = 4 };  // number of SIMD elements
  union {
    __m128 m128;
    float f[4];
    int i[4];
  };  // data

  ////////////////////////////////////////////////////////////////////////////////
  /// Constructors, Assignment & Cast Operators
  ////////////////////////////////////////////////////////////////////////////////

  __forceinline ssef()
  {
  }
  __forceinline ssef(const ssef &other)
  {
    m128 = other.m128;
  }
  __forceinline ssef &operator=(const ssef &other)
  {
    m128 = other.m128;
    return *this;
  }

  __forceinline ssef(const __m128 a) : m128(a)
  {
  }
  __forceinline operator const __m128 &() const
  {
    return m128;
  }
  __forceinline operator __m128 &()
  {
    return m128;
  }

  __forceinline ssef(float a) : m128(_mm_set1_ps(a))
  {
  }
  __forceinline ssef(float a, float b, float c, float d) : m128(_mm_setr_ps(a, b, c, d))
  {
  }

  __forceinline explicit ssef(const __m128i a) : m128(_mm_cvtepi32_ps(a))
  {
  }

  ////////////////////////////////////////////////////////////////////////////////
  /// Loads and Stores
  ////////////////////////////////////////////////////////////////////////////////

#  if defined(__KERNEL_AVX__)
  static __forceinline ssef broadcast(const void *const a)
  {
    return _mm_broadcast_ss((float *)a);
  }
#  else
  static __forceinline ssef broadcast(const void *const a)
  {
    return _mm_set1_ps(*(float *)a);
  }
#  endif

  ////////////////////////////////////////////////////////////////////////////////
  /// Array Access
  ////////////////////////////////////////////////////////////////////////////////

  __forceinline const float &operator[](const size_t i) const
  {
    assert(i < 4);
    return f[i];
  }
  __forceinline float &operator[](const size_t i)
  {
    assert(i < 4);
    return f[i];
  }
};

////////////////////////////////////////////////////////////////////////////////
/// Unary Operators
////////////////////////////////////////////////////////////////////////////////

__forceinline const ssef cast(const __m128i &a)
{
  return _mm_castsi128_ps(a);
}
__forceinline const ssef operator+(const ssef &a)
{
  return a;
}
__forceinline const ssef operator-(const ssef &a)
{
  return _mm_xor_ps(a.m128, _mm_castsi128_ps(_mm_set1_epi32(0x80000000)));
}
__forceinline const ssef abs(const ssef &a)
{
  return _mm_and_ps(a.m128, _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff)));
}
#  if defined(__KERNEL_SSE41__)
__forceinline const ssef sign(const ssef &a)
{
  return _mm_blendv_ps(ssef(1.0f), -ssef(1.0f), _mm_cmplt_ps(a, ssef(0.0f)));
}
#  endif
__forceinline const ssef signmsk(const ssef &a)
{
  return _mm_and_ps(a.m128, _mm_castsi128_ps(_mm_set1_epi32(0x80000000)));
}

__forceinline const ssef rcp(const ssef &a)
{
  const ssef r = _mm_rcp_ps(a.m128);
  return _mm_sub_ps(_mm_add_ps(r, r), _mm_mul_ps(_mm_mul_ps(r, r), a));
}
__forceinline const ssef sqr(const ssef &a)
{
  return _mm_mul_ps(a, a);
}
__forceinline const ssef mm_sqrt(const ssef &a)
{
  return _mm_sqrt_ps(a.m128);
}
__forceinline const ssef rsqrt(const ssef &a)
{
  const ssef r = _mm_rsqrt_ps(a.m128);
  return _mm_add_ps(
      _mm_mul_ps(_mm_set_ps(1.5f, 1.5f, 1.5f, 1.5f), r),
      _mm_mul_ps(_mm_mul_ps(_mm_mul_ps(a, _mm_set_ps(-0.5f, -0.5f, -0.5f, -0.5f)), r),
                 _mm_mul_ps(r, r)));
}

////////////////////////////////////////////////////////////////////////////////
/// Binary Operators
////////////////////////////////////////////////////////////////////////////////

__forceinline const ssef operator+(const ssef &a, const ssef &b)
{
  return _mm_add_ps(a.m128, b.m128);
}
__forceinline const ssef operator+(const ssef &a, const float &b)
{
  return a + ssef(b);
}
__forceinline const ssef operator+(const float &a, const ssef &b)
{
  return ssef(a) + b;
}

__forceinline const ssef operator-(const ssef &a, const ssef &b)
{
  return _mm_sub_ps(a.m128, b.m128);
}
__forceinline const ssef operator-(const ssef &a, const float &b)
{
  return a - ssef(b);
}
__forceinline const ssef operator-(const float &a, const ssef &b)
{
  return ssef(a) - b;
}

__forceinline const ssef operator*(const ssef &a, const ssef &b)
{
  return _mm_mul_ps(a.m128, b.m128);
}
__forceinline const ssef operator*(const ssef &a, const float &b)
{
  return a * ssef(b);
}
__forceinline const ssef operator*(const float &a, const ssef &b)
{
  return ssef(a) * b;
}

__forceinline const ssef operator/(const ssef &a, const ssef &b)
{
  return _mm_div_ps(a.m128, b.m128);
}
__forceinline const ssef operator/(const ssef &a, const float &b)
{
  return a / ssef(b);
}
__forceinline const ssef operator/(const float &a, const ssef &b)
{
  return ssef(a) / b;
}

__forceinline const ssef operator^(const ssef &a, const ssef &b)
{
  return _mm_xor_ps(a.m128, b.m128);
}
__forceinline const ssef operator^(const ssef &a, const ssei &b)
{
  return _mm_xor_ps(a.m128, _mm_castsi128_ps(b.m128));
}

__forceinline const ssef operator&(const ssef &a, const ssef &b)
{
  return _mm_and_ps(a.m128, b.m128);
}
__forceinline const ssef operator&(const ssef &a, const ssei &b)
{
  return _mm_and_ps(a.m128, _mm_castsi128_ps(b.m128));
}

__forceinline const ssef operator|(const ssef &a, const ssef &b)
{
  return _mm_or_ps(a.m128, b.m128);
}
__forceinline const ssef operator|(const ssef &a, const ssei &b)
{
  return _mm_or_ps(a.m128, _mm_castsi128_ps(b.m128));
}

__forceinline const ssef andnot(const ssef &a, const ssef &b)
{
  return _mm_andnot_ps(a.m128, b.m128);
}

__forceinline const ssef min(const ssef &a, const ssef &b)
{
  return _mm_min_ps(a.m128, b.m128);
}
__forceinline const ssef min(const ssef &a, const float &b)
{
  return _mm_min_ps(a.m128, ssef(b));
}
__forceinline const ssef min(const float &a, const ssef &b)
{
  return _mm_min_ps(ssef(a), b.m128);
}

__forceinline const ssef max(const ssef &a, const ssef &b)
{
  return _mm_max_ps(a.m128, b.m128);
}
__forceinline const ssef max(const ssef &a, const float &b)
{
  return _mm_max_ps(a.m128, ssef(b));
}
__forceinline const ssef max(const float &a, const ssef &b)
{
  return _mm_max_ps(ssef(a), b.m128);
}

#  if defined(__KERNEL_SSE41__)
__forceinline ssef mini(const ssef &a, const ssef &b)
{
  const ssei ai = _mm_castps_si128(a);
  const ssei bi = _mm_castps_si128(b);
  const ssei ci = _mm_min_epi32(ai, bi);
  return _mm_castsi128_ps(ci);
}
#  endif

#  if defined(__KERNEL_SSE41__)
__forceinline ssef maxi(const ssef &a, const ssef &b)
{
  const ssei ai = _mm_castps_si128(a);
  const ssei bi = _mm_castps_si128(b);
  const ssei ci = _mm_max_epi32(ai, bi);
  return _mm_castsi128_ps(ci);
}
#  endif

////////////////////////////////////////////////////////////////////////////////
/// Ternary Operators
////////////////////////////////////////////////////////////////////////////////

__forceinline const ssef madd(const ssef &a, const ssef &b, const ssef &c)
{
#  if defined(__KERNEL_NEON__)
  return vfmaq_f32(c, a, b);
#  elif defined(__KERNEL_AVX2__)
  return _mm_fmadd_ps(a, b, c);
#  else
  return a * b + c;
#  endif
}
__forceinline const ssef msub(const ssef &a, const ssef &b, const ssef &c)
{
#  if defined(__KERNEL_NEON__)
  return vfmaq_f32(vnegq_f32(c), a, b);
#  elif defined(__KERNEL_AVX2__)
  return _mm_fmsub_ps(a, b, c);
#  else
  return a * b - c;
#  endif
}
__forceinline const ssef nmadd(const ssef &a, const ssef &b, const ssef &c)
{
#  if defined(__KERNEL_NEON__)
  return vfmsq_f32(c, a, b);
#  elif defined(__KERNEL_AVX2__)
  return _mm_fnmadd_ps(a, b, c);
#  else
  return c - a * b;
#  endif
}
__forceinline const ssef nmsub(const ssef &a, const ssef &b, const ssef &c)
{
#  if defined(__KERNEL_NEON__)
  return vfmsq_f32(vnegq_f32(c), a, b);
#  elif defined(__KERNEL_AVX2__)
  return _mm_fnmsub_ps(a, b, c);
#  else
  return -a * b - c;
#  endif
}

////////////////////////////////////////////////////////////////////////////////
/// Assignment Operators
////////////////////////////////////////////////////////////////////////////////

__forceinline ssef &operator+=(ssef &a, const ssef &b)
{
  return a = a + b;
}
__forceinline ssef &operator+=(ssef &a, const float &b)
{
  return a = a + b;
}

__forceinline ssef &operator-=(ssef &a, const ssef &b)
{
  return a = a - b;
}
__forceinline ssef &operator-=(ssef &a, const float &b)
{
  return a = a - b;
}

__forceinline ssef &operator*=(ssef &a, const ssef &b)
{
  return a = a * b;
}
__forceinline ssef &operator*=(ssef &a, const float &b)
{
  return a = a * b;
}

__forceinline ssef &operator/=(ssef &a, const ssef &b)
{
  return a = a / b;
}
__forceinline ssef &operator/=(ssef &a, const float &b)
{
  return a = a / b;
}

////////////////////////////////////////////////////////////////////////////////
/// Comparison Operators + Select
////////////////////////////////////////////////////////////////////////////////

__forceinline const sseb operator==(const ssef &a, const ssef &b)
{
  return _mm_cmpeq_ps(a.m128, b.m128);
}
__forceinline const sseb operator==(const ssef &a, const float &b)
{
  return a == ssef(b);
}
__forceinline const sseb operator==(const float &a, const ssef &b)
{
  return ssef(a) == b;
}

__forceinline const sseb operator!=(const ssef &a, const ssef &b)
{
  return _mm_cmpneq_ps(a.m128, b.m128);
}
__forceinline const sseb operator!=(const ssef &a, const float &b)
{
  return a != ssef(b);
}
__forceinline const sseb operator!=(const float &a, const ssef &b)
{
  return ssef(a) != b;
}

__forceinline const sseb operator<(const ssef &a, const ssef &b)
{
  return _mm_cmplt_ps(a.m128, b.m128);
}
__forceinline const sseb operator<(const ssef &a, const float &b)
{
  return a < ssef(b);
}
__forceinline const sseb operator<(const float &a, const ssef &b)
{
  return ssef(a) < b;
}

__forceinline const sseb operator>=(const ssef &a, const ssef &b)
{
  return _mm_cmpnlt_ps(a.m128, b.m128);
}
__forceinline const sseb operator>=(const ssef &a, const float &b)
{
  return a >= ssef(b);
}
__forceinline const sseb operator>=(const float &a, const ssef &b)
{
  return ssef(a) >= b;
}

__forceinline const sseb operator>(const ssef &a, const ssef &b)
{
  return _mm_cmpnle_ps(a.m128, b.m128);
}
__forceinline const sseb operator>(const ssef &a, const float &b)
{
  return a > ssef(b);
}
__forceinline const sseb operator>(const float &a, const ssef &b)
{
  return ssef(a) > b;
}

__forceinline const sseb operator<=(const ssef &a, const ssef &b)
{
  return _mm_cmple_ps(a.m128, b.m128);
}
__forceinline const sseb operator<=(const ssef &a, const float &b)
{
  return a <= ssef(b);
}
__forceinline const sseb operator<=(const float &a, const ssef &b)
{
  return ssef(a) <= b;
}

__forceinline const ssef select(const sseb &m, const ssef &t, const ssef &f)
{
#  ifdef __KERNEL_SSE41__
  return _mm_blendv_ps(f, t, m);
#  else
  return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f));
#  endif
}

__forceinline const ssef select(const ssef &m, const ssef &t, const ssef &f)
{
#  ifdef __KERNEL_SSE41__
  return _mm_blendv_ps(f, t, m);
#  else
  return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f));
#  endif
}

__forceinline const ssef select(const int mask, const ssef &t, const ssef &f)
{
#  if defined(__KERNEL_SSE41__) && \
      ((!defined(__clang__) && !defined(_MSC_VER)) || defined(__INTEL_COMPILER))
  return _mm_blend_ps(f, t, mask);
#  else
  return select(sseb(mask), t, f);
#  endif
}

////////////////////////////////////////////////////////////////////////////////
/// Rounding Functions
////////////////////////////////////////////////////////////////////////////////

#  if defined(__KERNEL_SSE41__)
__forceinline const ssef round_even(const ssef &a)
{
#    ifdef __KERNEL_NEON__
  return vrndnq_f32(a);
#    else
  return _mm_round_ps(a, _MM_FROUND_TO_NEAREST_INT);
#    endif
}
__forceinline const ssef round_down(const ssef &a)
{
#    ifdef __KERNEL_NEON__
  return vrndmq_f32(a);
#    else
  return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF);
#    endif
}
__forceinline const ssef round_up(const ssef &a)
{
#    ifdef __KERNEL_NEON__
  return vrndpq_f32(a);
#    else
  return _mm_round_ps(a, _MM_FROUND_TO_POS_INF);
#    endif
}
__forceinline const ssef round_zero(const ssef &a)
{
#    ifdef __KERNEL_NEON__
  return vrndq_f32(a);
#    else
  return _mm_round_ps(a, _MM_FROUND_TO_ZERO);
#    endif
}
__forceinline const ssef floor(const ssef &a)
{
#    ifdef __KERNEL_NEON__
  return vrndmq_f32(a);
#    else
  return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF);
#    endif
}
__forceinline const ssef ceil(const ssef &a)
{
#    ifdef __KERNEL_NEON__
  return vrndpq_f32(a);
#    else
  return _mm_round_ps(a, _MM_FROUND_TO_POS_INF);
#    endif
}
#  else
/* Non-SSE4.1 fallback, needed for floorfrac. */
__forceinline const ssef floor(const ssef &a)
{
  return _mm_set_ps(floorf(a.f[3]), floorf(a.f[2]), floorf(a.f[1]), floorf(a.f[0]));
}
#  endif

__forceinline ssei truncatei(const ssef &a)
{
  return _mm_cvttps_epi32(a.m128);
}

__forceinline ssef floorfrac(const ssef &x, ssei *i)
{
  ssef f = floor(x);
  *i = truncatei(f);
  return x - f;
}

////////////////////////////////////////////////////////////////////////////////
/// Common Functions
////////////////////////////////////////////////////////////////////////////////

__forceinline ssef mix(const ssef &a, const ssef &b, const ssef &t)
{
  return madd(t, b, (ssef(1.0f) - t) * a);
}

////////////////////////////////////////////////////////////////////////////////
/// Movement/Shifting/Shuffling Functions
////////////////////////////////////////////////////////////////////////////////

__forceinline ssef unpacklo(const ssef &a, const ssef &b)
{
  return _mm_unpacklo_ps(a.m128, b.m128);
}
__forceinline ssef unpackhi(const ssef &a, const ssef &b)
{
  return _mm_unpackhi_ps(a.m128, b.m128);
}

template<size_t i0, size_t i1, size_t i2, size_t i3>
__forceinline const ssef shuffle(const ssef &b)
{
#  ifdef __KERNEL_NEON__
  return shuffle_neon<float32x4_t, i0, i1, i2, i3>(b.m128);
#  else
  return _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(b), _MM_SHUFFLE(i3, i2, i1, i0)));
#  endif
}

template<> __forceinline const ssef shuffle<0, 1, 0, 1>(const ssef &a)
{
  return _mm_movelh_ps(a, a);
}

template<> __forceinline const ssef shuffle<2, 3, 2, 3>(const ssef &a)
{
  return _mm_movehl_ps(a, a);
}

template<size_t i0, size_t i1, size_t i2, size_t i3>
__forceinline const ssef shuffle(const ssef &a, const ssef &b)
{
#  ifdef __KERNEL_NEON__
  return shuffle_neon<float32x4_t, i0, i1, i2, i3>(a, b);
#  else
  return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0));
#  endif
}

template<size_t i0> __forceinline const ssef shuffle(const ssef &a, const ssef &b)
{
#  ifdef __KERNEL_NEON__
  return shuffle_neon<float32x4_t, i0, i0, i0, i0>(a, b);
#  else
  return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i0, i0, i0, i0));
#  endif
}

#  ifndef __KERNEL_NEON__
template<> __forceinline const ssef shuffle<0, 1, 0, 1>(const ssef &a, const ssef &b)
{
  return _mm_movelh_ps(a, b);
}

template<> __forceinline const ssef shuffle<2, 3, 2, 3>(const ssef &a, const ssef &b)
{
  return _mm_movehl_ps(b, a);
}
#  endif

#  if defined(__KERNEL_SSSE3__)
__forceinline const ssef shuffle8(const ssef &a, const ssei &shuf)
{
  return _mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(a), shuf));
}
#  endif

#  if defined(__KERNEL_SSE3__)
template<> __forceinline const ssef shuffle<0, 0, 2, 2>(const ssef &b)
{
  return _mm_moveldup_ps(b);
}
template<> __forceinline const ssef shuffle<1, 1, 3, 3>(const ssef &b)
{
  return _mm_movehdup_ps(b);
}
#  endif

template<size_t i0> __forceinline const ssef shuffle(const ssef &b)
{
  return shuffle<i0, i0, i0, i0>(b);
}

#  if defined(__KERNEL_AVX__)
__forceinline const ssef shuffle(const ssef &a, const ssei &shuf)
{
  return _mm_permutevar_ps(a, shuf);
}
#  endif

template<size_t i> __forceinline float extract(const ssef &a)
{
  return _mm_cvtss_f32(shuffle<i, i, i, i>(a));
}
template<> __forceinline float extract<0>(const ssef &a)
{
  return _mm_cvtss_f32(a);
}

#  if defined(__KERNEL_SSE41__)
template<size_t dst, size_t src, size_t clr>
__forceinline const ssef insert(const ssef &a, const ssef &b)
{
#    ifdef __KERNEL_NEON__
  ssef res = a;
  if (clr)
    res[dst] = 0;
  else
    res[dst] = b[src];
  return res;
#    else
  return _mm_insert_ps(a, b, (dst << 4) | (src << 6) | clr);
#    endif
}
template<size_t dst, size_t src> __forceinline const ssef insert(const ssef &a, const ssef &b)
{
  return insert<dst, src, 0>(a, b);
}
template<size_t dst> __forceinline const ssef insert(const ssef &a, const float b)
{
  return insert<dst, 0>(a, _mm_set_ss(b));
}
#  else
template<size_t dst> __forceinline const ssef insert(const ssef &a, const float b)
{
  ssef c = a;
  c[dst] = b;
  return c;
}
#  endif

////////////////////////////////////////////////////////////////////////////////
/// Transpose
////////////////////////////////////////////////////////////////////////////////

__forceinline void transpose(const ssef &r0,
                             const ssef &r1,
                             const ssef &r2,
                             const ssef &r3,
                             ssef &c0,
                             ssef &c1,
                             ssef &c2,
                             ssef &c3)
{
  ssef l02 = unpacklo(r0, r2);
  ssef h02 = unpackhi(r0, r2);
  ssef l13 = unpacklo(r1, r3);
  ssef h13 = unpackhi(r1, r3);
  c0 = unpacklo(l02, l13);
  c1 = unpackhi(l02, l13);
  c2 = unpacklo(h02, h13);
  c3 = unpackhi(h02, h13);
}

__forceinline void transpose(
    const ssef &r0, const ssef &r1, const ssef &r2, const ssef &r3, ssef &c0, ssef &c1, ssef &c2)
{
  ssef l02 = unpacklo(r0, r2);
  ssef h02 = unpackhi(r0, r2);
  ssef l13 = unpacklo(r1, r3);
  ssef h13 = unpackhi(r1, r3);
  c0 = unpacklo(l02, l13);
  c1 = unpackhi(l02, l13);
  c2 = unpacklo(h02, h13);
}

////////////////////////////////////////////////////////////////////////////////
/// Reductions
////////////////////////////////////////////////////////////////////////////////

__forceinline const ssef vreduce_min(const ssef &v)
{
#  ifdef __KERNEL_NEON__
  return vdupq_n_f32(vminvq_f32(v));
#  else
  ssef h = min(shuffle<1, 0, 3, 2>(v), v);
  return min(shuffle<2, 3, 0, 1>(h), h);
#  endif
}
__forceinline const ssef vreduce_max(const ssef &v)
{
#  ifdef __KERNEL_NEON__
  return vdupq_n_f32(vmaxvq_f32(v));
#  else
  ssef h = max(shuffle<1, 0, 3, 2>(v), v);
  return max(shuffle<2, 3, 0, 1>(h), h);
#  endif
}
__forceinline const ssef vreduce_add(const ssef &v)
{
#  ifdef __KERNEL_NEON__
  return vdupq_n_f32(vaddvq_f32(v));
#  else
  ssef h = shuffle<1, 0, 3, 2>(v) + v;
  return shuffle<2, 3, 0, 1>(h) + h;
#  endif
}

__forceinline float reduce_min(const ssef &v)
{
#  ifdef __KERNEL_NEON__
  return vminvq_f32(v);
#  else
  return _mm_cvtss_f32(vreduce_min(v));
#  endif
}
__forceinline float reduce_max(const ssef &v)
{
#  ifdef __KERNEL_NEON__
  return vmaxvq_f32(v);
#  else
  return _mm_cvtss_f32(vreduce_max(v));
#  endif
}
__forceinline float reduce_add(const ssef &v)
{
#  ifdef __KERNEL_NEON__
  return vaddvq_f32(v);
#  else
  return _mm_cvtss_f32(vreduce_add(v));
#  endif
}

__forceinline uint32_t select_min(const ssef &v)
{
  return __bsf(movemask(v == vreduce_min(v)));
}
__forceinline uint32_t select_max(const ssef &v)
{
  return __bsf(movemask(v == vreduce_max(v)));
}

__forceinline uint32_t select_min(const sseb &valid, const ssef &v)
{
  const ssef a = select(valid, v, ssef(pos_inf));
  return __bsf(movemask(valid & (a == vreduce_min(a))));
}
__forceinline uint32_t select_max(const sseb &valid, const ssef &v)
{
  const ssef a = select(valid, v, ssef(neg_inf));
  return __bsf(movemask(valid & (a == vreduce_max(a))));
}

__forceinline uint32_t movemask(const ssef &a)
{
  return _mm_movemask_ps(a);
}

////////////////////////////////////////////////////////////////////////////////
/// Memory load and store operations
////////////////////////////////////////////////////////////////////////////////

__forceinline ssef load4f(const float4 &a)
{
#  ifdef __KERNEL_WITH_SSE_ALIGN__
  return _mm_load_ps(&a.x);
#  else
  return _mm_loadu_ps(&a.x);
#  endif
}

__forceinline ssef load4f(const float3 &a)
{
#  ifdef __KERNEL_WITH_SSE_ALIGN__
  return _mm_load_ps(&a.x);
#  else
  return _mm_loadu_ps(&a.x);
#  endif
}

__forceinline ssef load4f(const void *const a)
{
  return _mm_load_ps((float *)a);
}

__forceinline ssef load1f_first(const float a)
{
  return _mm_set_ss(a);
}

__forceinline void store4f(void *ptr, const ssef &v)
{
  _mm_store_ps((float *)ptr, v);
}

__forceinline ssef loadu4f(const void *const a)
{
  return _mm_loadu_ps((float *)a);
}

__forceinline void storeu4f(void *ptr, const ssef &v)
{
  _mm_storeu_ps((float *)ptr, v);
}

__forceinline void store4f(const sseb &mask, void *ptr, const ssef &f)
{
#  if defined(__KERNEL_AVX__)
  _mm_maskstore_ps((float *)ptr, (__m128i)mask, f);
#  else
  *(ssef *)ptr = select(mask, f, *(ssef *)ptr);
#  endif
}

__forceinline ssef load4f_nt(void *ptr)
{
#  if defined(__KERNEL_SSE41__)
  return _mm_castsi128_ps(_mm_stream_load_si128((__m128i *)ptr));
#  else
  return _mm_load_ps((float *)ptr);
#  endif
}

__forceinline void store4f_nt(void *ptr, const ssef &v)
{
#  if defined(__KERNEL_SSE41__)
  _mm_stream_ps((float *)ptr, v);
#  else
  _mm_store_ps((float *)ptr, v);
#  endif
}

////////////////////////////////////////////////////////////////////////////////
/// Euclidean Space Operators
////////////////////////////////////////////////////////////////////////////////

__forceinline float dot(const ssef &a, const ssef &b)
{
  return reduce_add(a * b);
}

/* calculate shuffled cross product, useful when order of components does not matter */
__forceinline ssef cross_zxy(const ssef &a, const ssef &b)
{
  const ssef a0 = a;
  const ssef b0 = shuffle<1, 2, 0, 3>(b);
  const ssef a1 = shuffle<1, 2, 0, 3>(a);
  const ssef b1 = b;
  return msub(a0, b0, a1 * b1);
}

__forceinline ssef cross(const ssef &a, const ssef &b)
{
  return shuffle<1, 2, 0, 3>(cross_zxy(a, b));
}

ccl_device_inline const ssef dot3_splat(const ssef &a, const ssef &b)
{
#  ifdef __KERNEL_SSE41__
  return _mm_dp_ps(a.m128, b.m128, 0x7f);
#  else
  ssef t = a * b;
  return ssef(((float *)&t)[0] + ((float *)&t)[1] + ((float *)&t)[2]);
#  endif
}

/* squared length taking only specified axes into account */
template<size_t X, size_t Y, size_t Z, size_t W> ccl_device_inline float len_squared(const ssef &a)
{
#  ifndef __KERNEL_SSE41__
  float4 &t = (float4 &)a;
  return (X ? t.x * t.x : 0.0f) + (Y ? t.y * t.y : 0.0f) + (Z ? t.z * t.z : 0.0f) +
         (W ? t.w * t.w : 0.0f);
#  else
  return extract<0>(
      ssef(_mm_dp_ps(a.m128, a.m128, (X << 4) | (Y << 5) | (Z << 6) | (W << 7) | 0xf)));
#  endif
}

ccl_device_inline float dot3(const ssef &a, const ssef &b)
{
#  ifdef __KERNEL_SSE41__
  return extract<0>(ssef(_mm_dp_ps(a.m128, b.m128, 0x7f)));
#  else
  ssef t = a * b;
  return ((float *)&t)[0] + ((float *)&t)[1] + ((float *)&t)[2];
#  endif
}

ccl_device_inline const ssef len3_squared_splat(const ssef &a)
{
  return dot3_splat(a, a);
}

ccl_device_inline float len3_squared(const ssef &a)
{
  return dot3(a, a);
}

ccl_device_inline float len3(const ssef &a)
{
  return extract<0>(mm_sqrt(dot3_splat(a, a)));
}

/* SSE shuffle utility functions */

#  ifdef __KERNEL_SSSE3__

/* faster version for SSSE3 */
typedef ssei shuffle_swap_t;

ccl_device_inline shuffle_swap_t shuffle_swap_identity()
{
  return _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
}

ccl_device_inline shuffle_swap_t shuffle_swap_swap()
{
  return _mm_set_epi8(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8);
}

ccl_device_inline const ssef shuffle_swap(const ssef &a, const shuffle_swap_t &shuf)
{
  return cast(_mm_shuffle_epi8(cast(a), shuf));
}

#  else

/* somewhat slower version for SSE2 */
typedef int shuffle_swap_t;

ccl_device_inline shuffle_swap_t shuffle_swap_identity()
{
  return 0;
}

ccl_device_inline shuffle_swap_t shuffle_swap_swap()
{
  return 1;
}

ccl_device_inline const ssef shuffle_swap(const ssef &a, shuffle_swap_t shuf)
{
  /* shuffle value must be a constant, so we need to branch */
  if (shuf)
    return shuffle<1, 0, 3, 2>(a);
  else
    return shuffle<3, 2, 1, 0>(a);
}

#  endif

#  if defined(__KERNEL_SSE41__) && !defined(__KERNEL_NEON__)

ccl_device_inline void gen_idirsplat_swap(const ssef &pn,
                                          const shuffle_swap_t &shuf_identity,
                                          const shuffle_swap_t &shuf_swap,
                                          const float3 &idir,
                                          ssef idirsplat[3],
                                          shuffle_swap_t shufflexyz[3])
{
  const __m128 idirsplat_raw[] = {_mm_set_ps1(idir.x), _mm_set_ps1(idir.y), _mm_set_ps1(idir.z)};
  idirsplat[0] = _mm_xor_ps(idirsplat_raw[0], pn);
  idirsplat[1] = _mm_xor_ps(idirsplat_raw[1], pn);
  idirsplat[2] = _mm_xor_ps(idirsplat_raw[2], pn);

  const ssef signmask = cast(ssei(0x80000000));
  const ssef shuf_identity_f = cast(shuf_identity);
  const ssef shuf_swap_f = cast(shuf_swap);

  shufflexyz[0] = _mm_castps_si128(
      _mm_blendv_ps(shuf_identity_f, shuf_swap_f, _mm_and_ps(idirsplat_raw[0], signmask)));
  shufflexyz[1] = _mm_castps_si128(
      _mm_blendv_ps(shuf_identity_f, shuf_swap_f, _mm_and_ps(idirsplat_raw[1], signmask)));
  shufflexyz[2] = _mm_castps_si128(
      _mm_blendv_ps(shuf_identity_f, shuf_swap_f, _mm_and_ps(idirsplat_raw[2], signmask)));
}

#  else

ccl_device_inline void gen_idirsplat_swap(const ssef &pn,
                                          const shuffle_swap_t &shuf_identity,
                                          const shuffle_swap_t &shuf_swap,
                                          const float3 &idir,
                                          ssef idirsplat[3],
                                          shuffle_swap_t shufflexyz[3])
{
  idirsplat[0] = ssef(idir.x) ^ pn;
  idirsplat[1] = ssef(idir.y) ^ pn;
  idirsplat[2] = ssef(idir.z) ^ pn;

  shufflexyz[0] = (idir.x >= 0) ? shuf_identity : shuf_swap;
  shufflexyz[1] = (idir.y >= 0) ? shuf_identity : shuf_swap;
  shufflexyz[2] = (idir.z >= 0) ? shuf_identity : shuf_swap;
}

#  endif

ccl_device_inline const ssef uint32_to_float(const ssei &in)
{
  ssei a = _mm_srli_epi32(in, 16);
  ssei b = _mm_and_si128(in, _mm_set1_epi32(0x0000ffff));
  ssei c = _mm_or_si128(a, _mm_set1_epi32(0x53000000));
  ssef d = _mm_cvtepi32_ps(b);
  ssef e = _mm_sub_ps(_mm_castsi128_ps(c), _mm_castsi128_ps(_mm_set1_epi32(0x53000000)));
  return _mm_add_ps(e, d);
}

template<size_t S1, size_t S2, size_t S3, size_t S4>
ccl_device_inline const ssef set_sign_bit(const ssef &a)
{
  return cast(cast(a) ^ ssei(S1 << 31, S2 << 31, S3 << 31, S4 << 31));
}

////////////////////////////////////////////////////////////////////////////////
/// Debug Functions
////////////////////////////////////////////////////////////////////////////////

ccl_device_inline void print_ssef(const char *label, const ssef &a)
{
  printf(
      "%s: %.8f %.8f %.8f %.8f\n", label, (double)a[0], (double)a[1], (double)a[2], (double)a[3]);
}

#endif

CCL_NAMESPACE_END

#endif