From e1b3d9112730bc3b569ffff732a1558752ded146 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Tue, 1 Nov 2022 15:16:55 +0100 Subject: Refactor: replace Cycles sse/avx types by vectorized float4/int4/float8/int8 The distinction existed for legacy reasons, to easily port of Embree intersection code without affecting the main vector types. However we are now using SIMD for these types as well, so no good reason to keep the distinction. Also more consistently pass these vector types by value in inline functions. Previously it was partially changed for functions used by Metal to avoid having to add address space qualifiers, simple to do it everywhere. Also removes function declarations for vector math headers, serves no real purpose. Differential Revision: https://developer.blender.org/D16146 --- intern/cycles/util/ssef.h | 1090 --------------------------------------------- 1 file changed, 1090 deletions(-) delete mode 100644 intern/cycles/util/ssef.h (limited to 'intern/cycles/util/ssef.h') diff --git a/intern/cycles/util/ssef.h b/intern/cycles/util/ssef.h deleted file mode 100644 index 1e2bfa90354..00000000000 --- a/intern/cycles/util/ssef.h +++ /dev/null @@ -1,1090 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 - * Copyright 2011-2013 Intel Corporation - * Modifications Copyright 2014-2022 Blender Foundation. */ - -#ifndef __UTIL_SSEF_H__ -#define __UTIL_SSEF_H__ - -#include - -#include "util/ssei.h" - -CCL_NAMESPACE_BEGIN - -#ifdef __KERNEL_SSE2__ - -struct sseb; -struct ssef; - -/*! 4-wide SSE float type. */ -struct ssef { - typedef sseb Mask; // mask type - typedef ssei Int; // int type - typedef ssef Float; // float type - - enum { size = 4 }; // number of SIMD elements - union { - __m128 m128; - float f[4]; - int i[4]; - }; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline ssef() - { - } - __forceinline ssef(const ssef &other) - { - m128 = other.m128; - } - __forceinline ssef &operator=(const ssef &other) - { - m128 = other.m128; - return *this; - } - - __forceinline ssef(const __m128 a) : m128(a) - { - } - __forceinline operator const __m128 &() const - { - return m128; - } - __forceinline operator __m128 &() - { - return m128; - } - - __forceinline ssef(float a) : m128(_mm_set1_ps(a)) - { - } - __forceinline ssef(float a, float b, float c, float d) : m128(_mm_setr_ps(a, b, c, d)) - { - } - - __forceinline explicit ssef(const __m128i a) : m128(_mm_cvtepi32_ps(a)) - { - } - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - -# if defined(__KERNEL_AVX__) - static __forceinline ssef broadcast(const void *const a) - { - return _mm_broadcast_ss((float *)a); - } -# else - static __forceinline ssef broadcast(const void *const a) - { - return _mm_set1_ps(*(float *)a); - } -# endif - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const float &operator[](const size_t i) const - { - assert(i < 4); - return f[i]; - } - __forceinline float &operator[](const size_t i) - { - assert(i < 4); - return f[i]; - } -}; - -//////////////////////////////////////////////////////////////////////////////// -/// Unary Operators -//////////////////////////////////////////////////////////////////////////////// - -__forceinline const ssef cast(const __m128i &a) -{ - return _mm_castsi128_ps(a); -} -__forceinline const ssef operator+(const ssef &a) -{ - return a; -} -__forceinline const ssef operator-(const ssef &a) -{ - return _mm_xor_ps(a.m128, _mm_castsi128_ps(_mm_set1_epi32(0x80000000))); -} -__forceinline const ssef abs(const ssef &a) -{ - return _mm_and_ps(a.m128, _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff))); -} -# if defined(__KERNEL_SSE41__) -__forceinline const ssef sign(const ssef &a) -{ - return _mm_blendv_ps(ssef(1.0f), -ssef(1.0f), _mm_cmplt_ps(a, ssef(0.0f))); -} -# endif -__forceinline const ssef signmsk(const ssef &a) -{ - return _mm_and_ps(a.m128, _mm_castsi128_ps(_mm_set1_epi32(0x80000000))); -} - -__forceinline const ssef rcp(const ssef &a) -{ - const ssef r = _mm_rcp_ps(a.m128); - return _mm_sub_ps(_mm_add_ps(r, r), _mm_mul_ps(_mm_mul_ps(r, r), a)); -} -__forceinline const ssef sqr(const ssef &a) -{ - return _mm_mul_ps(a, a); -} -__forceinline const ssef mm_sqrt(const ssef &a) -{ - return _mm_sqrt_ps(a.m128); -} -__forceinline const ssef rsqrt(const ssef &a) -{ - const ssef r = _mm_rsqrt_ps(a.m128); - return _mm_add_ps( - _mm_mul_ps(_mm_set_ps(1.5f, 1.5f, 1.5f, 1.5f), r), - _mm_mul_ps(_mm_mul_ps(_mm_mul_ps(a, _mm_set_ps(-0.5f, -0.5f, -0.5f, -0.5f)), r), - _mm_mul_ps(r, r))); -} - -//////////////////////////////////////////////////////////////////////////////// -/// Binary Operators -//////////////////////////////////////////////////////////////////////////////// - -__forceinline const ssef operator+(const ssef &a, const ssef &b) -{ - return _mm_add_ps(a.m128, b.m128); -} -__forceinline const ssef operator+(const ssef &a, const float &b) -{ - return a + ssef(b); -} -__forceinline const ssef operator+(const float &a, const ssef &b) -{ - return ssef(a) + b; -} - -__forceinline const ssef operator-(const ssef &a, const ssef &b) -{ - return _mm_sub_ps(a.m128, b.m128); -} -__forceinline const ssef operator-(const ssef &a, const float &b) -{ - return a - ssef(b); -} -__forceinline const ssef operator-(const float &a, const ssef &b) -{ - return ssef(a) - b; -} - -__forceinline const ssef operator*(const ssef &a, const ssef &b) -{ - return _mm_mul_ps(a.m128, b.m128); -} -__forceinline const ssef operator*(const ssef &a, const float &b) -{ - return a * ssef(b); -} -__forceinline const ssef operator*(const float &a, const ssef &b) -{ - return ssef(a) * b; -} - -__forceinline const ssef operator/(const ssef &a, const ssef &b) -{ - return _mm_div_ps(a.m128, b.m128); -} -__forceinline const ssef operator/(const ssef &a, const float &b) -{ - return a / ssef(b); -} -__forceinline const ssef operator/(const float &a, const ssef &b) -{ - return ssef(a) / b; -} - -__forceinline const ssef operator^(const ssef &a, const ssef &b) -{ - return _mm_xor_ps(a.m128, b.m128); -} -__forceinline const ssef operator^(const ssef &a, const ssei &b) -{ - return _mm_xor_ps(a.m128, _mm_castsi128_ps(b.m128)); -} - -__forceinline const ssef operator&(const ssef &a, const ssef &b) -{ - return _mm_and_ps(a.m128, b.m128); -} -__forceinline const ssef operator&(const ssef &a, const ssei &b) -{ - return _mm_and_ps(a.m128, _mm_castsi128_ps(b.m128)); -} - -__forceinline const ssef operator|(const ssef &a, const ssef &b) -{ - return _mm_or_ps(a.m128, b.m128); -} -__forceinline const ssef operator|(const ssef &a, const ssei &b) -{ - return _mm_or_ps(a.m128, _mm_castsi128_ps(b.m128)); -} - -__forceinline const ssef andnot(const ssef &a, const ssef &b) -{ - return _mm_andnot_ps(a.m128, b.m128); -} - -__forceinline const ssef min(const ssef &a, const ssef &b) -{ - return _mm_min_ps(a.m128, b.m128); -} -__forceinline const ssef min(const ssef &a, const float &b) -{ - return _mm_min_ps(a.m128, ssef(b)); -} -__forceinline const ssef min(const float &a, const ssef &b) -{ - return _mm_min_ps(ssef(a), b.m128); -} - -__forceinline const ssef max(const ssef &a, const ssef &b) -{ - return _mm_max_ps(a.m128, b.m128); -} -__forceinline const ssef max(const ssef &a, const float &b) -{ - return _mm_max_ps(a.m128, ssef(b)); -} -__forceinline const ssef max(const float &a, const ssef &b) -{ - return _mm_max_ps(ssef(a), b.m128); -} - -# if defined(__KERNEL_SSE41__) -__forceinline ssef mini(const ssef &a, const ssef &b) -{ - const ssei ai = _mm_castps_si128(a); - const ssei bi = _mm_castps_si128(b); - const ssei ci = _mm_min_epi32(ai, bi); - return _mm_castsi128_ps(ci); -} -# endif - -# if defined(__KERNEL_SSE41__) -__forceinline ssef maxi(const ssef &a, const ssef &b) -{ - const ssei ai = _mm_castps_si128(a); - const ssei bi = _mm_castps_si128(b); - const ssei ci = _mm_max_epi32(ai, bi); - return _mm_castsi128_ps(ci); -} -# endif - -//////////////////////////////////////////////////////////////////////////////// -/// Ternary Operators -//////////////////////////////////////////////////////////////////////////////// - -__forceinline const ssef madd(const ssef &a, const ssef &b, const ssef &c) -{ -# if defined(__KERNEL_NEON__) - return vfmaq_f32(c, a, b); -# elif defined(__KERNEL_AVX2__) - return _mm_fmadd_ps(a, b, c); -# else - return a * b + c; -# endif -} -__forceinline const ssef msub(const ssef &a, const ssef &b, const ssef &c) -{ -# if defined(__KERNEL_NEON__) - return vfmaq_f32(vnegq_f32(c), a, b); -# elif defined(__KERNEL_AVX2__) - return _mm_fmsub_ps(a, b, c); -# else - return a * b - c; -# endif -} -__forceinline const ssef nmadd(const ssef &a, const ssef &b, const ssef &c) -{ -# if defined(__KERNEL_NEON__) - return vfmsq_f32(c, a, b); -# elif defined(__KERNEL_AVX2__) - return _mm_fnmadd_ps(a, b, c); -# else - return c - a * b; -# endif -} -__forceinline const ssef nmsub(const ssef &a, const ssef &b, const ssef &c) -{ -# if defined(__KERNEL_NEON__) - return vfmsq_f32(vnegq_f32(c), a, b); -# elif defined(__KERNEL_AVX2__) - return _mm_fnmsub_ps(a, b, c); -# else - return -a * b - c; -# endif -} - -//////////////////////////////////////////////////////////////////////////////// -/// Assignment Operators -//////////////////////////////////////////////////////////////////////////////// - -__forceinline ssef &operator+=(ssef &a, const ssef &b) -{ - return a = a + b; -} -__forceinline ssef &operator+=(ssef &a, const float &b) -{ - return a = a + b; -} - -__forceinline ssef &operator-=(ssef &a, const ssef &b) -{ - return a = a - b; -} -__forceinline ssef &operator-=(ssef &a, const float &b) -{ - return a = a - b; -} - -__forceinline ssef &operator*=(ssef &a, const ssef &b) -{ - return a = a * b; -} -__forceinline ssef &operator*=(ssef &a, const float &b) -{ - return a = a * b; -} - -__forceinline ssef &operator/=(ssef &a, const ssef &b) -{ - return a = a / b; -} -__forceinline ssef &operator/=(ssef &a, const float &b) -{ - return a = a / b; -} - -//////////////////////////////////////////////////////////////////////////////// -/// Comparison Operators + Select -//////////////////////////////////////////////////////////////////////////////// - -__forceinline const sseb operator==(const ssef &a, const ssef &b) -{ - return _mm_cmpeq_ps(a.m128, b.m128); -} -__forceinline const sseb operator==(const ssef &a, const float &b) -{ - return a == ssef(b); -} -__forceinline const sseb operator==(const float &a, const ssef &b) -{ - return ssef(a) == b; -} - -__forceinline const sseb operator!=(const ssef &a, const ssef &b) -{ - return _mm_cmpneq_ps(a.m128, b.m128); -} -__forceinline const sseb operator!=(const ssef &a, const float &b) -{ - return a != ssef(b); -} -__forceinline const sseb operator!=(const float &a, const ssef &b) -{ - return ssef(a) != b; -} - -__forceinline const sseb operator<(const ssef &a, const ssef &b) -{ - return _mm_cmplt_ps(a.m128, b.m128); -} -__forceinline const sseb operator<(const ssef &a, const float &b) -{ - return a < ssef(b); -} -__forceinline const sseb operator<(const float &a, const ssef &b) -{ - return ssef(a) < b; -} - -__forceinline const sseb operator>=(const ssef &a, const ssef &b) -{ - return _mm_cmpnlt_ps(a.m128, b.m128); -} -__forceinline const sseb operator>=(const ssef &a, const float &b) -{ - return a >= ssef(b); -} -__forceinline const sseb operator>=(const float &a, const ssef &b) -{ - return ssef(a) >= b; -} - -__forceinline const sseb operator>(const ssef &a, const ssef &b) -{ - return _mm_cmpnle_ps(a.m128, b.m128); -} -__forceinline const sseb operator>(const ssef &a, const float &b) -{ - return a > ssef(b); -} -__forceinline const sseb operator>(const float &a, const ssef &b) -{ - return ssef(a) > b; -} - -__forceinline const sseb operator<=(const ssef &a, const ssef &b) -{ - return _mm_cmple_ps(a.m128, b.m128); -} -__forceinline const sseb operator<=(const ssef &a, const float &b) -{ - return a <= ssef(b); -} -__forceinline const sseb operator<=(const float &a, const ssef &b) -{ - return ssef(a) <= b; -} - -__forceinline const ssef select(const sseb &m, const ssef &t, const ssef &f) -{ -# ifdef __KERNEL_SSE41__ - return _mm_blendv_ps(f, t, m); -# else - return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f)); -# endif -} - -__forceinline const ssef select(const ssef &m, const ssef &t, const ssef &f) -{ -# ifdef __KERNEL_SSE41__ - return _mm_blendv_ps(f, t, m); -# else - return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f)); -# endif -} - -__forceinline const ssef select(const int mask, const ssef &t, const ssef &f) -{ -# if defined(__KERNEL_SSE41__) && \ - ((!defined(__clang__) && !defined(_MSC_VER)) || defined(__INTEL_COMPILER)) - return _mm_blend_ps(f, t, mask); -# else - return select(sseb(mask), t, f); -# endif -} - -//////////////////////////////////////////////////////////////////////////////// -/// Rounding Functions -//////////////////////////////////////////////////////////////////////////////// - -# if defined(__KERNEL_SSE41__) -__forceinline const ssef round_even(const ssef &a) -{ -# ifdef __KERNEL_NEON__ - return vrndnq_f32(a); -# else - return _mm_round_ps(a, _MM_FROUND_TO_NEAREST_INT); -# endif -} -__forceinline const ssef round_down(const ssef &a) -{ -# ifdef __KERNEL_NEON__ - return vrndmq_f32(a); -# else - return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF); -# endif -} -__forceinline const ssef round_up(const ssef &a) -{ -# ifdef __KERNEL_NEON__ - return vrndpq_f32(a); -# else - return _mm_round_ps(a, _MM_FROUND_TO_POS_INF); -# endif -} -__forceinline const ssef round_zero(const ssef &a) -{ -# ifdef __KERNEL_NEON__ - return vrndq_f32(a); -# else - return _mm_round_ps(a, _MM_FROUND_TO_ZERO); -# endif -} -__forceinline const ssef floor(const ssef &a) -{ -# ifdef __KERNEL_NEON__ - return vrndmq_f32(a); -# else - return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF); -# endif -} -__forceinline const ssef ceil(const ssef &a) -{ -# ifdef __KERNEL_NEON__ - return vrndpq_f32(a); -# else - return _mm_round_ps(a, _MM_FROUND_TO_POS_INF); -# endif -} -# else -/* Non-SSE4.1 fallback, needed for floorfrac. */ -__forceinline const ssef floor(const ssef &a) -{ - return _mm_set_ps(floorf(a.f[3]), floorf(a.f[2]), floorf(a.f[1]), floorf(a.f[0])); -} -# endif - -__forceinline ssei truncatei(const ssef &a) -{ - return _mm_cvttps_epi32(a.m128); -} - -__forceinline ssef floorfrac(const ssef &x, ssei *i) -{ - ssef f = floor(x); - *i = truncatei(f); - return x - f; -} - -//////////////////////////////////////////////////////////////////////////////// -/// Common Functions -//////////////////////////////////////////////////////////////////////////////// - -__forceinline ssef mix(const ssef &a, const ssef &b, const ssef &t) -{ - return madd(t, b, (ssef(1.0f) - t) * a); -} - -//////////////////////////////////////////////////////////////////////////////// -/// Movement/Shifting/Shuffling Functions -//////////////////////////////////////////////////////////////////////////////// - -__forceinline ssef unpacklo(const ssef &a, const ssef &b) -{ - return _mm_unpacklo_ps(a.m128, b.m128); -} -__forceinline ssef unpackhi(const ssef &a, const ssef &b) -{ - return _mm_unpackhi_ps(a.m128, b.m128); -} - -template -__forceinline const ssef shuffle(const ssef &b) -{ -# ifdef __KERNEL_NEON__ - return shuffle_neon(b.m128); -# else - return _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(b), _MM_SHUFFLE(i3, i2, i1, i0))); -# endif -} - -template<> __forceinline const ssef shuffle<0, 1, 0, 1>(const ssef &a) -{ - return _mm_movelh_ps(a, a); -} - -template<> __forceinline const ssef shuffle<2, 3, 2, 3>(const ssef &a) -{ - return _mm_movehl_ps(a, a); -} - -template -__forceinline const ssef shuffle(const ssef &a, const ssef &b) -{ -# ifdef __KERNEL_NEON__ - return shuffle_neon(a, b); -# else - return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0)); -# endif -} - -template __forceinline const ssef shuffle(const ssef &a, const ssef &b) -{ -# ifdef __KERNEL_NEON__ - return shuffle_neon(a, b); -# else - return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i0, i0, i0, i0)); -# endif -} - -# ifndef __KERNEL_NEON__ -template<> __forceinline const ssef shuffle<0, 1, 0, 1>(const ssef &a, const ssef &b) -{ - return _mm_movelh_ps(a, b); -} - -template<> __forceinline const ssef shuffle<2, 3, 2, 3>(const ssef &a, const ssef &b) -{ - return _mm_movehl_ps(b, a); -} -# endif - -# if defined(__KERNEL_SSSE3__) -__forceinline const ssef shuffle8(const ssef &a, const ssei &shuf) -{ - return _mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(a), shuf)); -} -# endif - -# if defined(__KERNEL_SSE3__) -template<> __forceinline const ssef shuffle<0, 0, 2, 2>(const ssef &b) -{ - return _mm_moveldup_ps(b); -} -template<> __forceinline const ssef shuffle<1, 1, 3, 3>(const ssef &b) -{ - return _mm_movehdup_ps(b); -} -# endif - -template __forceinline const ssef shuffle(const ssef &b) -{ - return shuffle(b); -} - -# if defined(__KERNEL_AVX__) -__forceinline const ssef shuffle(const ssef &a, const ssei &shuf) -{ - return _mm_permutevar_ps(a, shuf); -} -# endif - -template __forceinline float extract(const ssef &a) -{ - return _mm_cvtss_f32(shuffle(a)); -} -template<> __forceinline float extract<0>(const ssef &a) -{ - return _mm_cvtss_f32(a); -} - -# if defined(__KERNEL_SSE41__) -template -__forceinline const ssef insert(const ssef &a, const ssef &b) -{ -# ifdef __KERNEL_NEON__ - ssef res = a; - if (clr) - res[dst] = 0; - else - res[dst] = b[src]; - return res; -# else - return _mm_insert_ps(a, b, (dst << 4) | (src << 6) | clr); -# endif -} -template __forceinline const ssef insert(const ssef &a, const ssef &b) -{ - return insert(a, b); -} -template __forceinline const ssef insert(const ssef &a, const float b) -{ - return insert(a, _mm_set_ss(b)); -} -# else -template __forceinline const ssef insert(const ssef &a, const float b) -{ - ssef c = a; - c[dst] = b; - return c; -} -# endif - -//////////////////////////////////////////////////////////////////////////////// -/// Transpose -//////////////////////////////////////////////////////////////////////////////// - -__forceinline void transpose(const ssef &r0, - const ssef &r1, - const ssef &r2, - const ssef &r3, - ssef &c0, - ssef &c1, - ssef &c2, - ssef &c3) -{ - ssef l02 = unpacklo(r0, r2); - ssef h02 = unpackhi(r0, r2); - ssef l13 = unpacklo(r1, r3); - ssef h13 = unpackhi(r1, r3); - c0 = unpacklo(l02, l13); - c1 = unpackhi(l02, l13); - c2 = unpacklo(h02, h13); - c3 = unpackhi(h02, h13); -} - -__forceinline void transpose( - const ssef &r0, const ssef &r1, const ssef &r2, const ssef &r3, ssef &c0, ssef &c1, ssef &c2) -{ - ssef l02 = unpacklo(r0, r2); - ssef h02 = unpackhi(r0, r2); - ssef l13 = unpacklo(r1, r3); - ssef h13 = unpackhi(r1, r3); - c0 = unpacklo(l02, l13); - c1 = unpackhi(l02, l13); - c2 = unpacklo(h02, h13); -} - -//////////////////////////////////////////////////////////////////////////////// -/// Reductions -//////////////////////////////////////////////////////////////////////////////// - -__forceinline const ssef vreduce_min(const ssef &v) -{ -# ifdef __KERNEL_NEON__ - return vdupq_n_f32(vminvq_f32(v)); -# else - ssef h = min(shuffle<1, 0, 3, 2>(v), v); - return min(shuffle<2, 3, 0, 1>(h), h); -# endif -} -__forceinline const ssef vreduce_max(const ssef &v) -{ -# ifdef __KERNEL_NEON__ - return vdupq_n_f32(vmaxvq_f32(v)); -# else - ssef h = max(shuffle<1, 0, 3, 2>(v), v); - return max(shuffle<2, 3, 0, 1>(h), h); -# endif -} -__forceinline const ssef vreduce_add(const ssef &v) -{ -# ifdef __KERNEL_NEON__ - return vdupq_n_f32(vaddvq_f32(v)); -# else - ssef h = shuffle<1, 0, 3, 2>(v) + v; - return shuffle<2, 3, 0, 1>(h) + h; -# endif -} - -__forceinline float reduce_min(const ssef &v) -{ -# ifdef __KERNEL_NEON__ - return vminvq_f32(v); -# else - return _mm_cvtss_f32(vreduce_min(v)); -# endif -} -__forceinline float reduce_max(const ssef &v) -{ -# ifdef __KERNEL_NEON__ - return vmaxvq_f32(v); -# else - return _mm_cvtss_f32(vreduce_max(v)); -# endif -} -__forceinline float reduce_add(const ssef &v) -{ -# ifdef __KERNEL_NEON__ - return vaddvq_f32(v); -# else - return _mm_cvtss_f32(vreduce_add(v)); -# endif -} - -__forceinline uint32_t select_min(const ssef &v) -{ - return __bsf(movemask(v == vreduce_min(v))); -} -__forceinline uint32_t select_max(const ssef &v) -{ - return __bsf(movemask(v == vreduce_max(v))); -} - -__forceinline uint32_t select_min(const sseb &valid, const ssef &v) -{ - const ssef a = select(valid, v, ssef(pos_inf)); - return __bsf(movemask(valid & (a == vreduce_min(a)))); -} -__forceinline uint32_t select_max(const sseb &valid, const ssef &v) -{ - const ssef a = select(valid, v, ssef(neg_inf)); - return __bsf(movemask(valid & (a == vreduce_max(a)))); -} - -__forceinline uint32_t movemask(const ssef &a) -{ - return _mm_movemask_ps(a); -} - -//////////////////////////////////////////////////////////////////////////////// -/// Memory load and store operations -//////////////////////////////////////////////////////////////////////////////// - -__forceinline ssef load4f(const float4 &a) -{ -# ifdef __KERNEL_WITH_SSE_ALIGN__ - return _mm_load_ps(&a.x); -# else - return _mm_loadu_ps(&a.x); -# endif -} - -__forceinline ssef load4f(const float3 &a) -{ -# ifdef __KERNEL_WITH_SSE_ALIGN__ - return _mm_load_ps(&a.x); -# else - return _mm_loadu_ps(&a.x); -# endif -} - -__forceinline ssef load4f(const void *const a) -{ - return _mm_load_ps((float *)a); -} - -__forceinline ssef load1f_first(const float a) -{ - return _mm_set_ss(a); -} - -__forceinline void store4f(void *ptr, const ssef &v) -{ - _mm_store_ps((float *)ptr, v); -} - -__forceinline ssef loadu4f(const void *const a) -{ - return _mm_loadu_ps((float *)a); -} - -__forceinline void storeu4f(void *ptr, const ssef &v) -{ - _mm_storeu_ps((float *)ptr, v); -} - -__forceinline void store4f(const sseb &mask, void *ptr, const ssef &f) -{ -# if defined(__KERNEL_AVX__) - _mm_maskstore_ps((float *)ptr, (__m128i)mask, f); -# else - *(ssef *)ptr = select(mask, f, *(ssef *)ptr); -# endif -} - -__forceinline ssef load4f_nt(void *ptr) -{ -# if defined(__KERNEL_SSE41__) - return _mm_castsi128_ps(_mm_stream_load_si128((__m128i *)ptr)); -# else - return _mm_load_ps((float *)ptr); -# endif -} - -__forceinline void store4f_nt(void *ptr, const ssef &v) -{ -# if defined(__KERNEL_SSE41__) - _mm_stream_ps((float *)ptr, v); -# else - _mm_store_ps((float *)ptr, v); -# endif -} - -//////////////////////////////////////////////////////////////////////////////// -/// Euclidean Space Operators -//////////////////////////////////////////////////////////////////////////////// - -__forceinline float dot(const ssef &a, const ssef &b) -{ - return reduce_add(a * b); -} - -/* calculate shuffled cross product, useful when order of components does not matter */ -__forceinline ssef cross_zxy(const ssef &a, const ssef &b) -{ - const ssef a0 = a; - const ssef b0 = shuffle<1, 2, 0, 3>(b); - const ssef a1 = shuffle<1, 2, 0, 3>(a); - const ssef b1 = b; - return msub(a0, b0, a1 * b1); -} - -__forceinline ssef cross(const ssef &a, const ssef &b) -{ - return shuffle<1, 2, 0, 3>(cross_zxy(a, b)); -} - -ccl_device_inline const ssef dot3_splat(const ssef &a, const ssef &b) -{ -# ifdef __KERNEL_SSE41__ - return _mm_dp_ps(a.m128, b.m128, 0x7f); -# else - ssef t = a * b; - return ssef(((float *)&t)[0] + ((float *)&t)[1] + ((float *)&t)[2]); -# endif -} - -/* squared length taking only specified axes into account */ -template ccl_device_inline float len_squared(const ssef &a) -{ -# ifndef __KERNEL_SSE41__ - float4 &t = (float4 &)a; - return (X ? t.x * t.x : 0.0f) + (Y ? t.y * t.y : 0.0f) + (Z ? t.z * t.z : 0.0f) + - (W ? t.w * t.w : 0.0f); -# else - return extract<0>( - ssef(_mm_dp_ps(a.m128, a.m128, (X << 4) | (Y << 5) | (Z << 6) | (W << 7) | 0xf))); -# endif -} - -ccl_device_inline float dot3(const ssef &a, const ssef &b) -{ -# ifdef __KERNEL_SSE41__ - return extract<0>(ssef(_mm_dp_ps(a.m128, b.m128, 0x7f))); -# else - ssef t = a * b; - return ((float *)&t)[0] + ((float *)&t)[1] + ((float *)&t)[2]; -# endif -} - -ccl_device_inline const ssef len3_squared_splat(const ssef &a) -{ - return dot3_splat(a, a); -} - -ccl_device_inline float len3_squared(const ssef &a) -{ - return dot3(a, a); -} - -ccl_device_inline float len3(const ssef &a) -{ - return extract<0>(mm_sqrt(dot3_splat(a, a))); -} - -/* SSE shuffle utility functions */ - -# ifdef __KERNEL_SSSE3__ - -/* faster version for SSSE3 */ -typedef ssei shuffle_swap_t; - -ccl_device_inline shuffle_swap_t shuffle_swap_identity() -{ - return _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); -} - -ccl_device_inline shuffle_swap_t shuffle_swap_swap() -{ - return _mm_set_epi8(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); -} - -ccl_device_inline const ssef shuffle_swap(const ssef &a, const shuffle_swap_t &shuf) -{ - return cast(_mm_shuffle_epi8(cast(a), shuf)); -} - -# else - -/* somewhat slower version for SSE2 */ -typedef int shuffle_swap_t; - -ccl_device_inline shuffle_swap_t shuffle_swap_identity() -{ - return 0; -} - -ccl_device_inline shuffle_swap_t shuffle_swap_swap() -{ - return 1; -} - -ccl_device_inline const ssef shuffle_swap(const ssef &a, shuffle_swap_t shuf) -{ - /* shuffle value must be a constant, so we need to branch */ - if (shuf) - return shuffle<1, 0, 3, 2>(a); - else - return shuffle<3, 2, 1, 0>(a); -} - -# endif - -# if defined(__KERNEL_SSE41__) && !defined(__KERNEL_NEON__) - -ccl_device_inline void gen_idirsplat_swap(const ssef &pn, - const shuffle_swap_t &shuf_identity, - const shuffle_swap_t &shuf_swap, - const float3 &idir, - ssef idirsplat[3], - shuffle_swap_t shufflexyz[3]) -{ - const __m128 idirsplat_raw[] = {_mm_set_ps1(idir.x), _mm_set_ps1(idir.y), _mm_set_ps1(idir.z)}; - idirsplat[0] = _mm_xor_ps(idirsplat_raw[0], pn); - idirsplat[1] = _mm_xor_ps(idirsplat_raw[1], pn); - idirsplat[2] = _mm_xor_ps(idirsplat_raw[2], pn); - - const ssef signmask = cast(ssei(0x80000000)); - const ssef shuf_identity_f = cast(shuf_identity); - const ssef shuf_swap_f = cast(shuf_swap); - - shufflexyz[0] = _mm_castps_si128( - _mm_blendv_ps(shuf_identity_f, shuf_swap_f, _mm_and_ps(idirsplat_raw[0], signmask))); - shufflexyz[1] = _mm_castps_si128( - _mm_blendv_ps(shuf_identity_f, shuf_swap_f, _mm_and_ps(idirsplat_raw[1], signmask))); - shufflexyz[2] = _mm_castps_si128( - _mm_blendv_ps(shuf_identity_f, shuf_swap_f, _mm_and_ps(idirsplat_raw[2], signmask))); -} - -# else - -ccl_device_inline void gen_idirsplat_swap(const ssef &pn, - const shuffle_swap_t &shuf_identity, - const shuffle_swap_t &shuf_swap, - const float3 &idir, - ssef idirsplat[3], - shuffle_swap_t shufflexyz[3]) -{ - idirsplat[0] = ssef(idir.x) ^ pn; - idirsplat[1] = ssef(idir.y) ^ pn; - idirsplat[2] = ssef(idir.z) ^ pn; - - shufflexyz[0] = (idir.x >= 0) ? shuf_identity : shuf_swap; - shufflexyz[1] = (idir.y >= 0) ? shuf_identity : shuf_swap; - shufflexyz[2] = (idir.z >= 0) ? shuf_identity : shuf_swap; -} - -# endif - -ccl_device_inline const ssef uint32_to_float(const ssei &in) -{ - ssei a = _mm_srli_epi32(in, 16); - ssei b = _mm_and_si128(in, _mm_set1_epi32(0x0000ffff)); - ssei c = _mm_or_si128(a, _mm_set1_epi32(0x53000000)); - ssef d = _mm_cvtepi32_ps(b); - ssef e = _mm_sub_ps(_mm_castsi128_ps(c), _mm_castsi128_ps(_mm_set1_epi32(0x53000000))); - return _mm_add_ps(e, d); -} - -template -ccl_device_inline const ssef set_sign_bit(const ssef &a) -{ - return cast(cast(a) ^ ssei(S1 << 31, S2 << 31, S3 << 31, S4 << 31)); -} - -//////////////////////////////////////////////////////////////////////////////// -/// Debug Functions -//////////////////////////////////////////////////////////////////////////////// - -ccl_device_inline void print_ssef(const char *label, const ssef &a) -{ - printf( - "%s: %.8f %.8f %.8f %.8f\n", label, (double)a[0], (double)a[1], (double)a[2], (double)a[3]); -} - -#endif - -CCL_NAMESPACE_END - -#endif -- cgit v1.2.3