diff options
Diffstat (limited to 'extern/Eigen3/Eigen/src/Core/arch')
| -rw-r--r-- | extern/Eigen3/Eigen/src/Core/arch/AltiVec/PacketMath.h | 3 | ||||
| -rw-r--r-- | extern/Eigen3/Eigen/src/Core/arch/NEON/Complex.h | 10 | ||||
| -rw-r--r-- | extern/Eigen3/Eigen/src/Core/arch/NEON/PacketMath.h | 50 | ||||
| -rw-r--r-- | extern/Eigen3/Eigen/src/Core/arch/SSE/Complex.h | 22 | ||||
| -rw-r--r-- | extern/Eigen3/Eigen/src/Core/arch/SSE/MathFunctions.h | 101 | ||||
| -rw-r--r-- | extern/Eigen3/Eigen/src/Core/arch/SSE/PacketMath.h | 29 |
6 files changed, 156 insertions, 59 deletions
diff --git a/extern/Eigen3/Eigen/src/Core/arch/AltiVec/PacketMath.h b/extern/Eigen3/Eigen/src/Core/arch/AltiVec/PacketMath.h index 75de1931198..e4089962dea 100644 --- a/extern/Eigen3/Eigen/src/Core/arch/AltiVec/PacketMath.h +++ b/extern/Eigen3/Eigen/src/Core/arch/AltiVec/PacketMath.h @@ -173,6 +173,9 @@ template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return psub<Packet4f>(p4f_ZERO, a); } template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return psub<Packet4i>(p4i_ZERO, a); } +template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } +template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } + template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_madd(a,b,p4f_ZERO); } /* Commented out: it's actually slower than processing it scalar * diff --git a/extern/Eigen3/Eigen/src/Core/arch/NEON/Complex.h b/extern/Eigen3/Eigen/src/Core/arch/NEON/Complex.h index 795b4be7303..f183d31de2a 100644 --- a/extern/Eigen3/Eigen/src/Core/arch/NEON/Complex.h +++ b/extern/Eigen3/Eigen/src/Core/arch/NEON/Complex.h @@ -68,7 +68,6 @@ template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { Packet4f v1, v2; - float32x2_t a_lo, a_hi; // Get the real values of a | a1_re | a1_re | a2_re | a2_re | v1 = vcombine_f32(vdup_lane_f32(vget_low_f32(a.v), 0), vdup_lane_f32(vget_high_f32(a.v), 0)); @@ -81,9 +80,7 @@ template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, con // Conjugate v2 v2 = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v2), p4ui_CONJ_XOR)); // Swap real/imag elements in v2. - a_lo = vrev64_f32(vget_low_f32(v2)); - a_hi = vrev64_f32(vget_high_f32(v2)); - v2 = vcombine_f32(a_lo, a_hi); + v2 = vrev64q_f32(v2); // Add and return the result return Packet2cf(vaddq_f32(v1, v2)); } @@ -241,13 +238,10 @@ template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, con // TODO optimize it for AltiVec Packet2cf res = conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a,b); Packet4f s, rev_s; - float32x2_t a_lo, a_hi; // this computes the norm s = vmulq_f32(b.v, b.v); - a_lo = vrev64_f32(vget_low_f32(s)); - a_hi = vrev64_f32(vget_high_f32(s)); - rev_s = vcombine_f32(a_lo, a_hi); + rev_s = vrev64q_f32(s); return Packet2cf(pdiv(res.v, vaddq_f32(s,rev_s))); } diff --git a/extern/Eigen3/Eigen/src/Core/arch/NEON/PacketMath.h b/extern/Eigen3/Eigen/src/Core/arch/NEON/PacketMath.h index a20250f7c65..163bac215e6 100644 --- a/extern/Eigen3/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/extern/Eigen3/Eigen/src/Core/arch/NEON/PacketMath.h @@ -115,6 +115,9 @@ template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return vnegq_f32(a); } template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return vnegq_s32(a); } +template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } +template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } + template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmulq_f32(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmulq_s32(a,b); } @@ -188,15 +191,15 @@ template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) { EI template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) { float32x2_t lo, hi; - lo = vdup_n_f32(*from); - hi = vdup_n_f32(*(from+1)); + lo = vld1_dup_f32(from); + hi = vld1_dup_f32(from+1); return vcombine_f32(lo, hi); } template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) { int32x2_t lo, hi; - lo = vdup_n_s32(*from); - hi = vdup_n_s32(*(from+1)); + lo = vld1_dup_s32(from); + hi = vld1_dup_s32(from+1); return vcombine_s32(lo, hi); } @@ -237,15 +240,12 @@ template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) { float32x2_t a_lo, a_hi, sum; - float s[2]; a_lo = vget_low_f32(a); a_hi = vget_high_f32(a); sum = vpadd_f32(a_lo, a_hi); sum = vpadd_f32(sum, sum); - vst1_f32(s, sum); - - return s[0]; + return vget_lane_f32(sum, 0); } template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) @@ -271,15 +271,12 @@ template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs) template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, sum; - int32_t s[2]; a_lo = vget_low_s32(a); a_hi = vget_high_s32(a); sum = vpadd_s32(a_lo, a_hi); sum = vpadd_s32(sum, sum); - vst1_s32(s, sum); - - return s[0]; + return vget_lane_s32(sum, 0); } template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs) @@ -307,7 +304,6 @@ template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs) template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) { float32x2_t a_lo, a_hi, prod; - float s[2]; // Get a_lo = |a1|a2| and a_hi = |a3|a4| a_lo = vget_low_f32(a); @@ -316,14 +312,12 @@ template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) prod = vmul_f32(a_lo, a_hi); // Multiply prod with its swapped value |a2*a4|a1*a3| prod = vmul_f32(prod, vrev64_f32(prod)); - vst1_f32(s, prod); - return s[0]; + return vget_lane_f32(prod, 0); } template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, prod; - int32_t s[2]; // Get a_lo = |a1|a2| and a_hi = |a3|a4| a_lo = vget_low_s32(a); @@ -332,65 +326,57 @@ template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) prod = vmul_s32(a_lo, a_hi); // Multiply prod with its swapped value |a2*a4|a1*a3| prod = vmul_s32(prod, vrev64_s32(prod)); - vst1_s32(s, prod); - return s[0]; + return vget_lane_s32(prod, 0); } // min template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) { float32x2_t a_lo, a_hi, min; - float s[2]; a_lo = vget_low_f32(a); a_hi = vget_high_f32(a); min = vpmin_f32(a_lo, a_hi); min = vpmin_f32(min, min); - vst1_f32(s, min); - return s[0]; + return vget_lane_f32(min, 0); } + template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, min; - int32_t s[2]; a_lo = vget_low_s32(a); a_hi = vget_high_s32(a); min = vpmin_s32(a_lo, a_hi); min = vpmin_s32(min, min); - vst1_s32(s, min); - - return s[0]; + + return vget_lane_s32(min, 0); } // max template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) { float32x2_t a_lo, a_hi, max; - float s[2]; a_lo = vget_low_f32(a); a_hi = vget_high_f32(a); max = vpmax_f32(a_lo, a_hi); max = vpmax_f32(max, max); - vst1_f32(s, max); - return s[0]; + return vget_lane_f32(max, 0); } + template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) { int32x2_t a_lo, a_hi, max; - int32_t s[2]; a_lo = vget_low_s32(a); a_hi = vget_high_s32(a); max = vpmax_s32(a_lo, a_hi); - max = vpmax_s32(max, max); - vst1_s32(s, max); - return s[0]; + return vget_lane_s32(max, 0); } // this PALIGN_NEON business is to work around a bug in LLVM Clang 3.0 causing incorrect compilation errors, diff --git a/extern/Eigen3/Eigen/src/Core/arch/SSE/Complex.h b/extern/Eigen3/Eigen/src/Core/arch/SSE/Complex.h index 12df987754c..91bba5e38fe 100644 --- a/extern/Eigen3/Eigen/src/Core/arch/SSE/Complex.h +++ b/extern/Eigen3/Eigen/src/Core/arch/SSE/Complex.h @@ -81,25 +81,31 @@ template<> EIGEN_STRONG_INLINE Packet2cf por <Packet2cf>(const Packet2cf& a, template<> EIGEN_STRONG_INLINE Packet2cf pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_xor_ps(a.v,b.v)); } template<> EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(_mm_andnot_ps(a.v,b.v)); } -template<> EIGEN_STRONG_INLINE Packet2cf pload <Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>(&real_ref(*from))); } -template<> EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>(&real_ref(*from))); } +template<> EIGEN_STRONG_INLINE Packet2cf pload <Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>(&numext::real_ref(*from))); } +template<> EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>(&numext::real_ref(*from))); } template<> EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) { Packet2cf res; - #if EIGEN_GNUC_AT_MOST(4,2) - // workaround annoying "may be used uninitialized in this function" warning with gcc 4.2 +#if EIGEN_GNUC_AT_MOST(4,2) + // Workaround annoying "may be used uninitialized in this function" warning with gcc 4.2 res.v = _mm_loadl_pi(_mm_set1_ps(0.0f), reinterpret_cast<const __m64*>(&from)); - #else +#elif EIGEN_GNUC_AT_LEAST(4,6) + // Suppress annoying "may be used uninitialized in this function" warning with gcc >= 4.6 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wuninitialized" res.v = _mm_loadl_pi(res.v, (const __m64*)&from); - #endif + #pragma GCC diagnostic pop +#else + res.v = _mm_loadl_pi(res.v, (const __m64*)&from); +#endif return Packet2cf(_mm_movelh_ps(res.v,res.v)); } template<> EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) { return pset1<Packet2cf>(*from); } -template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&real_ref(*to), from.v); } -template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&real_ref(*to), from.v); } +template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); } +template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); } template<> EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float> * addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); } diff --git a/extern/Eigen3/Eigen/src/Core/arch/SSE/MathFunctions.h b/extern/Eigen3/Eigen/src/Core/arch/SSE/MathFunctions.h index 3f41a4e2600..99cbd0d95be 100644 --- a/extern/Eigen3/Eigen/src/Core/arch/SSE/MathFunctions.h +++ b/extern/Eigen3/Eigen/src/Core/arch/SSE/MathFunctions.h @@ -31,7 +31,8 @@ Packet4f plog<Packet4f>(const Packet4f& _x) /* the smallest non denormalized float number */ _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(min_norm_pos, 0x00800000); - + _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(minus_inf, 0xff800000);//-1.f/0.f); + /* natural logarithm computed for 4 simultaneous float return NaN for x <= 0 */ @@ -51,7 +52,8 @@ Packet4f plog<Packet4f>(const Packet4f& _x) Packet4i emm0; - Packet4f invalid_mask = _mm_cmple_ps(x, _mm_setzero_ps()); + Packet4f invalid_mask = _mm_cmplt_ps(x, _mm_setzero_ps()); + Packet4f iszero_mask = _mm_cmpeq_ps(x, _mm_setzero_ps()); x = pmax(x, p4f_min_norm_pos); /* cut off denormalized stuff */ emm0 = _mm_srli_epi32(_mm_castps_si128(x), 23); @@ -96,7 +98,9 @@ Packet4f plog<Packet4f>(const Packet4f& _x) y2 = pmul(e, p4f_cephes_log_q2); x = padd(x, y); x = padd(x, y2); - return _mm_or_ps(x, invalid_mask); // negative arg will be NAN + // negative arg will be NAN, 0 will be -INF + return _mm_or_ps(_mm_andnot_ps(iszero_mask, _mm_or_ps(x, invalid_mask)), + _mm_and_ps(iszero_mask, p4f_minus_inf)); } template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED @@ -131,13 +135,16 @@ Packet4f pexp<Packet4f>(const Packet4f& _x) /* express exp(x) as exp(g + n*log(2)) */ fx = pmadd(x, p4f_cephes_LOG2EF, p4f_half); - /* how to perform a floorf with SSE: just below */ +#ifdef EIGEN_VECTORIZE_SSE4_1 + fx = _mm_floor_ps(fx); +#else emm0 = _mm_cvttps_epi32(fx); tmp = _mm_cvtepi32_ps(emm0); /* if greater, substract 1 */ Packet4f mask = _mm_cmpgt_ps(tmp, fx); mask = _mm_and_ps(mask, p4f_1); fx = psub(tmp, mask); +#endif tmp = pmul(fx, p4f_cephes_exp_C1); Packet4f z = pmul(fx, p4f_cephes_exp_C2); @@ -161,6 +168,79 @@ Packet4f pexp<Packet4f>(const Packet4f& _x) emm0 = _mm_slli_epi32(emm0, 23); return pmul(y, _mm_castsi128_ps(emm0)); } +template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED +Packet2d pexp<Packet2d>(const Packet2d& _x) +{ + Packet2d x = _x; + + _EIGEN_DECLARE_CONST_Packet2d(1 , 1.0); + _EIGEN_DECLARE_CONST_Packet2d(2 , 2.0); + _EIGEN_DECLARE_CONST_Packet2d(half, 0.5); + + _EIGEN_DECLARE_CONST_Packet2d(exp_hi, 709.437); + _EIGEN_DECLARE_CONST_Packet2d(exp_lo, -709.436139303); + + _EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599); + + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4); + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2); + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1); + + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6); + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3); + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1); + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0); + + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125); + _EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6); + static const __m128i p4i_1023_0 = _mm_setr_epi32(1023, 1023, 0, 0); + + Packet2d tmp = _mm_setzero_pd(), fx; + Packet4i emm0; + + // clamp x + x = pmax(pmin(x, p2d_exp_hi), p2d_exp_lo); + /* express exp(x) as exp(g + n*log(2)) */ + fx = pmadd(p2d_cephes_LOG2EF, x, p2d_half); + +#ifdef EIGEN_VECTORIZE_SSE4_1 + fx = _mm_floor_pd(fx); +#else + emm0 = _mm_cvttpd_epi32(fx); + tmp = _mm_cvtepi32_pd(emm0); + /* if greater, substract 1 */ + Packet2d mask = _mm_cmpgt_pd(tmp, fx); + mask = _mm_and_pd(mask, p2d_1); + fx = psub(tmp, mask); +#endif + + tmp = pmul(fx, p2d_cephes_exp_C1); + Packet2d z = pmul(fx, p2d_cephes_exp_C2); + x = psub(x, tmp); + x = psub(x, z); + + Packet2d x2 = pmul(x,x); + + Packet2d px = p2d_cephes_exp_p0; + px = pmadd(px, x2, p2d_cephes_exp_p1); + px = pmadd(px, x2, p2d_cephes_exp_p2); + px = pmul (px, x); + + Packet2d qx = p2d_cephes_exp_q0; + qx = pmadd(qx, x2, p2d_cephes_exp_q1); + qx = pmadd(qx, x2, p2d_cephes_exp_q2); + qx = pmadd(qx, x2, p2d_cephes_exp_q3); + + x = pdiv(px,psub(qx,px)); + x = pmadd(p2d_2,x,p2d_1); + + // build 2^n + emm0 = _mm_cvttpd_epi32(fx); + emm0 = _mm_add_epi32(emm0, p4i_1023_0); + emm0 = _mm_slli_epi32(emm0, 20); + emm0 = _mm_shuffle_epi32(emm0, _MM_SHUFFLE(1,2,0,3)); + return pmul(x, _mm_castsi128_pd(emm0)); +} /* evaluation of 4 sines at onces, using SSE2 intrinsics. @@ -362,21 +442,32 @@ Packet4f pcos<Packet4f>(const Packet4f& _x) return _mm_xor_ps(y, sign_bit); } +#if EIGEN_FAST_MATH + // This is based on Quake3's fast inverse square root. // For detail see here: http://www.beyond3d.com/content/articles/8/ +// It lacks 1 (or 2 bits in some rare cases) of precision, and does not handle negative, +inf, or denormalized numbers correctly. template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psqrt<Packet4f>(const Packet4f& _x) { Packet4f half = pmul(_x, pset1<Packet4f>(.5f)); /* select only the inverse sqrt of non-zero inputs */ - Packet4f non_zero_mask = _mm_cmpgt_ps(_x, pset1<Packet4f>(std::numeric_limits<float>::epsilon())); + Packet4f non_zero_mask = _mm_cmpge_ps(_x, pset1<Packet4f>((std::numeric_limits<float>::min)())); Packet4f x = _mm_and_ps(non_zero_mask, _mm_rsqrt_ps(_x)); x = pmul(x, psub(pset1<Packet4f>(1.5f), pmul(half, pmul(x,x)))); return pmul(_x,x); } +#else + +template<> EIGEN_STRONG_INLINE Packet4f psqrt<Packet4f>(const Packet4f& x) { return _mm_sqrt_ps(x); } + +#endif + +template<> EIGEN_STRONG_INLINE Packet2d psqrt<Packet2d>(const Packet2d& x) { return _mm_sqrt_pd(x); } + } // end namespace internal } // end namespace Eigen diff --git a/extern/Eigen3/Eigen/src/Core/arch/SSE/PacketMath.h b/extern/Eigen3/Eigen/src/Core/arch/SSE/PacketMath.h index 10d9182190f..fc8ae50fed7 100644 --- a/extern/Eigen3/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/extern/Eigen3/Eigen/src/Core/arch/SSE/PacketMath.h @@ -48,6 +48,9 @@ template<> struct is_arithmetic<__m128d> { enum { value = true }; }; #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ const Packet4f p4f_##NAME = pset1<Packet4f>(X) +#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ + const Packet2d p2d_##NAME = pset1<Packet2d>(X) + #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ const Packet4f p4f_##NAME = _mm_castsi128_ps(pset1<Packet4i>(X)) @@ -63,7 +66,7 @@ template<> struct packet_traits<float> : default_packet_traits AlignedOnScalar = 1, size=4, - HasDiv = 1, + HasDiv = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, @@ -79,7 +82,9 @@ template<> struct packet_traits<double> : default_packet_traits AlignedOnScalar = 1, size=2, - HasDiv = 1 + HasDiv = 1, + HasExp = 1, + HasSqrt = 1 }; }; template<> struct packet_traits<int> : default_packet_traits @@ -137,6 +142,10 @@ template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) return psub(_mm_setr_epi32(0,0,0,0), a); } +template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; } +template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; } +template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; } + template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) @@ -169,18 +178,26 @@ template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_min_epi32(a,b); +#else // after some bench, this version *is* faster than a scalar implementation Packet4i mask = _mm_cmplt_epi32(a,b); return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); +#endif } template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); } template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); } template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { +#ifdef EIGEN_VECTORIZE_SSE4_1 + return _mm_max_epi32(a,b); +#else // after some bench, this version *is* faster than a scalar implementation Packet4i mask = _mm_cmpgt_epi32(a,b); return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b)); +#endif } template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); } @@ -491,8 +508,8 @@ template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) // for GCC (eg., it does not like using std::min after the pstore !!) EIGEN_ALIGN16 int aux[4]; pstore(aux, a); - register int aux0 = aux[0]<aux[1] ? aux[0] : aux[1]; - register int aux2 = aux[2]<aux[3] ? aux[2] : aux[3]; + int aux0 = aux[0]<aux[1] ? aux[0] : aux[1]; + int aux2 = aux[2]<aux[3] ? aux[2] : aux[3]; return aux0<aux2 ? aux0 : aux2; } @@ -512,8 +529,8 @@ template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) // for GCC (eg., it does not like using std::min after the pstore !!) EIGEN_ALIGN16 int aux[4]; pstore(aux, a); - register int aux0 = aux[0]>aux[1] ? aux[0] : aux[1]; - register int aux2 = aux[2]>aux[3] ? aux[2] : aux[3]; + int aux0 = aux[0]>aux[1] ? aux[0] : aux[1]; + int aux2 = aux[2]>aux[3] ? aux[2] : aux[3]; return aux0>aux2 ? aux0 : aux2; } |