From bd438de8c6bee75c6ae7946a01a1b86e5cb05565 Mon Sep 17 00:00:00 2001 From: "Sv. Lockal" Date: Tue, 4 Feb 2014 23:38:53 +0400 Subject: Cycles: bvh_cardinal_curve_intersect intro SSE optimization Gives 5% speedup for koro_final.blend, 10-15% for hair.blend Reviewed By: brecht Differential Revision: https://developer.blender.org/D225 --- intern/cycles/kernel/kernel_bvh.h | 81 ++++++++++++++++++++++++++++- intern/cycles/kernel/kernel_bvh_traversal.h | 4 +- intern/cycles/util/util_simd.h | 12 +++++ intern/cycles/util/util_types.h | 26 +++++---- 4 files changed, 109 insertions(+), 14 deletions(-) (limited to 'intern') diff --git a/intern/cycles/kernel/kernel_bvh.h b/intern/cycles/kernel/kernel_bvh.h index 4ddee6f656c..93e546eaece 100644 --- a/intern/cycles/kernel/kernel_bvh.h +++ b/intern/cycles/kernel/kernel_bvh.h @@ -211,8 +211,21 @@ ccl_device_inline void curvebounds(float *lower, float *upper, float *extremta, } } +#ifdef __KERNEL_SSE2__ +ccl_device_inline __m128 transform_point_T3(const __m128 t[3], const __m128 &a) +{ + return fma(broadcast<0>(a), t[0], fma(broadcast<1>(a), t[1], _mm_mul_ps(broadcast<2>(a), t[2]))); +} +#endif + +#ifdef __KERNEL_SSE2__ +/* Pass P and idir by reference to aligned vector */ +ccl_device_inline bool bvh_cardinal_curve_intersect(KernelGlobals *kg, Intersection *isect, + const float3 &P, const float3 &idir, uint visibility, int object, int curveAddr, int segment, uint *lcg_state, float difl, float extmax) +#else ccl_device_inline bool bvh_cardinal_curve_intersect(KernelGlobals *kg, Intersection *isect, float3 P, float3 idir, uint visibility, int object, int curveAddr, int segment, uint *lcg_state, float difl, float extmax) +#endif { float epsilon = 0.0f; float r_st, r_en; @@ -220,7 +233,59 @@ ccl_device_inline bool bvh_cardinal_curve_intersect(KernelGlobals *kg, Intersect int depth = kernel_data.curve.subdivisions; int flags = kernel_data.curve.curveflags; int prim = kernel_tex_fetch(__prim_index, curveAddr); + +#ifdef __KERNEL_SSE2__ + __m128 vdir = _mm_div_ps(_mm_set1_ps(1.0f), (__m128 &)idir); + __m128 vcurve_coef[4]; + const float3 *curve_coef = (float3 *)vcurve_coef; + { + __m128 dtmp = _mm_mul_ps(vdir, vdir); + __m128 d_ss = _mm_sqrt_ss(_mm_add_ss(dtmp, broadcast<2>(dtmp))); + __m128 rd_ss = _mm_div_ss(_mm_set_ss(1.0f), d_ss); + + __m128i v00vec = _mm_load_si128((__m128i *)&kg->__curves.data[prim]); + int2 &v00 = (int2 &)v00vec; + + int k0 = v00.x + segment; + int k1 = k0 + 1; + int ka = max(k0 - 1, v00.x); + int kb = min(k1 + 1, v00.x + v00.y - 1); + + __m128 P0 = _mm_load_ps(&kg->__curve_keys.data[ka].x); + __m128 P1 = _mm_load_ps(&kg->__curve_keys.data[k0].x); + __m128 P2 = _mm_load_ps(&kg->__curve_keys.data[k1].x); + __m128 P3 = _mm_load_ps(&kg->__curve_keys.data[kb].x); + + __m128 rd_sgn = set_sign_bit<0, 1, 1, 1>(broadcast<0>(rd_ss)); + __m128 mul_zxxy = _mm_mul_ps(shuffle<2, 0, 0, 1>(vdir), rd_sgn); + __m128 mul_yz = _mm_mul_ps(shuffle<1, 2, 1, 2>(vdir), mul_zxxy); + __m128 mul_shuf = shuffle<0, 1, 2, 3>(mul_zxxy, mul_yz); + __m128 vdir0 = _mm_and_ps(vdir, _mm_castsi128_ps(_mm_setr_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0))); + + __m128 htfm0 = shuffle<0, 2, 0, 3>(mul_shuf, vdir0); + __m128 htfm1 = shuffle<1, 0, 1, 3>(_mm_set_ss(_mm_cvtss_f32(d_ss)), vdir0); + __m128 htfm2 = shuffle<1, 3, 2, 3>(mul_shuf, vdir0); + + __m128 htfm[] = { htfm0, htfm1, htfm2 }; + __m128 p0 = transform_point_T3(htfm, _mm_sub_ps(P0, (__m128 &)P)); + __m128 p1 = transform_point_T3(htfm, _mm_sub_ps(P1, (__m128 &)P)); + __m128 p2 = transform_point_T3(htfm, _mm_sub_ps(P2, (__m128 &)P)); + __m128 p3 = transform_point_T3(htfm, _mm_sub_ps(P3, (__m128 &)P)); + + float fc = 0.71f; + __m128 vfc = _mm_set1_ps(fc); + __m128 vfcxp3 = _mm_mul_ps(vfc, p3); + + vcurve_coef[0] = p1; + vcurve_coef[1] = _mm_mul_ps(vfc, _mm_sub_ps(p2, p0)); + vcurve_coef[2] = fma(_mm_set1_ps(fc * 2.0f), p0, fma(_mm_set1_ps(fc - 3.0f), p1, fms(_mm_set1_ps(3.0f - 2.0f * fc), p2, vfcxp3))); + vcurve_coef[3] = fms(_mm_set1_ps(fc - 2.0f), _mm_sub_ps(p2, p1), fms(vfc, p0, vfcxp3)); + + r_st = ((float4 &)P1).w; + r_en = ((float4 &)P2).w; + } +#else float3 curve_coef[4]; /* curve Intersection check */ @@ -263,7 +328,7 @@ ccl_device_inline bool bvh_cardinal_curve_intersect(KernelGlobals *kg, Intersect r_st = P1.w; r_en = P2.w; } - +#endif float r_curr = max(r_st, r_en); @@ -302,6 +367,19 @@ ccl_device_inline bool bvh_cardinal_curve_intersect(KernelGlobals *kg, Intersect while(!(tree >> (depth))) { float i_st = tree * resol; float i_en = i_st + (level * resol); +#ifdef __KERNEL_SSE2__ + __m128 vi_st = _mm_set1_ps(i_st), vi_en = _mm_set1_ps(i_en); + __m128 vp_st = fma(fma(fma(vcurve_coef[3], vi_st, vcurve_coef[2]), vi_st, vcurve_coef[1]), vi_st, vcurve_coef[0]); + __m128 vp_en = fma(fma(fma(vcurve_coef[3], vi_en, vcurve_coef[2]), vi_en, vcurve_coef[1]), vi_en, vcurve_coef[0]); + + __m128 vbmin = _mm_min_ps(vp_st, vp_en); + __m128 vbmax = _mm_max_ps(vp_st, vp_en); + + float3 &bmin = (float3 &)vbmin, &bmax = (float3 &)vbmax; + float &bminx = bmin.x, &bminy = bmin.y, &bminz = bmin.z; + float &bmaxx = bmax.x, &bmaxy = bmax.y, &bmaxz = bmax.z; + float3 &p_st = (float3 &)vp_st, &p_en = (float3 &)vp_en; +#else float3 p_st = ((curve_coef[3] * i_st + curve_coef[2]) * i_st + curve_coef[1]) * i_st + curve_coef[0]; float3 p_en = ((curve_coef[3] * i_en + curve_coef[2]) * i_en + curve_coef[1]) * i_en + curve_coef[0]; @@ -311,6 +389,7 @@ ccl_device_inline bool bvh_cardinal_curve_intersect(KernelGlobals *kg, Intersect float bmaxy = max(p_st.y, p_en.y); float bminz = min(p_st.z, p_en.z); float bmaxz = max(p_st.z, p_en.z); +#endif if(xextrem[0] >= i_st && xextrem[0] <= i_en) { bminx = min(bminx,xextrem[1]); diff --git a/intern/cycles/kernel/kernel_bvh_traversal.h b/intern/cycles/kernel/kernel_bvh_traversal.h index b4c63f5682c..5309a80f31d 100644 --- a/intern/cycles/kernel/kernel_bvh_traversal.h +++ b/intern/cycles/kernel/kernel_bvh_traversal.h @@ -55,8 +55,8 @@ ccl_device bool BVH_FUNCTION_NAME /* ray parameters in registers */ const float tmax = ray->t; - float3 P = ray->P; - float3 idir = bvh_inverse_direction(ray->D); + ccl_align(16) float3 P = ray->P; + ccl_align(16) float3 idir = bvh_inverse_direction(ray->D); int object = ~0; #if FEATURE(BVH_MOTION) diff --git a/intern/cycles/util/util_simd.h b/intern/cycles/util/util_simd.h index 2d3a927f227..fd5ba1de37b 100644 --- a/intern/cycles/util/util_simd.h +++ b/intern/cycles/util/util_simd.h @@ -148,6 +148,12 @@ ccl_device_inline const __m128 fma(const __m128& a, const __m128& b, const __m12 return _mm_add_ps(_mm_mul_ps(a, b), c); } +/* calculate a*b-c (replacement for fused multiply-subtract on SSE CPUs) */ +ccl_device_inline const __m128 fms(const __m128& a, const __m128& b, const __m128& c) +{ + return _mm_sub_ps(_mm_mul_ps(a, b), c); +} + template ccl_device_inline const __m128 broadcast(const __m128& a) { return _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(a), _MM_SHUFFLE(N, N, N, N))); @@ -168,6 +174,12 @@ ccl_device_inline const __m128 uint32_to_float(const __m128i &in) return _mm_add_ps(e, d); } +template +ccl_device_inline const __m128 set_sign_bit(const __m128 &a) +{ + return _mm_xor_ps(a, _mm_castsi128_ps(_mm_setr_epi32(S1 << 31, S2 << 31, S3 << 31, S4 << 31))); +} + #endif /* __KERNEL_SSE2__ */ CCL_NAMESPACE_END diff --git a/intern/cycles/util/util_types.h b/intern/cycles/util/util_types.h index 66aab766667..c770931c69b 100644 --- a/intern/cycles/util/util_types.h +++ b/intern/cycles/util/util_types.h @@ -41,10 +41,11 @@ #if defined(_WIN32) && !defined(FREE_WINDOWS) #define ccl_device_inline static __forceinline -#ifdef __KERNEL_64_BIT__ #define ccl_align(...) __declspec(align(__VA_ARGS__)) +#ifdef __KERNEL_64_BIT__ +#define ccl_try_align(...) __declspec(align(__VA_ARGS__)) #else -#define ccl_align(...) /* not support for function arguments (error C2719) */ +#define ccl_try_align(...) /* not support for function arguments (error C2719) */ #endif #define ccl_may_alias #define ccl_always_inline __forceinline @@ -52,15 +53,18 @@ #else #define ccl_device_inline static inline __attribute__((always_inline)) +#define ccl_align(...) __attribute__((aligned(__VA_ARGS__))) #ifndef FREE_WINDOWS64 #define __forceinline inline __attribute__((always_inline)) #endif -#define ccl_align(...) __attribute__((aligned(__VA_ARGS__))) +#define ccl_try_align(...) __attribute__((aligned(__VA_ARGS__))) #define ccl_may_alias __attribute__((__may_alias__)) #define ccl_always_inline __attribute__((always_inline)) #endif +#else +#define ccl_align(...) #endif /* Standard Integer Types */ @@ -156,7 +160,7 @@ struct int2 { }; #ifdef __KERNEL_SSE__ -struct ccl_align(16) int3 { +struct ccl_try_align(16) int3 { union { __m128i m128; struct { int x, y, z, w; }; @@ -167,7 +171,7 @@ struct ccl_align(16) int3 { __forceinline operator const __m128i&(void) const { return m128; } __forceinline operator __m128i&(void) { return m128; } #else -struct ccl_align(16) int3 { +struct ccl_try_align(16) int3 { int x, y, z, w; #endif @@ -176,7 +180,7 @@ struct ccl_align(16) int3 { }; #ifdef __KERNEL_SSE__ -struct ccl_align(16) int4 { +struct ccl_try_align(16) int4 { union { __m128i m128; struct { int x, y, z, w; }; @@ -187,7 +191,7 @@ struct ccl_align(16) int4 { __forceinline operator const __m128i&(void) const { return m128; } __forceinline operator __m128i&(void) { return m128; } #else -struct ccl_align(16) int4 { +struct ccl_try_align(16) int4 { int x, y, z, w; #endif @@ -224,7 +228,7 @@ struct float2 { }; #ifdef __KERNEL_SSE__ -struct ccl_align(16) float3 { +struct ccl_try_align(16) float3 { union { __m128 m128; struct { float x, y, z, w; }; @@ -235,7 +239,7 @@ struct ccl_align(16) float3 { __forceinline operator const __m128&(void) const { return m128; } __forceinline operator __m128&(void) { return m128; } #else -struct ccl_align(16) float3 { +struct ccl_try_align(16) float3 { float x, y, z, w; #endif @@ -244,7 +248,7 @@ struct ccl_align(16) float3 { }; #ifdef __KERNEL_SSE__ -struct ccl_align(16) float4 { +struct ccl_try_align(16) float4 { union { __m128 m128; struct { float x, y, z, w; }; @@ -255,7 +259,7 @@ struct ccl_align(16) float4 { __forceinline operator const __m128&(void) const { return m128; } __forceinline operator __m128&(void) { return m128; } #else -struct ccl_align(16) float4 { +struct ccl_try_align(16) float4 { float x, y, z, w; #endif -- cgit v1.2.3