Fix undefined behavior in tangent space computation

Use an improved implementation for circular shift.

Differential Revision: https://developer.blender.org/D6677

1 files changed, 13 insertions, 1 deletions

diff --git a/intern/mikktspace/mikktspace.c b/intern/mikktspace/mikktspace.c
index 4f120b7d83c..285529298eb 100644
--- a/intern/mikktspace/mikktspace.c
+++ b/intern/mikktspace/mikktspace.c
@@ -27,6 +27,7 @@
 #include <string.h>
 #include <float.h>
 #include <stdlib.h>
+#include <limits.h>
 
 #include "mikktspace.h"
 
@@ -135,6 +136,17 @@ MIKK_INLINE tbool VNotZero(const SVec3 v)
 }
 #endif
 
+// Shift operations in C are only defined for shift values which are
+// not negative and smaller than sizeof(value) * CHAR_BIT.
+// The mask, used with bitwise-and (&), prevents undefined behaviour
+// when the shift count is 0 or >= the width of unsigned int.
+MIKK_INLINE unsigned int rotl(unsigned int value, unsigned int count)
+{
+  const unsigned int mask = CHAR_BIT * sizeof(value) - 1;
+  count &= mask;
+  return (value << count) | (value >> (-count & mask));
+}
+
 typedef struct {
   int iNrFaces;
   int *pTriMembers;
@@ -1605,7 +1617,7 @@ static void QuickSortEdges(
 
   // Random
   t = uSeed & 31;
-  t = (uSeed << t) | (uSeed >> (32 - t));
+  t = rotl(uSeed, t);
   uSeed = uSeed + t + 3;
   // Random end