Fix T47758: rigidbody bug with constraint breaking and disable collisions.

This is a regression in Bullet, reverted the problematic change for now with
a custom patch.

3 files changed, 112 insertions, 33 deletions

diff --git a/extern/bullet2/patches/blender.patch b/extern/bullet2/patches/blender.patch
index f67d6910422..96357ddd315 100644
--- a/extern/bullet2/patches/blender.patch
+++ b/extern/bullet2/patches/blender.patch
@@ -233,3 +233,92 @@ index e05bdcc..dbcf2b6 100644
  
  
  #include "btVector3.h"
+diff --git a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp
+index e0e8bc7..a788268 100644
+--- a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp
++++ b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp
+@@ -425,50 +425,38 @@ void btRigidBody::setCenterOfMassTransform(const btTransform& xform)
+ }
+ 
+ 
++bool btRigidBody::checkCollideWithOverride(const  btCollisionObject* co) const
++{
++	const btRigidBody* otherRb = btRigidBody::upcast(co);
++	if (!otherRb)
++		return true;
++
++	for (int i = 0; i < m_constraintRefs.size(); ++i)
++	{
++		const btTypedConstraint* c = m_constraintRefs[i];
++		if (c->isEnabled())
++			if (&c->getRigidBodyA() == otherRb || &c->getRigidBodyB() == otherRb)
++				return false;
++	}
++
++	return true;
++}
+ 
+ 
+ 
+ void btRigidBody::addConstraintRef(btTypedConstraint* c)
+ {
+-	///disable collision with the 'other' body
+-
+ 	int index = m_constraintRefs.findLinearSearch(c);
+-	//don't add constraints that are already referenced
+-	//btAssert(index == m_constraintRefs.size());
+ 	if (index == m_constraintRefs.size())
+-	{
+-		m_constraintRefs.push_back(c);
+-		btCollisionObject* colObjA = &c->getRigidBodyA();
+-		btCollisionObject* colObjB = &c->getRigidBodyB();
+-		if (colObjA == this)
+-		{
+-			colObjA->setIgnoreCollisionCheck(colObjB, true);
+-		}
+-		else
+-		{
+-			colObjB->setIgnoreCollisionCheck(colObjA, true);
+-		}
+-	} 
++		m_constraintRefs.push_back(c); 
++
++	m_checkCollideWith = true;
+ }
+ 
+ void btRigidBody::removeConstraintRef(btTypedConstraint* c)
+ {
+-	int index = m_constraintRefs.findLinearSearch(c);
+-	//don't remove constraints that are not referenced
+-	if(index < m_constraintRefs.size())
+-    {
+-        m_constraintRefs.remove(c);
+-        btCollisionObject* colObjA = &c->getRigidBodyA();
+-        btCollisionObject* colObjB = &c->getRigidBodyB();
+-        if (colObjA == this)
+-        {
+-            colObjA->setIgnoreCollisionCheck(colObjB, false);
+-        }
+-        else
+-        {
+-            colObjB->setIgnoreCollisionCheck(colObjA, false);
+-        }
+-    }
++	m_constraintRefs.remove(c);
++	m_checkCollideWith = m_constraintRefs.size() > 0;
+ }
+ 
+ int	btRigidBody::calculateSerializeBufferSize()	const
+diff --git a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h
+index 1d177db..c2f8c5d 100644
+--- a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h
++++ b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h
+@@ -509,6 +509,8 @@ public:
+ 		return (getBroadphaseProxy() != 0);
+ 	}
+ 
++	virtual bool checkCollideWithOverride(const  btCollisionObject* co) const;
++
+ 	void addConstraintRef(btTypedConstraint* c);
+ 	void removeConstraintRef(btTypedConstraint* c);
+ 
diff --git a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp
index e0e8bc70f57..a7882684bf1 100644
--- a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp
+++ b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.cpp
@@ -425,50 +425,38 @@ void btRigidBody::setCenterOfMassTransform(const btTransform& xform)
 }
 
 
+bool btRigidBody::checkCollideWithOverride(const  btCollisionObject* co) const
+{
+	const btRigidBody* otherRb = btRigidBody::upcast(co);
+	if (!otherRb)
+		return true;
+
+	for (int i = 0; i < m_constraintRefs.size(); ++i)
+	{
+		const btTypedConstraint* c = m_constraintRefs[i];
+		if (c->isEnabled())
+			if (&c->getRigidBodyA() == otherRb || &c->getRigidBodyB() == otherRb)
+				return false;
+	}
+
+	return true;
+}
 
 
 
 void btRigidBody::addConstraintRef(btTypedConstraint* c)
 {
-	///disable collision with the 'other' body
-
 	int index = m_constraintRefs.findLinearSearch(c);
-	//don't add constraints that are already referenced
-	//btAssert(index == m_constraintRefs.size());
 	if (index == m_constraintRefs.size())
-	{
-		m_constraintRefs.push_back(c);
-		btCollisionObject* colObjA = &c->getRigidBodyA();
-		btCollisionObject* colObjB = &c->getRigidBodyB();
-		if (colObjA == this)
-		{
-			colObjA->setIgnoreCollisionCheck(colObjB, true);
-		}
-		else
-		{
-			colObjB->setIgnoreCollisionCheck(colObjA, true);
-		}
-	} 
+		m_constraintRefs.push_back(c); 
+
+	m_checkCollideWith = true;
 }
 
 void btRigidBody::removeConstraintRef(btTypedConstraint* c)
 {
-	int index = m_constraintRefs.findLinearSearch(c);
-	//don't remove constraints that are not referenced
-	if(index < m_constraintRefs.size())
-    {
-        m_constraintRefs.remove(c);
-        btCollisionObject* colObjA = &c->getRigidBodyA();
-        btCollisionObject* colObjB = &c->getRigidBodyB();
-        if (colObjA == this)
-        {
-            colObjA->setIgnoreCollisionCheck(colObjB, false);
-        }
-        else
-        {
-            colObjB->setIgnoreCollisionCheck(colObjA, false);
-        }
-    }
+	m_constraintRefs.remove(c);
+	m_checkCollideWith = m_constraintRefs.size() > 0;
 }
 
 int	btRigidBody::calculateSerializeBufferSize()	const
diff --git a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h
index 1d177db802a..c2f8c5d64ae 100644
--- a/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h
+++ b/extern/bullet2/src/BulletDynamics/Dynamics/btRigidBody.h
@@ -509,6 +509,8 @@ public:
 		return (getBroadphaseProxy() != 0);
 	}
 
+	virtual bool checkCollideWithOverride(const  btCollisionObject* co) const;
+
 	void addConstraintRef(btTypedConstraint* c);
 	void removeConstraintRef(btTypedConstraint* c);