BGE : Collision mask support in raycast + and raycast cleanup.

I have removed the m_pHitObject, m_xray and m_testPropName and replace them by a temporary struct "RayCastData" which contains these datas and a collision mask. Finally i add a collision mask argument in the python function "rayCast" :
```
rayCast(to, from, dist, prop, face, xray, poly, mask)
```

It can be useful to hit only object which are on the right colision layer. for example if you have hitbox for a charater or vehicle you don't want to hit it with raycast.

test file : {F237337}
left mouse click on two planes and see console messages.

Somewhat more elaborate test file by @sybren: {F237779}
Look around and click on the cubes. One cube lamp responds, the other doesn't, based on their collision groups.

Reviewers: moguri, hg1, agoose77, campbellbarton, sybren

Reviewed By: agoose77, campbellbarton, sybren

Subscribers: campbellbarton, sergey, blueprintrandom, sybren

Projects: #game_engine, #game_physics

Differential Revision: https://developer.blender.org/D1239

2 files changed, 8 insertions, 5 deletions

diff --git a/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.cpp b/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.cpp
index 1758b7abf9f..bd84c3c96b1 100644
--- a/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.cpp
+++ b/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.cpp
@@ -1215,10 +1215,9 @@ void RAS_OpenGLRasterizer::RemoveLight(RAS_ILightObject* lightobject)
 		m_lights.erase(lit);
 }
 
-bool RAS_OpenGLRasterizer::RayHit(struct KX_ClientObjectInfo *client, KX_RayCast *result, void * const data)
+bool RAS_OpenGLRasterizer::RayHit(struct KX_ClientObjectInfo *client, KX_RayCast *result, double *oglmatrix)
 {
 	if (result->m_hitMesh) {
-		double* const oglmatrix = (double* const) data;
 
 		RAS_Polygon* poly = result->m_hitMesh->GetPolygon(result->m_hitPolygon);
 		if (!poly->IsVisible())
@@ -1328,7 +1327,7 @@ void RAS_OpenGLRasterizer::applyTransform(double* oglmatrix,int objectdrawmode )
 			if (!physics_controller && parent)
 				physics_controller = parent->GetPhysicsController();
 
-			KX_RayCast::Callback<RAS_OpenGLRasterizer> callback(this, physics_controller, oglmatrix);
+			KX_RayCast::Callback<RAS_OpenGLRasterizer, double> callback(this, physics_controller, oglmatrix);
 			if (!KX_RayCast::RayTest(physics_environment, frompoint, topoint, callback))
 			{
 				// couldn't find something to cast the shadow on...
diff --git a/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.h b/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.h
index ad49ebe5179..67a11f64726 100644
--- a/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.h
+++ b/source/gameengine/Rasterizer/RAS_OpenGLRasterizer/RAS_OpenGLRasterizer.h
@@ -45,6 +45,8 @@ using namespace std;
 #include "RAS_MaterialBucket.h"
 #include "RAS_IPolygonMaterial.h"
 
+#include "BLI_utildefines.h"
+
 class RAS_IStorage;
 class RAS_ICanvas;
 class RAS_OpenGLLight;
@@ -306,8 +308,10 @@ public:
 	void PushMatrix();
 	void PopMatrix();
 
-	bool RayHit(struct KX_ClientObjectInfo *client, class KX_RayCast *result, void * const data);
-	bool NeedRayCast(struct KX_ClientObjectInfo *) { return true; }
+	/// \see KX_RayCast
+	bool RayHit(struct KX_ClientObjectInfo *client, class KX_RayCast *result, double *oglmatrix);
+	/// \see KX_RayCast
+	bool NeedRayCast(struct KX_ClientObjectInfo *, void *UNUSED(data)) { return true; }
 
 	RAS_ILightObject* CreateLight();
 	void AddLight(RAS_ILightObject* lightobject);