BGE: Fix Orthographic mode and viewport scaling

- the BGE now uses correct glOrtho projection whe camera is in orthographic mode
- 

5 files changed, 136 insertions, 115 deletions

diff --git a/source/gameengine/Ketsji/KX_Camera.cpp b/source/gameengine/Ketsji/KX_Camera.cpp
index e536d1f1e7e..8565346b30e 100644
--- a/source/gameengine/Ketsji/KX_Camera.cpp
+++ b/source/gameengine/Ketsji/KX_Camera.cpp
@@ -47,7 +47,7 @@ KX_Camera::KX_Camera(void* sgReplicationInfo,
 					m_camdata(camdata),
 					m_dirty(true),
 					m_normalized(false),
-					m_frustum_culling(frustum_culling && camdata.m_perspective),
+					m_frustum_culling(frustum_culling),
 					m_set_projection_matrix(false),
 					m_set_frustum_center(false)
 {
@@ -184,6 +184,11 @@ float KX_Camera::GetLens() const
 	return m_camdata.m_lens;
 }
 
+float KX_Camera::GetScale() const
+{
+	return m_camdata.m_scale;
+}
+
 
 
 float KX_Camera::GetCameraNear() const
@@ -264,80 +269,83 @@ void KX_Camera::ExtractFrustumSphere()
 	MT_Matrix4x4 clip_camcs_matrix = m_projection_matrix;
 	clip_camcs_matrix.invert();
 
-    // detect which of the corner of the far clipping plane is the farthest to the origin
-	MT_Vector4 nfar;    // far point in device normalized coordinate
-    MT_Point3 farpoint; // most extreme far point in camera coordinate
-    MT_Point3 nearpoint;// most extreme near point in camera coordinate
-    MT_Point3 farcenter(0.,0.,0.);// center of far cliping plane in camera coordinate
-    MT_Scalar F=1.0, N; // square distance of far and near point to origin
-    MT_Scalar f, n;     // distance of far and near point to z axis. f is always > 0 but n can be < 0
-    MT_Scalar e, s;     // far and near clipping distance (<0)
-    MT_Scalar c;        // slope of center line = distance of far clipping center to z axis / far clipping distance
-    MT_Scalar z;        // projection of sphere center on z axis (<0)
-    // tmp value
-    MT_Vector4 npoint(1., 1., 1., 1.);
-    MT_Vector4 hpoint;
-    MT_Point3 point;
-    MT_Scalar len;
-    for (int i=0; i<4; i++)
-    {
-    	hpoint = clip_camcs_matrix*npoint;
-        point.setValue(hpoint[0]/hpoint[3], hpoint[1]/hpoint[3], hpoint[2]/hpoint[3]);
-        len = point.dot(point);
-        if (len > F)
-        {
-            nfar = npoint;
-            farpoint = point;
-            F = len;
-        }
-        // rotate by 90 degree along the z axis to walk through the 4 extreme points of the far clipping plane
-        len = npoint[0];
-        npoint[0] = -npoint[1];
-        npoint[1] = len;
-        farcenter += point;
-    }
-    // the far center is the average of the far clipping points
-    farcenter *= 0.25;
-    // the extreme near point is the opposite point on the near clipping plane
-    nfar.setValue(-nfar[0], -nfar[1], -1., 1.);
-   	nfar = clip_camcs_matrix*nfar;
-    nearpoint.setValue(nfar[0]/nfar[3], nfar[1]/nfar[3], nfar[2]/nfar[3]);
-    N = nearpoint.dot(nearpoint);
-    e = farpoint[2];
-    s = nearpoint[2];
-    // projection on XY plane for distance to axis computation
-    MT_Point2 farxy(farpoint[0], farpoint[1]);
-    // f is forced positive by construction
-    f = farxy.length();
-    // get corresponding point on the near plane
-    farxy *= s/e;
-    // this formula preserve the sign of n
-    n = f*s/e - MT_Point2(nearpoint[0]-farxy[0], nearpoint[1]-farxy[1]).length();
-    c = MT_Point2(farcenter[0], farcenter[1]).length()/e;
-    // the big formula, it simplifies to (F-N)/(2(e-s)) for the symmetric case
-    z = (F-N)/(2.0*(e-s+c*(f-n)));
-	m_frustum_center = MT_Point3(farcenter[0]*z/e, farcenter[1]*z/e, z);
-	m_frustum_radius = m_frustum_center.distance(farpoint);
-
-#if 0
-	// The most extreme points on the near and far plane. (normalized device coords)
-	MT_Vector4 hnear(1., 1., 0., 1.), hfar(1., 1., 1., 1.);
-	
-	// Transform to hom camera local space
-	hnear = clip_camcs_matrix*hnear;
-	hfar = clip_camcs_matrix*hfar;
-	
-	// Tranform to 3d camera local space.
-	MT_Point3 nearpoint(hnear[0]/hnear[3], hnear[1]/hnear[3], hnear[2]/hnear[3]);
-	MT_Point3 farpoint(hfar[0]/hfar[3], hfar[1]/hfar[3], hfar[2]/hfar[3]);
-	
-	// Compute center
-    // don't use camera data in case the user specifies the matrix directly
-	m_frustum_center = MT_Point3(0., 0.,
-		(nearpoint.dot(nearpoint) - farpoint.dot(farpoint))/(2.0*(nearpoint[2]-farpoint[2] /*m_camdata.m_clipend - m_camdata.m_clipstart*/)));
-	m_frustum_radius = m_frustum_center.distance(farpoint);
-#endif
-
+	if (m_projection_matrix[3][3] == MT_Scalar(0.0)) 
+	{
+		// frustrum projection
+		// detect which of the corner of the far clipping plane is the farthest to the origin
+		MT_Vector4 nfar;    // far point in device normalized coordinate
+		MT_Point3 farpoint; // most extreme far point in camera coordinate
+		MT_Point3 nearpoint;// most extreme near point in camera coordinate
+		MT_Point3 farcenter(0.,0.,0.);// center of far cliping plane in camera coordinate
+		MT_Scalar F=-1.0, N; // square distance of far and near point to origin
+		MT_Scalar f, n;     // distance of far and near point to z axis. f is always > 0 but n can be < 0
+		MT_Scalar e, s;     // far and near clipping distance (<0)
+		MT_Scalar c;        // slope of center line = distance of far clipping center to z axis / far clipping distance
+		MT_Scalar z;        // projection of sphere center on z axis (<0)
+		// tmp value
+		MT_Vector4 npoint(1., 1., 1., 1.);
+		MT_Vector4 hpoint;
+		MT_Point3 point;
+		MT_Scalar len;
+		for (int i=0; i<4; i++)
+		{
+    		hpoint = clip_camcs_matrix*npoint;
+			point.setValue(hpoint[0]/hpoint[3], hpoint[1]/hpoint[3], hpoint[2]/hpoint[3]);
+			len = point.dot(point);
+			if (len > F)
+			{
+				nfar = npoint;
+				farpoint = point;
+				F = len;
+			}
+			// rotate by 90 degree along the z axis to walk through the 4 extreme points of the far clipping plane
+			len = npoint[0];
+			npoint[0] = -npoint[1];
+			npoint[1] = len;
+			farcenter += point;
+		}
+		// the far center is the average of the far clipping points
+		farcenter *= 0.25;
+		// the extreme near point is the opposite point on the near clipping plane
+		nfar.setValue(-nfar[0], -nfar[1], -1., 1.);
+   		nfar = clip_camcs_matrix*nfar;
+		nearpoint.setValue(nfar[0]/nfar[3], nfar[1]/nfar[3], nfar[2]/nfar[3]);
+		// this is a frustrum projection
+		N = nearpoint.dot(nearpoint);
+		e = farpoint[2];
+		s = nearpoint[2];
+		// projection on XY plane for distance to axis computation
+		MT_Point2 farxy(farpoint[0], farpoint[1]);
+		// f is forced positive by construction
+		f = farxy.length();
+		// get corresponding point on the near plane
+		farxy *= s/e;
+		// this formula preserve the sign of n
+		n = f*s/e - MT_Point2(nearpoint[0]-farxy[0], nearpoint[1]-farxy[1]).length();
+		c = MT_Point2(farcenter[0], farcenter[1]).length()/e;
+		// the big formula, it simplifies to (F-N)/(2(e-s)) for the symmetric case
+		z = (F-N)/(2.0*(e-s+c*(f-n)));
+		m_frustum_center = MT_Point3(farcenter[0]*z/e, farcenter[1]*z/e, z);
+		m_frustum_radius = m_frustum_center.distance(farpoint);
+	} 
+	else
+	{
+		// orthographic projection
+		// The most extreme points on the near and far plane. (normalized device coords)
+		MT_Vector4 hnear(1., 1., 1., 1.), hfar(-1., -1., -1., 1.);
+		
+		// Transform to hom camera local space
+		hnear = clip_camcs_matrix*hnear;
+		hfar = clip_camcs_matrix*hfar;
+		
+		// Tranform to 3d camera local space.
+		MT_Point3 nearpoint(hnear[0]/hnear[3], hnear[1]/hnear[3], hnear[2]/hnear[3]);
+		MT_Point3 farpoint(hfar[0]/hfar[3], hfar[1]/hfar[3], hfar[2]/hfar[3]);
+		
+		// just use mediant point
+		m_frustum_center = (farpoint + nearpoint)*0.5;
+		m_frustum_radius = m_frustum_center.distance(farpoint);
+	}
 	// Transform to world space.
 	m_frustum_center = GetCameraToWorld()(m_frustum_center);
 	m_frustum_radius /= fabs(NodeGetWorldScaling()[NodeGetWorldScaling().closestAxis()]);
diff --git a/source/gameengine/Ketsji/KX_Camera.h b/source/gameengine/Ketsji/KX_Camera.h
index d570fac213a..83316972ca0 100644
--- a/source/gameengine/Ketsji/KX_Camera.h
+++ b/source/gameengine/Ketsji/KX_Camera.h
@@ -184,6 +184,8 @@ public:
 
 	/** Gets the aperture. */
 	float				GetLens() const;
+	/** Gets the ortho scale. */
+	float				GetScale() const;
 	/** Gets the near clip distance. */
 	float				GetCameraNear() const;
 	/** Gets the far clip distance. */
diff --git a/source/gameengine/Ketsji/KX_Dome.cpp b/source/gameengine/Ketsji/KX_Dome.cpp
index 077e2ce33b7..9565c53e0bf 100644
--- a/source/gameengine/Ketsji/KX_Dome.cpp
+++ b/source/gameengine/Ketsji/KX_Dome.cpp
@@ -1507,8 +1507,7 @@ void KX_Dome::RotateCamera(KX_Camera* cam, int i)
 
 	MT_Transform camtrans(cam->GetWorldToCamera());
 	MT_Matrix4x4 viewmat(camtrans);
-	m_rasterizer->SetViewMatrix(viewmat, cam->NodeGetWorldPosition(),
-		cam->GetCameraLocation(), cam->GetCameraOrientation());
+	m_rasterizer->SetViewMatrix(viewmat, cam->NodeGetWorldOrientation(), cam->NodeGetWorldPosition(), cam->GetCameraData()->m_perspective);
 	cam->SetModelviewMatrix(viewmat);
 
 	// restore the original orientation
@@ -1914,8 +1913,7 @@ void KX_Dome::RenderDomeFrame(KX_Scene* scene, KX_Camera* cam, int i)
 
 	MT_Transform camtrans(cam->GetWorldToCamera());
 	MT_Matrix4x4 viewmat(camtrans);
-	m_rasterizer->SetViewMatrix(viewmat, cam->NodeGetWorldPosition(),
-		cam->GetCameraLocation(), cam->GetCameraOrientation());
+	m_rasterizer->SetViewMatrix(viewmat, cam->NodeGetWorldOrientation(), cam->NodeGetWorldPosition(), cam->GetCameraData()->m_perspective);
 	cam->SetModelviewMatrix(viewmat);
 
 	scene->CalculateVisibleMeshes(m_rasterizer,cam);
diff --git a/source/gameengine/Ketsji/KX_KetsjiEngine.cpp b/source/gameengine/Ketsji/KX_KetsjiEngine.cpp
index 63db54a296e..9bac0f7d758 100644
--- a/source/gameengine/Ketsji/KX_KetsjiEngine.cpp
+++ b/source/gameengine/Ketsji/KX_KetsjiEngine.cpp
@@ -1093,7 +1093,7 @@ void KX_KetsjiEngine::GetSceneViewport(KX_Scene *scene, KX_Camera* cam, RAS_Rect
 
 		area = userviewport;
 	}
-	else if ( m_overrideCam || (scene->GetName() != m_overrideSceneName) ||  m_overrideCamUseOrtho ) {
+	else if ( !m_overrideCam || (scene->GetName() != m_overrideSceneName) ||  m_overrideCamUseOrtho ) {
 		RAS_FramingManager::ComputeViewport(
 			scene->GetFramingType(),
 			m_canvas->GetDisplayArea(),
@@ -1163,13 +1163,11 @@ void KX_KetsjiEngine::RenderFrame(KX_Scene* scene, KX_Camera* cam)
 {
 	bool override_camera;
 	RAS_Rect viewport, area;
-	float left, right, bottom, top, nearfrust, farfrust, focallength;
-	const float ortho = 100.0;
+	float nearfrust, farfrust, focallength;
 //	KX_Camera* cam = scene->GetActiveCamera();
 	
 	if (!cam)
 		return;
-
 	GetSceneViewport(scene, cam, area, viewport);
 
 	// store the computed viewport in the scene
@@ -1187,19 +1185,24 @@ void KX_KetsjiEngine::RenderFrame(KX_Scene* scene, KX_Camera* cam)
 	override_camera = override_camera && (cam->GetName() == "__default__cam__");
 
 	if (override_camera && m_overrideCamUseOrtho) {
-		MT_CmMatrix4x4 projmat = m_overrideCamProjMat;
-		m_rasterizer->SetProjectionMatrix(projmat);
+		m_rasterizer->SetProjectionMatrix(m_overrideCamProjMat);
+		if (!cam->hasValidProjectionMatrix()) {
+			// needed to get frustrum planes for culling
+			MT_Matrix4x4 projmat;
+			projmat.setValue(m_overrideCamProjMat.getPointer());
+			cam->SetProjectionMatrix(projmat);
+		}
 	} else if (cam->hasValidProjectionMatrix() && !cam->GetViewport() )
 	{
 		m_rasterizer->SetProjectionMatrix(cam->GetProjectionMatrix());
 	} else
 	{
 		RAS_FrameFrustum frustum;
-		float lens = cam->GetLens();
 		bool orthographic = !cam->GetCameraData()->m_perspective;
 		nearfrust = cam->GetCameraNear();
 		farfrust = cam->GetCameraFar();
 		focallength = cam->GetFocalLength();
+		MT_Matrix4x4 projmat;
 
 		if(override_camera) {
 			nearfrust = m_overrideCamNear;
@@ -1207,45 +1210,56 @@ void KX_KetsjiEngine::RenderFrame(KX_Scene* scene, KX_Camera* cam)
 		}
 
 		if (orthographic) {
-			lens *= ortho;
-			nearfrust = (nearfrust + 1.0)*ortho;
-			farfrust *= ortho;
-		}
-		
-		RAS_FramingManager::ComputeFrustum(
-			scene->GetFramingType(),
-			area,
-			viewport,
-			lens,
-			nearfrust,
-			farfrust,
-			frustum
-		);
 
-		left = frustum.x1 * m_cameraZoom;
-		right = frustum.x2 * m_cameraZoom;
-		bottom = frustum.y1 * m_cameraZoom;
-		top = frustum.y2 * m_cameraZoom;
-		nearfrust = frustum.camnear;
-		farfrust = frustum.camfar;
+			RAS_FramingManager::ComputeOrtho(
+				scene->GetFramingType(),
+				area,
+				viewport,
+				cam->GetScale(),
+				nearfrust,
+				farfrust,
+				frustum
+			);
+			if (!cam->GetViewport()) {
+				frustum.x1 *= m_cameraZoom;
+				frustum.x2 *= m_cameraZoom;
+				frustum.y1 *= m_cameraZoom;
+				frustum.y2 *= m_cameraZoom;
+			}
+			projmat = m_rasterizer->GetOrthoMatrix(
+				frustum.x1, frustum.x2, frustum.y1, frustum.y2, frustum.camnear, frustum.camfar);
 
-		MT_Matrix4x4 projmat = m_rasterizer->GetFrustumMatrix(
-			left, right, bottom, top, nearfrust, farfrust, focallength);
+		} else {
+			RAS_FramingManager::ComputeFrustum(
+				scene->GetFramingType(),
+				area,
+				viewport,
+				cam->GetLens(),
+				nearfrust,
+				farfrust,
+				frustum
+			);
 
+			if (!cam->GetViewport()) {
+				frustum.x1 *= m_cameraZoom;
+				frustum.x2 *= m_cameraZoom;
+				frustum.y1 *= m_cameraZoom;
+				frustum.y2 *= m_cameraZoom;
+			}
+			projmat = m_rasterizer->GetFrustumMatrix(
+				frustum.x1, frustum.x2, frustum.y1, frustum.y2, frustum.camnear, frustum.camfar, focallength);
+		}
 		cam->SetProjectionMatrix(projmat);
 		
 		// Otherwise the projection matrix for each eye will be the same...
-		if (m_rasterizer->Stereo())
+		if (!orthographic && m_rasterizer->Stereo())
 			cam->InvalidateProjectionMatrix();
 	}
 
 	MT_Transform camtrans(cam->GetWorldToCamera());
-	if (!cam->GetCameraData()->m_perspective)
-		camtrans.getOrigin()[2] *= ortho;
 	MT_Matrix4x4 viewmat(camtrans);
 	
-	m_rasterizer->SetViewMatrix(viewmat, cam->NodeGetWorldPosition(),
-		cam->GetCameraLocation(), cam->GetCameraOrientation());
+	m_rasterizer->SetViewMatrix(viewmat, cam->NodeGetWorldOrientation(), cam->NodeGetWorldPosition(), cam->GetCameraData()->m_perspective);
 	cam->SetModelviewMatrix(viewmat);
 
 	//redundant, already done in Render()
diff --git a/source/gameengine/Ketsji/KX_Light.cpp b/source/gameengine/Ketsji/KX_Light.cpp
index 7bc982111fe..498eb7262b5 100644
--- a/source/gameengine/Ketsji/KX_Light.cpp
+++ b/source/gameengine/Ketsji/KX_Light.cpp
@@ -160,8 +160,7 @@ void KX_LightObject::BindShadowBuffer(RAS_IRasterizer *ras, KX_Camera *cam, MT_T
 
 	/* setup rasterizer transformations */
 	ras->SetProjectionMatrix(projectionmat);
-	ras->SetViewMatrix(modelviewmat, cam->NodeGetWorldPosition(),
-		cam->GetCameraLocation(), cam->GetCameraOrientation());
+	ras->SetViewMatrix(modelviewmat, cam->NodeGetWorldOrientation(), cam->NodeGetWorldPosition(), cam->GetCameraData()->m_perspective);
 }
 
 void KX_LightObject::UnbindShadowBuffer(RAS_IRasterizer *ras)