BGE: add audio/video synchronization capability to VideoTexture

Add optional parameter to VideoTexture.Texture refresh() method
to specify timestamp (in seconds from start of movie) of the frame
to be loaded. This value is passed down to image source and for
VideoFFmpeg source, it is used instead of current time to load
the frame from the video file.

When combined with an audio actuator, it can be used to synchronize
the sound and the image: specify the same video file in the sound
actuator and use the KX_SoundActuator time attribute as timestamp
to refresh: the frame corresponding to the sound will be loaded:

GameLogic.video.refresh(True, soundAct.time)

11 files changed, 46 insertions, 26 deletions

diff --git a/source/gameengine/VideoTexture/ImageBase.cpp b/source/gameengine/VideoTexture/ImageBase.cpp
index 0740afed2c6..1c5425c932c 100644
--- a/source/gameengine/VideoTexture/ImageBase.cpp
+++ b/source/gameengine/VideoTexture/ImageBase.cpp
@@ -71,7 +71,7 @@ bool ImageBase::release (void)
 
 
 // get image
-unsigned int * ImageBase::getImage (unsigned int texId)
+unsigned int * ImageBase::getImage (unsigned int texId, double ts)
 {
 	// if image is not available
 	if (!m_avail)
@@ -82,12 +82,12 @@ unsigned int * ImageBase::getImage (unsigned int texId)
 			// get images from sources
 			for (ImageSourceList::iterator it = m_sources.begin(); it != m_sources.end(); ++it)
 				// get source image
-				(*it)->getImage();
+				(*it)->getImage(ts);
 			// init image
 			init(m_sources[0]->getSize()[0], m_sources[0]->getSize()[1]);
 		}
 		// calculate new image
-		calcImage(texId);
+		calcImage(texId, ts);
 	}
 	// if image is available, return it, otherwise NULL
 	return m_avail ? m_image : NULL;
@@ -305,12 +305,12 @@ void ImageSource::setSource (PyImage * source)
 
 
 // get image from source
-unsigned int * ImageSource::getImage (void)
+unsigned int * ImageSource::getImage (double ts)
 {
 	// if source is available
 	if (m_source != NULL)
 		// get image from source
-		m_image = m_source->m_image->getImage();
+		m_image = m_source->m_image->getImage(0, ts);
 	// otherwise reset buffer
 	else
 		m_image = NULL;
diff --git a/source/gameengine/VideoTexture/ImageBase.h b/source/gameengine/VideoTexture/ImageBase.h
index 138580ce701..70b1929b91d 100644
--- a/source/gameengine/VideoTexture/ImageBase.h
+++ b/source/gameengine/VideoTexture/ImageBase.h
@@ -54,7 +54,7 @@ public:
 	virtual bool release (void);
 
 	/// get image
-	unsigned int * getImage (unsigned int texId = 0);
+	unsigned int * getImage (unsigned int texId = 0, double timestamp=-1.0);
 	/// get image size
 	short * getSize (void) { return m_size; }
 	/// get image buffer size
@@ -123,7 +123,7 @@ protected:
 	bool checkSourceSizes (void);
 
 	/// calculate image from sources and set its availability
-	virtual void calcImage (unsigned int texId) {}
+	virtual void calcImage (unsigned int texId, double ts) {}
 
 	/// perform loop detection
 	bool loopDetect (ImageBase * img);
@@ -269,7 +269,7 @@ public:
 	void setSource (PyImage * source);
 
 	/// get image from source
-	unsigned int * getImage (void);
+	unsigned int * getImage (double ts=-1.0);
 	/// get buffered image
 	unsigned int * getImageBuf (void) { return m_image; }
 	/// refresh source
diff --git a/source/gameengine/VideoTexture/ImageMix.cpp b/source/gameengine/VideoTexture/ImageMix.cpp
index 2560467c3db..a4a61e7f55a 100644
--- a/source/gameengine/VideoTexture/ImageMix.cpp
+++ b/source/gameengine/VideoTexture/ImageMix.cpp
@@ -63,7 +63,7 @@ ExceptionID ImageSizesNotMatch;
 ExpDesc ImageSizesNotMatchDesc (ImageSizesNotMatch, "Image sizes of sources are different");
 
 // calculate image from sources and set its availability
-void ImageMix::calcImage (unsigned int texId)
+void ImageMix::calcImage (unsigned int texId, double ts)
 {
 	// check source sizes
 	if (!checkSourceSizes()) THRWEXCP(ImageSizesNotMatch, S_OK);
diff --git a/source/gameengine/VideoTexture/ImageMix.h b/source/gameengine/VideoTexture/ImageMix.h
index b4842bd6b40..47bd644860f 100644
--- a/source/gameengine/VideoTexture/ImageMix.h
+++ b/source/gameengine/VideoTexture/ImageMix.h
@@ -78,7 +78,7 @@ protected:
 	virtual ImageSource * newSource (const char * id) { return new ImageSourceMix(id); }
 
 	/// calculate image from sources and set its availability
-	virtual void calcImage (unsigned int texId);
+	virtual void calcImage (unsigned int texId, double ts);
 };
 
 
diff --git a/source/gameengine/VideoTexture/ImageRender.cpp b/source/gameengine/VideoTexture/ImageRender.cpp
index 62594f9552c..8fd0a8968f8 100644
--- a/source/gameengine/VideoTexture/ImageRender.cpp
+++ b/source/gameengine/VideoTexture/ImageRender.cpp
@@ -92,7 +92,7 @@ void ImageRender::setBackground (int red, int green, int blue, int alpha)
 
 
 // capture image from viewport
-void ImageRender::calcImage (unsigned int texId)
+void ImageRender::calcImage (unsigned int texId, double ts)
 {
     if (m_rasterizer->GetDrawingMode() != RAS_IRasterizer::KX_TEXTURED ||   // no need for texture
         m_camera->GetViewport() ||        // camera must be inactive
@@ -105,7 +105,7 @@ void ImageRender::calcImage (unsigned int texId)
     // render the scene from the camera
     Render();
 	// get image from viewport
-	ImageViewport::calcImage(texId);
+	ImageViewport::calcImage(texId, ts);
     // restore OpenGL state
     m_canvas->EndFrame();
 }
diff --git a/source/gameengine/VideoTexture/ImageRender.h b/source/gameengine/VideoTexture/ImageRender.h
index c94e2f1e718..d49544ce42a 100644
--- a/source/gameengine/VideoTexture/ImageRender.h
+++ b/source/gameengine/VideoTexture/ImageRender.h
@@ -90,7 +90,7 @@ protected:
 
 
 	/// render 3d scene to image
-	virtual void calcImage (unsigned int texId);
+	virtual void calcImage (unsigned int texId, double ts);
 
 	void Render();
 	void SetupRenderFrame(KX_Scene *scene, KX_Camera* cam);
diff --git a/source/gameengine/VideoTexture/ImageViewport.cpp b/source/gameengine/VideoTexture/ImageViewport.cpp
index 691a983970a..c39173a96f9 100644
--- a/source/gameengine/VideoTexture/ImageViewport.cpp
+++ b/source/gameengine/VideoTexture/ImageViewport.cpp
@@ -105,7 +105,7 @@ void ImageViewport::setPosition (GLint * pos)
 
 
 // capture image from viewport
-void ImageViewport::calcImage (unsigned int texId)
+void ImageViewport::calcImage (unsigned int texId, double ts)
 {
 	// if scale was changed
 	if (m_scaleChange)
diff --git a/source/gameengine/VideoTexture/ImageViewport.h b/source/gameengine/VideoTexture/ImageViewport.h
index 0449249cf95..49db56bcf19 100644
--- a/source/gameengine/VideoTexture/ImageViewport.h
+++ b/source/gameengine/VideoTexture/ImageViewport.h
@@ -81,7 +81,7 @@ protected:
 	bool m_texInit;
 
 	/// capture image from viewport
-	virtual void calcImage (unsigned int texId);
+	virtual void calcImage (unsigned int texId, double ts);
 
 	/// get viewport size
 	GLint * getViewportSize (void) { return m_viewport + 2; }
diff --git a/source/gameengine/VideoTexture/Texture.cpp b/source/gameengine/VideoTexture/Texture.cpp
index 04b39f0b05c..09c95b929c5 100644
--- a/source/gameengine/VideoTexture/Texture.cpp
+++ b/source/gameengine/VideoTexture/Texture.cpp
@@ -279,7 +279,9 @@ PyObject * Texture_refresh (Texture * self, PyObject * args)
 {
 	// get parameter - refresh source
 	PyObject * param;
-	if (!PyArg_ParseTuple(args, "O:refresh", &param) || !PyBool_Check(param))
+	double ts = -1.0;
+
+	if (!PyArg_ParseTuple(args, "O|d:refresh", &param, &ts) || !PyBool_Check(param))
 	{
 		// report error
 		PyErr_SetString(PyExc_TypeError, "The value must be a bool");
@@ -315,7 +317,7 @@ PyObject * Texture_refresh (Texture * self, PyObject * args)
 				}
 
 				// get texture
-				unsigned int * texture = self->m_source->m_image->getImage(self->m_actTex);
+				unsigned int * texture = self->m_source->m_image->getImage(self->m_actTex, ts);
 				// if texture is available
 				if (texture != NULL)
 				{
diff --git a/source/gameengine/VideoTexture/VideoFFmpeg.cpp b/source/gameengine/VideoTexture/VideoFFmpeg.cpp
index f21555a95c9..9136e619288 100644
--- a/source/gameengine/VideoTexture/VideoFFmpeg.cpp
+++ b/source/gameengine/VideoTexture/VideoFFmpeg.cpp
@@ -54,7 +54,7 @@ VideoFFmpeg::VideoFFmpeg (HRESULT * hRslt) : VideoBase(),
 m_codec(NULL), m_formatCtx(NULL), m_codecCtx(NULL), 
 m_frame(NULL), m_frameDeinterlaced(NULL), m_frameRGB(NULL), m_imgConvertCtx(NULL),
 m_deinterlace(false), m_preseek(0),	m_videoStream(-1), m_baseFrameRate(25.0),
-m_lastFrame(-1),  m_eof(false), m_curPosition(-1), m_startTime(0), 
+m_lastFrame(-1),  m_eof(false), m_externTime(false), m_curPosition(-1), m_startTime(0), 
 m_captWidth(0), m_captHeight(0), m_captRate(0.f), m_isImage(false),
 m_isThreaded(false), m_stopThread(false), m_cacheStarted(false)
 {
@@ -723,22 +723,37 @@ void VideoFFmpeg::setFrameRate (float rate)
 
 
 // image calculation
-void VideoFFmpeg::calcImage (unsigned int texId)
+void VideoFFmpeg::calcImage (unsigned int texId, double ts)
 {
-	loadFrame();
+	loadFrame(ts);
 }
 
 
 // load frame from video
-void VideoFFmpeg::loadFrame (void)
+void VideoFFmpeg::loadFrame (double ts)
 {
 	if (m_status == SourcePlaying)
 	{
 		// get actual time
 		double startTime = PIL_check_seconds_timer();
-		if (m_lastFrame == -1 && !m_isFile)
-			m_startTime = startTime;
-		double actTime = startTime - m_startTime;
+		double actTime;
+		if (m_isFile && ts >= 0.0)
+		{
+			// allow setting timestamp only when not streaming
+			actTime = ts;
+			if (m_eof && actTime * actFrameRate() < m_lastFrame) 
+			{
+				// user is asking to rewind while the playback is already finished in the cache.
+				// we must clean the cache otherwise the eof condition will prevent any further reading.
+				stopCache();
+			}
+		}
+		else
+		{
+			if (m_lastFrame == -1 && !m_isFile)
+				m_startTime = startTime;
+			actTime = startTime - m_startTime;
+		}
 		// if video has ended
 		if (m_isFile && actTime * m_frameRate >= m_range[1])
 		{
diff --git a/source/gameengine/VideoTexture/VideoFFmpeg.h b/source/gameengine/VideoTexture/VideoFFmpeg.h
index b56984588c1..355c58c496b 100644
--- a/source/gameengine/VideoTexture/VideoFFmpeg.h
+++ b/source/gameengine/VideoTexture/VideoFFmpeg.h
@@ -129,6 +129,9 @@ protected:
 	/// end of file reached
 	bool m_eof;
 
+	/// flag to indicate that time is coming from application
+	bool m_externTime;
+
 	/// current file pointer position in file expressed in frame number
 	long m_curPosition;
 
@@ -154,10 +157,10 @@ protected:
 	STR_String m_imageName;
 
 	/// image calculation
-	virtual void calcImage (unsigned int texId);
+	virtual void calcImage (unsigned int texId, double ts);
 
 	/// load frame from video
-	void loadFrame (void);
+	void loadFrame (double ts);
 
 	/// set actual position
 	void setPositions (void);