Merge pull request #6586 from nextcloud/backport/6569/stable23

[stable23] Fix flickering with background blur when changing quality

3 files changed, 63 insertions, 13 deletions

diff --git a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js
index 43e9313be..56e3148be 100644
--- a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js
+++ b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js
@@ -101,8 +101,12 @@ export default class JitsiStreamBackgroundEffect {
 	_startFx(e) {
 		switch (e.data.message) {
 		case 'inferenceRun':
-			this.runInference(e.data.segmentationResult)
-			this.runPostProcessing()
+			if (e.data.frameId === this._lastFrameId + 1) {
+				this._lastFrameId = e.data.frameId
+
+				this.runInference(e.data.segmentationResult)
+				this.runPostProcessing()
+			}
 			break
 		case 'loaded':
 			this._loaded = true
@@ -243,10 +247,23 @@ export default class JitsiStreamBackgroundEffect {
 	 * @return {void}
 	 */
 	_renderMask() {
-		this.resizeSource()
+		if (this._frameId < this._lastFrameId) {
+			console.debug('Fixing frame id, this should not happen', this._frameId, this._lastFrameId)
+
+			this._frameId = this._lastFrameId
+		}
+
+		// Calculate segmentation data only if the previous one finished
+		// already.
+		if (this._loaded && this._frameId === this._lastFrameId) {
+			this._frameId++
+
+			this.resizeSource()
+		}
+
 		this._maskFrameTimerWorker.postMessage({
 			id: SET_TIMEOUT,
-			timeMs: 1000 / 30,
+			timeMs: 1000 / this._frameRate,
 			message: 'this._maskFrameTimerWorker',
 		})
 	}
@@ -277,7 +294,7 @@ export default class JitsiStreamBackgroundEffect {
 			this._options.height
 		)
 
-		this._model.postMessage({ message: 'resizeSource', imageData })
+		this._model.postMessage({ message: 'resizeSource', imageData, frameId: this._frameId })
 	}
 
 	/**
@@ -305,6 +322,8 @@ export default class JitsiStreamBackgroundEffect {
 		const { height, frameRate, width }
             = firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints()
 
+		this._frameRate = parseInt(frameRate, 10)
+
 		this._segmentationMask = new ImageData(this._options.width, this._options.height)
 		this._segmentationMaskCanvas = document.createElement('canvas')
 		this._segmentationMaskCanvas.width = this._options.width
@@ -321,12 +340,36 @@ export default class JitsiStreamBackgroundEffect {
 		this._inputVideoElement.onloadeddata = () => {
 			this._maskFrameTimerWorker.postMessage({
 				id: SET_TIMEOUT,
-				timeMs: 1000 / 30,
+				timeMs: 1000 / this._frameRate,
 				message: 'this._maskFrameTimerWorker',
 			})
+			this._inputVideoElement.onloadeddata = null
 		}
 
-		return this._outputCanvasElement.captureStream(parseInt(frameRate, 10))
+		this._frameId = -1
+		this._lastFrameId = -1
+
+		this._outputStream = this._outputCanvasElement.captureStream(this._frameRate)
+
+		return this._outputStream
+	}
+
+	updateInputStream() {
+		const firstVideoTrack = this._stream.getVideoTracks()[0]
+		const { height, frameRate, width }
+            = firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints()
+
+		this._frameRate = parseInt(frameRate, 10)
+
+		this._outputStream.getVideoTracks()[0].applyConstraints({ frameRate: this._frameRate }).catch(error => {
+			console.error('Frame rate could not be adjusted in background effect', error)
+		})
+
+		this._inputVideoElement.width = parseInt(width, 10)
+		this._inputVideoElement.height = parseInt(height, 10)
+
+		this._frameId = -1
+		this._lastFrameId = -1
 	}
 
 	/**
diff --git a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js
index 4135f30a2..b96c1ba0e 100644
--- a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js
+++ b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js
@@ -21,7 +21,7 @@ self.onmessage = (e) => {
 		break
 	case 'resizeSource':
 		if (!self.compiled) return
-		resizeSource(e.data.imageData)
+		resizeSource(e.data.imageData, e.data.frameId)
 		break
 	case 'runInference':
 		runInference()
@@ -85,21 +85,22 @@ async function makeTFLite(isSimd) {
 
 /**
  * @param {ImageData} imageData the image data from the canvas
+ * @param {number} frameId the ID of the frame that the image data belongs to
  */
-function resizeSource(imageData) {
+function resizeSource(imageData, frameId) {
 	const inputMemoryOffset = self.tflite._getInputMemoryOffset() / 4
 	for (let i = 0; i < self.segmentationPixelCount; i++) {
 		self.tflite.HEAPF32[inputMemoryOffset + (i * 3)] = imageData.data[i * 4] / 255
 		self.tflite.HEAPF32[inputMemoryOffset + (i * 3) + 1] = imageData.data[(i * 4) + 1] / 255
 		self.tflite.HEAPF32[inputMemoryOffset + (i * 3) + 2] = imageData.data[(i * 4) + 2] / 255
 	}
-	runInference()
+	runInference(frameId)
 }
 
 /**
- *
+ * @param {number} frameId the ID of the frame that the image data belongs to
  */
-function runInference() {
+function runInference(frameId) {
 	self.tflite._runInference()
 	const outputMemoryOffset = self.tflite._getOutputMemoryOffset() / 4
 	const segmentationMaskData = []
@@ -118,7 +119,7 @@ function runInference() {
 			personExp: Math.exp(person - shift),
 		})
 	}
-	self.postMessage({ message: 'inferenceRun', segmentationResult: segmentationMaskData })
+	self.postMessage({ message: 'inferenceRun', segmentationResult: segmentationMaskData, frameId })
 }
 
 // This is needed to make the linter happy, but even if nothing is actually
diff --git a/src/utils/media/pipeline/VirtualBackground.js b/src/utils/media/pipeline/VirtualBackground.js
index dfb68ea01..1e7039832 100644
--- a/src/utils/media/pipeline/VirtualBackground.js
+++ b/src/utils/media/pipeline/VirtualBackground.js
@@ -222,6 +222,12 @@ export default class VirtualBackground extends TrackSinkSource {
 			return
 		}
 
+		if (newTrack === oldTrack && newTrack !== null) {
+			this._jitsiStreamBackgroundEffect.updateInputStream()
+
+			return
+		}
+
 		this._stopEffect()
 
 		if (!newTrack) {