diff options
3 files changed, 63 insertions, 13 deletions
diff --git a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js index 43e9313be..56e3148be 100644 --- a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js +++ b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js @@ -101,8 +101,12 @@ export default class JitsiStreamBackgroundEffect { _startFx(e) { switch (e.data.message) { case 'inferenceRun': - this.runInference(e.data.segmentationResult) - this.runPostProcessing() + if (e.data.frameId === this._lastFrameId + 1) { + this._lastFrameId = e.data.frameId + + this.runInference(e.data.segmentationResult) + this.runPostProcessing() + } break case 'loaded': this._loaded = true @@ -243,10 +247,23 @@ export default class JitsiStreamBackgroundEffect { * @return {void} */ _renderMask() { - this.resizeSource() + if (this._frameId < this._lastFrameId) { + console.debug('Fixing frame id, this should not happen', this._frameId, this._lastFrameId) + + this._frameId = this._lastFrameId + } + + // Calculate segmentation data only if the previous one finished + // already. + if (this._loaded && this._frameId === this._lastFrameId) { + this._frameId++ + + this.resizeSource() + } + this._maskFrameTimerWorker.postMessage({ id: SET_TIMEOUT, - timeMs: 1000 / 30, + timeMs: 1000 / this._frameRate, message: 'this._maskFrameTimerWorker', }) } @@ -277,7 +294,7 @@ export default class JitsiStreamBackgroundEffect { this._options.height ) - this._model.postMessage({ message: 'resizeSource', imageData }) + this._model.postMessage({ message: 'resizeSource', imageData, frameId: this._frameId }) } /** @@ -305,6 +322,8 @@ export default class JitsiStreamBackgroundEffect { const { height, frameRate, width } = firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints() + this._frameRate = parseInt(frameRate, 10) + this._segmentationMask = new ImageData(this._options.width, this._options.height) this._segmentationMaskCanvas = document.createElement('canvas') this._segmentationMaskCanvas.width = this._options.width @@ -321,12 +340,36 @@ export default class JitsiStreamBackgroundEffect { this._inputVideoElement.onloadeddata = () => { this._maskFrameTimerWorker.postMessage({ id: SET_TIMEOUT, - timeMs: 1000 / 30, + timeMs: 1000 / this._frameRate, message: 'this._maskFrameTimerWorker', }) + this._inputVideoElement.onloadeddata = null } - return this._outputCanvasElement.captureStream(parseInt(frameRate, 10)) + this._frameId = -1 + this._lastFrameId = -1 + + this._outputStream = this._outputCanvasElement.captureStream(this._frameRate) + + return this._outputStream + } + + updateInputStream() { + const firstVideoTrack = this._stream.getVideoTracks()[0] + const { height, frameRate, width } + = firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints() + + this._frameRate = parseInt(frameRate, 10) + + this._outputStream.getVideoTracks()[0].applyConstraints({ frameRate: this._frameRate }).catch(error => { + console.error('Frame rate could not be adjusted in background effect', error) + }) + + this._inputVideoElement.width = parseInt(width, 10) + this._inputVideoElement.height = parseInt(height, 10) + + this._frameId = -1 + this._lastFrameId = -1 } /** diff --git a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js index 4135f30a2..b96c1ba0e 100644 --- a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js +++ b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js @@ -21,7 +21,7 @@ self.onmessage = (e) => { break case 'resizeSource': if (!self.compiled) return - resizeSource(e.data.imageData) + resizeSource(e.data.imageData, e.data.frameId) break case 'runInference': runInference() @@ -85,21 +85,22 @@ async function makeTFLite(isSimd) { /** * @param {ImageData} imageData the image data from the canvas + * @param {number} frameId the ID of the frame that the image data belongs to */ -function resizeSource(imageData) { +function resizeSource(imageData, frameId) { const inputMemoryOffset = self.tflite._getInputMemoryOffset() / 4 for (let i = 0; i < self.segmentationPixelCount; i++) { self.tflite.HEAPF32[inputMemoryOffset + (i * 3)] = imageData.data[i * 4] / 255 self.tflite.HEAPF32[inputMemoryOffset + (i * 3) + 1] = imageData.data[(i * 4) + 1] / 255 self.tflite.HEAPF32[inputMemoryOffset + (i * 3) + 2] = imageData.data[(i * 4) + 2] / 255 } - runInference() + runInference(frameId) } /** - * + * @param {number} frameId the ID of the frame that the image data belongs to */ -function runInference() { +function runInference(frameId) { self.tflite._runInference() const outputMemoryOffset = self.tflite._getOutputMemoryOffset() / 4 const segmentationMaskData = [] @@ -118,7 +119,7 @@ function runInference() { personExp: Math.exp(person - shift), }) } - self.postMessage({ message: 'inferenceRun', segmentationResult: segmentationMaskData }) + self.postMessage({ message: 'inferenceRun', segmentationResult: segmentationMaskData, frameId }) } // This is needed to make the linter happy, but even if nothing is actually diff --git a/src/utils/media/pipeline/VirtualBackground.js b/src/utils/media/pipeline/VirtualBackground.js index dfb68ea01..1e7039832 100644 --- a/src/utils/media/pipeline/VirtualBackground.js +++ b/src/utils/media/pipeline/VirtualBackground.js @@ -222,6 +222,12 @@ export default class VirtualBackground extends TrackSinkSource { return } + if (newTrack === oldTrack && newTrack !== null) { + this._jitsiStreamBackgroundEffect.updateInputStream() + + return + } + this._stopEffect() if (!newTrack) { |