Merge pull request #4985 from nextcloud/replace-blur-with-average-color-in-video-backgrounds

Replace blur with average color in video backgrounds

10 files changed, 49 insertions, 329 deletions

diff --git a/lib/Listener/CSPListener.php b/lib/Listener/CSPListener.php
index d430a68bb..491a0d6fe 100644
--- a/lib/Listener/CSPListener.php
+++ b/lib/Listener/CSPListener.php
@@ -50,8 +50,6 @@ class CSPListener implements IEventListener {
 			$csp->addAllowedConnectDomain($server);
 		}
 
-		$csp->addAllowedWorkerSrcDomain('\'self\'');
-
 		$event->addPolicy($csp);
 	}
 }
diff --git a/package-lock.json b/package-lock.json
index 447d331d9..4154abdf4 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -7333,6 +7333,11 @@
 				"simple-swizzle": "^0.2.2"
 			}
 		},
+		"color.js": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/color.js/-/color.js-1.2.0.tgz",
+			"integrity": "sha512-0ajlNgWWOR7EK9N6l2h0YKsZPzMCLQG5bheCoTGpGfhkR8tB5eQNItdua1oFHDTeq9JKgSzQJqo+Gp3V/xW+Lw=="
+		},
 		"colorette": {
 			"version": "1.2.1",
 			"resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz",
diff --git a/package.json b/package.json
index 9fcac3984..cce26ecf1 100644
--- a/package.json
+++ b/package.json
@@ -30,6 +30,7 @@
 		"@nextcloud/vue": "^3.5.3",
 		"@nextcloud/vue-dashboard": "^1.0.1",
 		"attachmediastream": "^2.1.0",
+		"color.js": "^1.2.0",
 		"crypto-js": "^4.0.0",
 		"debounce": "^1.2.0",
 		"emoji-regex": "^9.2.0",
diff --git a/src/components/CallView/Grid/Grid.vue b/src/components/CallView/Grid/Grid.vue
index c56262518..c048850ed 100644
--- a/src/components/CallView/Grid/Grid.vue
+++ b/src/components/CallView/Grid/Grid.vue
@@ -72,7 +72,6 @@
 									:is-selected="isSelected(callParticipantModel)"
 									:fit-video="false"
 									:video-container-aspect-ratio="videoContainerAspectRatio"
-									:video-background-blur="videoBackgroundBlur"
 									:shared-data="sharedDatas[callParticipantModel.attributes.peerId]"
 									@click-video="handleClickVideo($event, callParticipantModel.attributes.peerId)" />
 							</template>
@@ -439,11 +438,6 @@ export default {
 			}
 		},
 
-		// Blur radius for each background in the grid
-		videoBackgroundBlur() {
-			return this.$store.getters.getBlurRadius(this.videoWidth, this.videoHeight)
-		},
-
 		stripeOpen() {
 			return this.$store.getters.isStripeOpen
 		},
diff --git a/src/components/CallView/shared/Video.vue b/src/components/CallView/shared/Video.vue
index 2d5c9f3fb..acaec6009 100644
--- a/src/components/CallView/shared/Video.vue
+++ b/src/components/CallView/shared/Video.vue
@@ -50,8 +50,7 @@
 				<template v-if="participantUserId">
 					<VideoBackground
 						:display-name="participantName"
-						:user="participantUserId"
-						:grid-blur="videoBackgroundBlur" />
+						:user="participantUserId" />
 					<Avatar
 						:size="avatarSize"
 						:disable-menu="true"
@@ -63,8 +62,7 @@
 				</template>
 				<template v-else>
 					<VideoBackground
-						:display-name="participantName"
-						:grid-blur="videoBackgroundBlur" />
+						:display-name="participantName" />
 					<div
 						:class="guestAvatarClass"
 						class="avatar guest">
@@ -158,11 +156,6 @@ export default {
 			type: Boolean,
 			default: false,
 		},
-		// Calculated once in the grid component for each video background
-		videoBackgroundBlur: {
-			type: Number,
-			default: 0,
-		},
 	},
 
 	computed: {
diff --git a/src/components/CallView/shared/VideoBackground.vue b/src/components/CallView/shared/VideoBackground.vue
index 8ff2878ba..4a0448d2d 100644
--- a/src/components/CallView/shared/VideoBackground.vue
+++ b/src/components/CallView/shared/VideoBackground.vue
@@ -22,34 +22,20 @@
 <template>
 	<div class="video-backgroundbackground">
 		<div
-			ref="darkener"
-			class="darken">
-			<ResizeObserver
-				v-if="gridBlur === 0"
-				class="observer"
-				@notify="setBlur" />
-		</div>
-		<img
-			v-if="hasPicture"
-			ref="backgroundImage"
-			:src="backgroundImage"
-			:style="backgroundStyle"
-			class="video-background__picture"
-			alt="">
-		<div v-else
 			:style="{'background-color': backgroundColor }"
 			class="video-background" />
+		<div
+			ref="darkener"
+			class="darken" />
 	</div>
 </template>
 
 <script>
+import { average } from 'color.js'
 import axios from '@nextcloud/axios'
 import usernameToColor from '@nextcloud/vue/dist/Functions/usernameToColor'
 import { generateUrl } from '@nextcloud/router'
-import { ResizeObserver } from 'vue-resize'
 import { getBuilder } from '@nextcloud/browser-storage'
-import browserCheck from '../../../mixins/browserCheck'
-import blur from '../../../utils/imageBlurrer'
 
 const browserStorage = getBuilder('nextcloud').persist().build()
 
@@ -68,13 +54,6 @@ function setUserHasAvatar(userId, flag) {
 
 export default {
 	name: 'VideoBackground',
-	components: {
-		ResizeObserver,
-	},
-
-	mixins: [
-		browserCheck,
-	],
 
 	props: {
 		displayName: {
@@ -85,27 +64,27 @@ export default {
 			type: String,
 			default: '',
 		},
-		gridBlur: {
-			type: Number,
-			default: 0,
-		},
 	},
 
 	data() {
 		return {
 			hasPicture: false,
-			useCssBlurFilter: true,
-			blur: 0,
-			blurredBackgroundImage: null,
-			blurredBackgroundImageCache: {},
-			blurredBackgroundImageSource: null,
-			pendingGenerateBlurredBackgroundImageCount: 0,
-			isDestroyed: false,
 		}
 	},
 
 	computed: {
+		backgroundImageAverageColor() {
+			if (!this.backgroundImageUrl) {
+				return ''
+			}
+
+			return this.$store.getters.getCachedBackgroundImageAverageColor(this.backgroundImageUrl)
+		},
 		backgroundColor() {
+			if (this.hasPicture) {
+				return this.backgroundImageAverageColor
+			}
+
 			// If the prop is empty. We're not checking for the default value
 			// because the user's displayName might be '?'
 			if (!this.displayName) {
@@ -115,9 +94,6 @@ export default {
 				return `rgb(${color.r}, ${color.g}, ${color.b})`
 			}
 		},
-		backgroundImage() {
-			return this.useCssBlurFilter ? this.backgroundImageUrl : this.blurredBackgroundImage
-		},
 		backgroundImageUrl() {
 			if (!this.user) {
 				return null
@@ -125,78 +101,32 @@ export default {
 
 			return generateUrl(`avatar/${this.user}/300`)
 		},
-		backgroundBlur() {
-			return this.gridBlur ? this.gridBlur : this.blur
-		},
-		backgroundStyle() {
-			if (!this.useCssBlurFilter) {
-				return {}
-			}
-
-			return {
-				filter: `blur(${this.backgroundBlur}px)`,
-			}
-		},
-		// Special computed property to combine the properties that should be
-		// watched to set (or not) the blurred background image source.
-		backgroundImageUrlToBlur() {
-			if (this.useCssBlurFilter) {
-				return null
-			}
-
-			return this.backgroundImageUrl
-		},
-		// Special computed property to combine the properties that should be
-		// watched to generate (or not) the blurred background image.
-		generatedBackgroundBlur() {
-			if (!this.hasPicture || this.useCssBlurFilter) {
-				return false
-			}
-
-			if (!this.blurredBackgroundImageSource) {
-				return false
-			}
-
-			return this.backgroundBlur
-		},
 	},
 
 	watch: {
-		backgroundImageUrlToBlur: {
+		backgroundImageUrl: {
 			immediate: true,
 			handler() {
-				this.blurredBackgroundImageSource = null
-
-				if (!this.backgroundImageUrlToBlur) {
+				if (!this.backgroundImageUrl) {
 					return
 				}
 
-				const image = new Image()
-				image.onload = () => {
-					createImageBitmap(image).then(imageBitmap => {
-						this.blurredBackgroundImageSource = imageBitmap
-					})
-				}
-				image.src = this.backgroundImageUrlToBlur
-			},
-		},
-		generatedBackgroundBlur: {
-			immediate: true,
-			handler() {
-				if (this.generatedBackgroundBlur === false) {
+				if (this.backgroundImageAverageColor) {
+					// Already calculated, no need to do it again.
 					return
 				}
 
-				this.generateBlurredBackgroundImage()
+				average(this.backgroundImageUrl, { format: 'hex' }).then(color => {
+					this.$store.dispatch('setCachedBackgroundImageAverageColor', {
+						videoBackgroundId: this.backgroundImageUrl,
+						backgroundImageAverageColor: color,
+					})
+				})
 			},
 		},
 	},
 
 	async beforeMount() {
-		if (this.isChrome) {
-			this.useCssBlurFilter = false
-		}
-
 		if (!this.user) {
 			return
 		}
@@ -217,81 +147,6 @@ export default {
 			console.debug(exception)
 		}
 	},
-
-	async mounted() {
-		if (!this.gridBlur) {
-			// Initialise blur
-			this.setBlur({
-				width: this.$refs['darkener'].clientWidth,
-				height: this.$refs['darkener'].clientHeight,
-			})
-		}
-	},
-
-	beforeDestroy() {
-		this.isDestroyed = true
-	},
-
-	methods: {
-		// Calculate the background blur based on the height of the background element
-		setBlur({ width, height }) {
-			this.blur = this.$store.getters.getBlurRadius(width, height)
-		},
-
-		generateBlurredBackgroundImage() {
-			// Reset image source so the width and height are adjusted to
-			// the element rather than to the previous image being shown.
-			this.$refs.backgroundImage.src = ''
-
-			let width = this.$refs.backgroundImage.width
-			let height = this.$refs.backgroundImage.height
-
-			// Restore the current background so it is shown instead of an empty
-			// background while the new one is being generated.
-			this.$refs.backgroundImage.src = this.blurredBackgroundImage
-
-			const sourceAspectRatio = this.blurredBackgroundImageSource.width / this.blurredBackgroundImageSource.height
-			const canvasAspectRatio = width / height
-
-			if (canvasAspectRatio > sourceAspectRatio) {
-				height = width / sourceAspectRatio
-			} else if (canvasAspectRatio < sourceAspectRatio) {
-				width = height * sourceAspectRatio
-			}
-
-			const cacheId = this.backgroundImageUrl + '-' + width + '-' + height + '-' + this.backgroundBlur
-			if (this.blurredBackgroundImageCache[cacheId]) {
-				this.blurredBackgroundImage = this.blurredBackgroundImageCache[cacheId]
-
-				return
-			}
-
-			if (this.pendingGenerateBlurredBackgroundImageCount) {
-				this.pendingGenerateBlurredBackgroundImageCount++
-
-				return
-			}
-
-			this.pendingGenerateBlurredBackgroundImageCount = 1
-
-			blur(this.blurredBackgroundImageSource, width, height, this.backgroundBlur).then(image => {
-				if (this.isDestroyed) {
-					return
-				}
-
-				this.blurredBackgroundImage = image
-				this.blurredBackgroundImageCache[cacheId] = this.blurredBackgroundImage
-
-				const generateBlurredBackgroundImageCalledAgain = this.pendingGenerateBlurredBackgroundImageCount > 1
-
-				this.pendingGenerateBlurredBackgroundImageCount = 0
-
-				if (generateBlurredBackgroundImageCalledAgain) {
-					this.generateBlurredBackgroundImage()
-				}
-			})
-		},
-	},
 }
 </script>
 
diff --git a/src/store/callViewStore.js b/src/store/callViewStore.js
index 5f67569bd..4af4c89d9 100644
--- a/src/store/callViewStore.js
+++ b/src/store/callViewStore.js
@@ -33,8 +33,8 @@ const state = {
 	lastIsStripeOpen: null,
 	presentationStarted: false,
 	selectedVideoPeerId: null,
-	videoBackgroundBlur: 1,
 	participantRaisedHands: {},
+	backgroundImageAverageColorCache: {},
 }
 
 const getters = {
@@ -46,19 +46,15 @@ const getters = {
 	selectedVideoPeerId: (state) => {
 		return state.selectedVideoPeerId
 	},
-	/**
-	 * @param {object} state the width and height to calculate the radius from
-	 * @returns {number} the blur radius to use, in pixels
-	 */
-	getBlurRadius: (state) => (width, height) => {
-		return (width * height * state.videoBackgroundBlur) / 1000
-	},
 	getParticipantRaisedHand: (state) => (sessionId) => {
 		return state.participantRaisedHands[sessionId] || { state: false, timestamp: null }
 	},
 	isParticipantRaisedHand: (state) => (sessionId) => {
 		return state.participantRaisedHands[sessionId]?.state
 	},
+	getCachedBackgroundImageAverageColor: (state) => (videoBackgroundId) => {
+		return state.backgroundImageAverageColorCache[videoBackgroundId]
+	},
 }
 
 const mutations = {
@@ -94,6 +90,12 @@ const mutations = {
 	clearParticipantHandRaised(state) {
 		state.participantRaisedHands = {}
 	},
+	setCachedBackgroundImageAverageColor(state, { videoBackgroundId, backgroundImageAverageColor }) {
+		Vue.set(state.backgroundImageAverageColorCache, videoBackgroundId, backgroundImageAverageColor)
+	},
+	clearBackgroundImageAverageColorCache(state) {
+		state.backgroundImageAverageColorCache = {}
+	},
 }
 
 const actions = {
@@ -118,6 +120,8 @@ const actions = {
 	leaveCall(context) {
 		// clear raised hands as they were specific to the call
 		context.commit('clearParticipantHandRaised')
+
+		context.commit('clearBackgroundImageAverageColorCache')
 	},
 
 	/**
@@ -152,6 +156,10 @@ const actions = {
 		context.commit('setParticipantHandRaised', { sessionId, raisedHand })
 	},
 
+	setCachedBackgroundImageAverageColor(context, { videoBackgroundId, backgroundImageAverageColor }) {
+		context.commit('setCachedBackgroundImageAverageColor', { videoBackgroundId, backgroundImageAverageColor })
+	},
+
 	/**
 	 * Starts presentation mode.
 	 *
diff --git a/src/utils/imageBlurrer.js b/src/utils/imageBlurrer.js
deleted file mode 100644
index 2a29e27d1..000000000
--- a/src/utils/imageBlurrer.js
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- *
- * @copyright Copyright (c) 2020, Daniel Calviño Sánchez (danxuliu@gmail.com)
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-import { generateFilePath } from '@nextcloud/router'
-
-let worker
-
-const pendingResults = {}
-let pendingResultsNextId = 0
-
-function loadWorker() {
-	try {
-		worker = new Worker(generateFilePath('spreed', '', 'js/image-blurrer-worker.js'))
-		worker.onmessage = function(message) {
-			const pendingResult = pendingResults[message.data.id]
-			if (!pendingResult) {
-				console.debug('No pending result for blurring image with id ' + message.data.id)
-
-				return
-			}
-
-			pendingResult(message.data.blurredImageAsDataUrl)
-
-			delete pendingResults[message.data.id]
-		}
-	} catch (exception) {
-		worker = null
-		console.error('Image blurrer worker could not be loaded', exception)
-	}
-}
-
-function blurSync(image, width, height, blurRadius) {
-	return new Promise((resolve, reject) => {
-		const canvas = document.createElement('canvas')
-		canvas.width = width
-		canvas.height = height
-
-		const context = canvas.getContext('2d')
-		context.filter = `blur(${blurRadius}px)`
-		context.drawImage(image, 0, 0, canvas.width, canvas.height)
-
-		resolve(canvas.toDataURL())
-	})
-}
-
-export default function blur(image, width, height, blurRadius) {
-	if (typeof OffscreenCanvas === 'undefined') {
-		return blurSync(image, width, height, blurRadius)
-	}
-
-	if (worker === undefined) {
-		loadWorker()
-	}
-
-	if (!worker) {
-		return blurSync(image, width, height, blurRadius)
-	}
-
-	const id = pendingResultsNextId
-
-	pendingResultsNextId++
-
-	return new Promise((resolve, reject) => {
-		pendingResults[id] = resolve
-
-		worker.postMessage({
-			id: id,
-			image: image,
-			width: width,
-			height: height,
-			blurRadius: blurRadius,
-		})
-	})
-}
diff --git a/src/utils/imageBlurrerWorker.js b/src/utils/imageBlurrerWorker.js
deleted file mode 100644
index 756da1ff0..000000000
--- a/src/utils/imageBlurrerWorker.js
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- *
- * @copyright Copyright (c) 2020, Daniel Calviño Sánchez (danxuliu@gmail.com)
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-const fileReaderSync = new global.FileReaderSync()
-
-onmessage = function(message) {
-	const offscreenCanvas = new OffscreenCanvas(message.data.width, message.data.height)
-
-	const context = offscreenCanvas.getContext('2d')
-	context.filter = `blur(${message.data.blurRadius}px)`
-	context.drawImage(message.data.image, 0, 0, offscreenCanvas.width, offscreenCanvas.height)
-
-	offscreenCanvas.convertToBlob().then(blob => {
-		postMessage({
-			id: message.data.id,
-			blurredImageAsDataUrl: fileReaderSync.readAsDataURL(blob),
-		})
-	})
-}
diff --git a/webpack.common.js b/webpack.common.js
index fd5e11bbd..959514bb7 100644
--- a/webpack.common.js
+++ b/webpack.common.js
@@ -7,11 +7,6 @@ module.exports = {
 	entry: {
 		'admin-settings': path.join(__dirname, 'src', 'mainAdminSettings.js'),
 		'collections': path.join(__dirname, 'src', 'collections.js'),
-		// There is a "worker-loader" plugin for Webpack, but I was not able to
-		// get it to work ("publicPath" uses "output.publicPath" rather than the
-		// one set in the plugin
-		// https://github.com/webpack-contrib/worker-loader/issues/281).
-		'image-blurrer-worker': path.join(__dirname, 'src', 'utils/imageBlurrerWorker.js'),
 		'talk': path.join(__dirname, 'src', 'main.js'),
 		'talk-files-sidebar': [
 			path.join(__dirname, 'src', 'mainFilesSidebar.js'),