diff options
-rw-r--r-- | src/components/CallView/CallView.vue | 32 | ||||
-rw-r--r-- | src/constants.js | 5 | ||||
-rw-r--r-- | src/utils/webrtc/models/CallParticipantModel.js | 18 | ||||
-rw-r--r-- | src/utils/webrtc/simplewebrtc/peer.js | 266 | ||||
-rw-r--r-- | src/utils/webrtc/simplewebrtc/simplewebrtc.js | 6 | ||||
-rw-r--r-- | src/utils/webrtc/simplewebrtc/webrtc.js | 6 | ||||
-rw-r--r-- | src/utils/webrtc/webrtc.js | 5 |
7 files changed, 338 insertions, 0 deletions
diff --git a/src/components/CallView/CallView.vue b/src/components/CallView/CallView.vue index ab2eab3f4..b22c5ca16 100644 --- a/src/components/CallView/CallView.vue +++ b/src/components/CallView/CallView.vue @@ -159,6 +159,7 @@ <script> import Grid from './Grid/Grid' +import { SIMULCAST } from '../../constants' import { localMediaModel, localCallParticipantModel, callParticipantCollection } from '../../utils/webrtc/index' import { fetchPeers } from '../../services/callsService' import { showMessage } from '@nextcloud/dialogs' @@ -344,6 +345,14 @@ export default { this.updateDataFromCallParticipantModels(models) }, + isGrid() { + this.adjustSimulcastQuality() + }, + + selectedVideoPeerId() { + this.adjustSimulcastQuality() + }, + speakers() { this._setPromotedParticipant() }, @@ -472,6 +481,8 @@ export default { }, function(raisedHand) { this._handleParticipantRaisedHand(addedModel, raisedHand) }) + + this.adjustSimulcastQualityForParticipant(addedModel) }) }, @@ -553,6 +564,8 @@ export default { if (!this.screenSharingActive && this.speakers.length) { this.sharedDatas[this.speakers[0].id].promoted = true } + + this.adjustSimulcastQuality() }, _switchScreenToId(id) { @@ -633,6 +646,25 @@ export default { handleToggleVideo({ peerId, value }) { this.sharedDatas[peerId].videoEnabled = value }, + + adjustSimulcastQuality() { + this.callParticipantModels.forEach(callParticipantModel => { + this.adjustSimulcastQualityForParticipant(callParticipantModel) + }) + }, + + adjustSimulcastQualityForParticipant(callParticipantModel) { + if (this.isGrid) { + callParticipantModel.setSimulcastVideoQuality(SIMULCAST.MEDIUM) + callParticipantModel.setSimulcastScreenQuality(SIMULCAST.MEDIUM) + } else if (this.sharedDatas[callParticipantModel.attributes.peerId].promoted || this.selectedVideoPeerId === callParticipantModel.attributes.peerId) { + callParticipantModel.setSimulcastVideoQuality(SIMULCAST.HIGH) + callParticipantModel.setSimulcastScreenQuality(SIMULCAST.HIGH) + } else { + callParticipantModel.setSimulcastVideoQuality(SIMULCAST.LOW) + callParticipantModel.setSimulcastScreenQuality(SIMULCAST.LOW) + } + }, }, } </script> diff --git a/src/constants.js b/src/constants.js index e422f9a7f..0a51cee3d 100644 --- a/src/constants.js +++ b/src/constants.js @@ -124,3 +124,8 @@ export const PRIVACY = { PUBLIC: 0, PRIVATE: 1, } +export const SIMULCAST = { + LOW: 0, + MEDIUM: 1, + HIGH: 2, +} diff --git a/src/utils/webrtc/models/CallParticipantModel.js b/src/utils/webrtc/models/CallParticipantModel.js index 6fbaa2cbf..a58d56470 100644 --- a/src/utils/webrtc/models/CallParticipantModel.js +++ b/src/utils/webrtc/models/CallParticipantModel.js @@ -355,4 +355,22 @@ CallParticipantModel.prototype = { this.set('nextcloudSessionId', nextcloudSessionId) }, + setSimulcastVideoQuality(simulcastVideoQuality) { + if (!this.get('peer') || !this.get('peer').enableSimulcast) { + return + } + + // Use same quality for simulcast and temporal layer. + this.get('peer').selectSimulcastStream(simulcastVideoQuality, simulcastVideoQuality) + }, + + setSimulcastScreenQuality(simulcastScreenQuality) { + if (!this.get('screenPeer') || !this.get('screenPeer').enableSimulcast) { + return + } + + // Use same quality for simulcast and temporal layer. + this.get('screenPeer').selectSimulcastStream(simulcastScreenQuality, simulcastScreenQuality) + }, + } diff --git a/src/utils/webrtc/simplewebrtc/peer.js b/src/utils/webrtc/simplewebrtc/peer.js index 893bdeb77..85d1acae8 100644 --- a/src/utils/webrtc/simplewebrtc/peer.js +++ b/src/utils/webrtc/simplewebrtc/peer.js @@ -2,6 +2,7 @@ const initialState = require('@nextcloud/initial-state') const sdpTransform = require('sdp-transform') +const adapter = require('webrtc-adapter') const util = require('util') const webrtcSupport = require('webrtcsupport') const WildEmitter = require('wildemitter') @@ -28,6 +29,8 @@ function Peer(options) { this.stream = options.stream this.sendVideoIfAvailable = options.sendVideoIfAvailable === undefined ? true : options.sendVideoIfAvailable this.enableDataChannels = options.enableDataChannels === undefined ? this.parent.config.enableDataChannels : options.enableDataChannels + this.enableSimulcast = options.enableSimulcast === undefined ? this.parent.config.enableSimulcast : options.enableSimulcast + this.maxBitrates = options.maxBitrates === undefined ? this.parent.config.maxBitrates : options.maxBitrates this.receiveMedia = options.receiveMedia || this.parent.config.receiveMedia this.channels = {} this.pendingDCMessages = [] // key (datachannel label) -> value (array[pending messages]) @@ -184,12 +187,260 @@ function preferH264VideoCodecIfAvailable(sessionDescription) { return sessionDescription } +// Helper method to munge an SDP to enable simulcasting (Chrome only) +// Taken from janus.js (MIT license). +/* eslint-disable */ +function mungeSdpForSimulcasting(sdp) { + // Let's munge the SDP to add the attributes for enabling simulcasting + // (based on https://gist.github.com/ggarber/a19b4c33510028b9c657) + var lines = sdp.split("\r\n"); + var video = false; + var ssrc = [ -1 ], ssrc_fid = [ -1 ]; + var cname = null, msid = null, mslabel = null, label = null; + var insertAt = -1; + for(var i=0; i<lines.length; i++) { + var mline = lines[i].match(/m=(\w+) */); + if(mline) { + var medium = mline[1]; + if(medium === "video") { + // New video m-line: make sure it's the first one + if(ssrc[0] < 0) { + video = true; + } else { + // We're done, let's add the new attributes here + insertAt = i; + break; + } + } else { + // New non-video m-line: do we have what we were looking for? + if(ssrc[0] > -1) { + // We're done, let's add the new attributes here + insertAt = i; + break; + } + } + continue; + } + if(!video) + continue; + var fid = lines[i].match(/a=ssrc-group:FID (\d+) (\d+)/); + if(fid) { + ssrc[0] = fid[1]; + ssrc_fid[0] = fid[2]; + lines.splice(i, 1); i--; + continue; + } + if(ssrc[0]) { + var match = lines[i].match('a=ssrc:' + ssrc[0] + ' cname:(.+)') + if(match) { + cname = match[1]; + } + match = lines[i].match('a=ssrc:' + ssrc[0] + ' msid:(.+)') + if(match) { + msid = match[1]; + } + match = lines[i].match('a=ssrc:' + ssrc[0] + ' mslabel:(.+)') + if(match) { + mslabel = match[1]; + } + match = lines[i].match('a=ssrc:' + ssrc[0] + ' label:(.+)') + if(match) { + label = match[1]; + } + if(lines[i].indexOf('a=ssrc:' + ssrc_fid[0]) === 0) { + lines.splice(i, 1); i--; + continue; + } + if(lines[i].indexOf('a=ssrc:' + ssrc[0]) === 0) { + lines.splice(i, 1); i--; + continue; + } + } + if(lines[i].length == 0) { + lines.splice(i, 1); i--; + continue; + } + } + if(ssrc[0] < 0) { + // Couldn't find a FID attribute, let's just take the first video SSRC we find + insertAt = -1; + video = false; + for(var i=0; i<lines.length; i++) { + var mline = lines[i].match(/m=(\w+) */); + if(mline) { + var medium = mline[1]; + if(medium === "video") { + // New video m-line: make sure it's the first one + if(ssrc[0] < 0) { + video = true; + } else { + // We're done, let's add the new attributes here + insertAt = i; + break; + } + } else { + // New non-video m-line: do we have what we were looking for? + if(ssrc[0] > -1) { + // We're done, let's add the new attributes here + insertAt = i; + break; + } + } + continue; + } + if(!video) + continue; + if(ssrc[0] < 0) { + var value = lines[i].match(/a=ssrc:(\d+)/); + if(value) { + ssrc[0] = value[1]; + lines.splice(i, 1); i--; + continue; + } + } else { + var match = lines[i].match('a=ssrc:' + ssrc[0] + ' cname:(.+)') + if(match) { + cname = match[1]; + } + match = lines[i].match('a=ssrc:' + ssrc[0] + ' msid:(.+)') + if(match) { + msid = match[1]; + } + match = lines[i].match('a=ssrc:' + ssrc[0] + ' mslabel:(.+)') + if(match) { + mslabel = match[1]; + } + match = lines[i].match('a=ssrc:' + ssrc[0] + ' label:(.+)') + if(match) { + label = match[1]; + } + if(lines[i].indexOf('a=ssrc:' + ssrc_fid[0]) === 0) { + lines.splice(i, 1); i--; + continue; + } + if(lines[i].indexOf('a=ssrc:' + ssrc[0]) === 0) { + lines.splice(i, 1); i--; + continue; + } + } + if(lines[i].length === 0) { + lines.splice(i, 1); i--; + continue; + } + } + } + if(ssrc[0] < 0) { + // Still nothing, let's just return the SDP we were asked to munge + console.warn("Couldn't find the video SSRC, simulcasting NOT enabled"); + return sdp; + } + if(insertAt < 0) { + // Append at the end + insertAt = lines.length; + } + // Generate a couple of SSRCs (for retransmissions too) + // Note: should we check if there are conflicts, here? + ssrc[1] = Math.floor(Math.random()*0xFFFFFFFF); + ssrc[2] = Math.floor(Math.random()*0xFFFFFFFF); + ssrc_fid[1] = Math.floor(Math.random()*0xFFFFFFFF); + ssrc_fid[2] = Math.floor(Math.random()*0xFFFFFFFF); + // Add attributes to the SDP + for(var i=0; i<ssrc.length; i++) { + if(cname) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' cname:' + cname); + insertAt++; + } + if(msid) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' msid:' + msid); + insertAt++; + } + if(mslabel) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' mslabel:' + mslabel); + insertAt++; + } + if(label) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc[i] + ' label:' + label); + insertAt++; + } + // Add the same info for the retransmission SSRC + if(cname) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' cname:' + cname); + insertAt++; + } + if(msid) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' msid:' + msid); + insertAt++; + } + if(mslabel) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' mslabel:' + mslabel); + insertAt++; + } + if(label) { + lines.splice(insertAt, 0, 'a=ssrc:' + ssrc_fid[i] + ' label:' + label); + insertAt++; + } + } + lines.splice(insertAt, 0, 'a=ssrc-group:FID ' + ssrc[2] + ' ' + ssrc_fid[2]); + lines.splice(insertAt, 0, 'a=ssrc-group:FID ' + ssrc[1] + ' ' + ssrc_fid[1]); + lines.splice(insertAt, 0, 'a=ssrc-group:FID ' + ssrc[0] + ' ' + ssrc_fid[0]); + lines.splice(insertAt, 0, 'a=ssrc-group:SIM ' + ssrc[0] + ' ' + ssrc[1] + ' ' + ssrc[2]); + sdp = lines.join("\r\n"); + if(!sdp.endsWith("\r\n")) + sdp += "\r\n"; + return sdp; +} +/* eslint-enable */ + Peer.prototype.offer = function(options) { + const sendVideo = this.sendVideoIfAvailable || this.type === 'screen' + if (sendVideo && this.enableSimulcast && adapter.browserDetails.browser === 'firefox') { + console.debug('Enabling Simulcasting for Firefox (RID)') + const sender = this.pc.getSenders().find(function(s) { + return s.track.kind === 'video' + }) + if (sender) { + let parameters = sender.getParameters() + if (!parameters) { + parameters = {} + } + parameters.encodings = [ + { + rid: 'h', + active: true, + maxBitrate: this.maxBitrates.high, + }, + { + rid: 'm', + active: true, + maxBitrate: this.maxBitrates.medium, + scaleResolutionDownBy: 2, + }, + { + rid: 'l', + active: true, + maxBitrate: this.maxBitrates.low, + scaleResolutionDownBy: 4, + }, + ] + sender.setParameters(parameters) + } + } this.pc.createOffer(options).then(function(offer) { if (shouldPreferH264()) { console.debug('Preferring hardware codec H.264 as per global configuration') offer = preferH264VideoCodecIfAvailable(offer) } + + if (sendVideo && this.enableSimulcast) { + // This SDP munging only works with Chrome (Safari STP may support it too) + if (adapter.browserDetails.browser === 'chrome' || adapter.browserDetails.browser === 'safari') { + console.debug('Enabling Simulcasting for Chrome (SDP munging)') + offer.sdp = mungeSdpForSimulcasting(offer.sdp) + } else if (adapter.browserDetails.browser !== 'firefox') { + console.debug('Simulcast can only be enabled on Chrome or Firefox') + } + } + this.pc.setLocalDescription(offer).then(function() { if (this.parent.config.nick) { // The offer is a RTCSessionDescription that only serializes @@ -250,6 +501,21 @@ Peer.prototype.handleAnswer = function(answer) { }) } +Peer.prototype.selectSimulcastStream = function(substream, temporal) { + if (this.substream === substream && this.temporal === temporal) { + console.debug('Simulcast stream not changed', this, substream, temporal) + return + } + + console.debug('Changing simulcast stream', this, substream, temporal) + this.send('selectStream', { + substream, + temporal, + }) + this.substream = substream + this.temporal = temporal +} + Peer.prototype.handleMessage = function(message) { const self = this diff --git a/src/utils/webrtc/simplewebrtc/simplewebrtc.js b/src/utils/webrtc/simplewebrtc/simplewebrtc.js index 1a8f0e439..c49287f3b 100644 --- a/src/utils/webrtc/simplewebrtc/simplewebrtc.js +++ b/src/utils/webrtc/simplewebrtc/simplewebrtc.js @@ -16,6 +16,12 @@ function SimpleWebRTC(opts) { localVideoEl: '', remoteVideosEl: '', enableDataChannels: true, + enableSimulcast: false, + maxBitrates: { + high: 900000, + medium: 300000, + low: 100000, + }, autoRequestMedia: false, autoRemoveVideos: true, adjustPeerVolume: false, diff --git a/src/utils/webrtc/simplewebrtc/webrtc.js b/src/utils/webrtc/simplewebrtc/webrtc.js index 1837d56d9..5c14c4913 100644 --- a/src/utils/webrtc/simplewebrtc/webrtc.js +++ b/src/utils/webrtc/simplewebrtc/webrtc.js @@ -20,6 +20,12 @@ function WebRTC(opts) { offerToReceiveVideo: 1, }, enableDataChannels: true, + enableSimulcast: false, + maxBitrates: { + high: 900000, + medium: 300000, + low: 100000, + }, } let item diff --git a/src/utils/webrtc/webrtc.js b/src/utils/webrtc/webrtc.js index 399e45fdc..40dd64871 100644 --- a/src/utils/webrtc/webrtc.js +++ b/src/utils/webrtc/webrtc.js @@ -72,6 +72,7 @@ function createScreensharingPeer(signaling, sessionId) { type: 'screen', sharemyscreen: true, enableDataChannels: false, + enableSimulcast: signaling.hasFeature('simulcast'), receiveMedia: { offerToReceiveAudio: 0, offerToReceiveVideo: 0, @@ -106,6 +107,7 @@ function createScreensharingPeer(signaling, sessionId) { type: 'screen', sharemyscreen: true, enableDataChannels: false, + enableSimulcast: signaling.hasFeature('simulcast'), receiveMedia: { offerToReceiveAudio: 0, offerToReceiveVideo: 0, @@ -136,6 +138,7 @@ function checkStartPublishOwnPeer(signaling) { id: currentSessionId, type: 'video', enableDataChannels: true, + enableSimulcast: signaling.hasFeature('simulcast'), receiveMedia: { offerToReceiveAudio: 0, offerToReceiveVideo: 0, @@ -285,6 +288,7 @@ function usersChanged(signaling, newUsers, disconnectedSessionIds) { id: sessionId, type: 'video', enableDataChannels: true, + enableSimulcast: signaling.hasFeature('simulcast'), receiveMedia: { offerToReceiveAudio: 1, offerToReceiveVideo: 1, @@ -530,6 +534,7 @@ export default function initWebRtc(signaling, _callParticipantCollection, _local detectSpeakingEvents: true, connection: signaling, enableDataChannels: true, + enableSimulcast: signaling.hasFeature('simulcast'), nick: store.getters.getDisplayName(), }) |