From 896650d0055179327a78d2a1ab54cb5a5c5658ed Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Fri, 9 Dec 2016 17:15:04 -0600 Subject: [PATCH 01/18] feat(remotecontrol): Implement basic remote control support --- app.js | 2 + conference.js | 38 ++++++- modules/API/API.js | 8 ++ modules/keycode/keycode.js | 160 ++++++++++++++++++++++++++++ modules/remotecontrol/Controller.js | 136 +++++++++++++++++++++++ modules/remotecontrol/Receiver.js | 47 ++++++++ 6 files changed, 387 insertions(+), 4 deletions(-) create mode 100644 modules/keycode/keycode.js create mode 100644 modules/remotecontrol/Controller.js create mode 100644 modules/remotecontrol/Receiver.js diff --git a/app.js b/app.js index 044c8c0fc..d8e73ae53 100644 --- a/app.js +++ b/app.js @@ -22,6 +22,8 @@ import conference from './conference'; import API from './modules/API/API'; import translation from "./modules/translation/translation"; +// For remote control testing: +// import remoteControlController from "./modules/remotecontrol/Controller"; const APP = { // Used by do_external_connect.js if we receive the attach data after diff --git a/conference.js b/conference.js index 9f00d7449..73e762d3d 100644 --- a/conference.js +++ b/conference.js @@ -17,6 +17,9 @@ import UIUtil from './modules/UI/util/UIUtil'; import analytics from './modules/analytics/analytics'; +// For remote control testing: +// import remoteControlReceiver from './modules/remotecontrol/Receiver'; + const ConnectionEvents = JitsiMeetJS.events.connection; const ConnectionErrors = JitsiMeetJS.errors.connection; @@ -981,6 +984,8 @@ export default { let externalInstallation = false; if (shareScreen) { + // For remote control testing: + // remoteControlReceiver.start(); createLocalTracks({ devices: ['desktop'], desktopSharingExtensionExternalInstallation: { @@ -1070,6 +1075,8 @@ export default { dialogTitleKey, dialogTxt, false); }); } else { + // For remote control testing: + // remoteControlReceiver.stop(); createLocalTracks({ devices: ['video'] }).then( ([stream]) => this.useVideoStream(stream) ).then(() => { @@ -1600,12 +1607,23 @@ export default { }, /** * Adds any room listener. - * @param eventName one of the ConferenceEvents - * @param callBack the function to be called when the event occurs + * @param {string} eventName one of the ConferenceEvents + * @param {Function} listener the function to be called when the event + * occurs */ - addConferenceListener(eventName, callBack) { - room.on(eventName, callBack); + addConferenceListener(eventName, listener) { + room.on(eventName, listener); }, + + /** + * Removes any room listener. + * @param {string} eventName one of the ConferenceEvents + * @param {Function} listener the listener to be removed. + */ + removeConferenceListener(eventName, listener) { + room.off(eventName, listener); + }, + /** * Inits list of current devices and event listener for device change. * @private @@ -1813,5 +1831,17 @@ export default { APP.settings.setAvatarUrl(url); APP.UI.setUserAvatarUrl(room.myUserId(), url); sendData(commands.AVATAR_URL, url); + }, + + /** + * Sends a message via the data channel. + * @param to {string} the id of the endpoint that should receive the + * message. If "" the message will be sent to all participants. + * @param payload {object} the payload of the message. + * @throws NetworkError or InvalidStateError or Error if the operation + * fails. + */ + sendEndpointMessage (to, payload) { + room.sendEndpointMessage(to, payload); } }; diff --git a/modules/API/API.js b/modules/API/API.js index fc2ce7bc4..b988c089f 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -201,6 +201,14 @@ export default { triggerEvent("video-ready-to-close", {}); }, + /** + * Sends remote control event. + * @param {object} event the event. + */ + sendRemoteControlEvent(event) { + sendMessage({method: "remote-control-event", params: event}); + }, + /** * Removes the listeners. */ diff --git a/modules/keycode/keycode.js b/modules/keycode/keycode.js new file mode 100644 index 000000000..94a81ba4b --- /dev/null +++ b/modules/keycode/keycode.js @@ -0,0 +1,160 @@ +/** + * Enumerates the supported keys. + */ +export const KEYS = { + BACKSPACE: "backspace" , + DELETE : "delete", + RETURN : "enter", + TAB : "tab", + ESCAPE : "escape", + UP : "up", + DOWN : "down", + RIGHT : "right", + LEFT : "left", + HOME : "home", + END : "end", + PAGEUP : "pageup", + PAGEDOWN : "pagedown", + + F1 : "f1", + F2 : "f2", + F3 : "f3", + F4 : "f4", + F5 : "f5", + F6 : "f6", + F7 : "f7", + F8 : "f8", + F9 : "f9", + F10 : "f10", + F11 : "f11", + F12 : "f12", + META : "command", + CMD_L: "command", + CMD_R: "command", + ALT : "alt", + CONTROL : "control", + SHIFT : "shift", + CAPS_LOCK: "caps_lock", //not supported by robotjs + SPACE : "space", + PRINTSCREEN : "printscreen", + INSERT : "insert", + + NUMPAD_0 : "numpad_0", + NUMPAD_1 : "numpad_1", + NUMPAD_2 : "numpad_2", + NUMPAD_3 : "numpad_3", + NUMPAD_4 : "numpad_4", + NUMPAD_5 : "numpad_5", + NUMPAD_6 : "numpad_6", + NUMPAD_7 : "numpad_7", + NUMPAD_8 : "numpad_8", + NUMPAD_9 : "numpad_9", + + COMMA: ",", + + PERIOD: ".", + SEMICOLON: ";", + QUOTE: "'", + BRACKET_LEFT: "[", + BRACKET_RIGHT: "]", + BACKQUOTE: "`", + BACKSLASH: "\\", + MINUS: "-", + EQUAL: "=", + SLASH: "/" +}; + +/** + * Mapping between the key codes and keys deined in KEYS. + * The mappings are based on + * https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/keyCode#Specifications + */ +let keyCodeToKey = { + 8: KEYS.BACKSPACE, + 9: KEYS.TAB, + 13: KEYS.RETURN, + 16: KEYS.SHIFT, + 17: KEYS.CONTROL, + 18: KEYS.ALT, + 20: KEYS.CAPS_LOCK, + 27: KEYS.ESCAPE, + 32: KEYS.SPACE, + 33: KEYS.PAGEUP, + 34: KEYS.PAGEDOWN, + 35: KEYS.END, + 36: KEYS.HOME, + 37: KEYS.LEFT, + 38: KEYS.UP, + 39: KEYS.RIGHT, + 40: KEYS.DOWN, + 42: KEYS.PRINTSCREEN, + 44: KEYS.PRINTSCREEN, + 45: KEYS.INSERT, + 46: KEYS.DELETE, + 59: KEYS.SEMICOLON, + 61: KEYS.EQUAL, + 91: KEYS.CMD_L, + 92: KEYS.CMD_R, + 93: KEYS.CMD_R, + 96: KEYS.NUMPAD_0, + 97: KEYS.NUMPAD_1, + 98: KEYS.NUMPAD_2, + 99: KEYS.NUMPAD_3, + 100: KEYS.NUMPAD_4, + 101: KEYS.NUMPAD_5, + 102: KEYS.NUMPAD_6, + 103: KEYS.NUMPAD_7, + 104: KEYS.NUMPAD_8, + 105: KEYS.NUMPAD_9, + 112: KEYS.F1, + 113: KEYS.F2, + 114: KEYS.F3, + 115: KEYS.F4, + 116: KEYS.F5, + 117: KEYS.F6, + 118: KEYS.F7, + 119: KEYS.F8, + 120: KEYS.F9, + 121: KEYS.F10, + 122: KEYS.F11, + 123: KEYS.F12, + 124: KEYS.PRINTSCREEN, + 173: KEYS.MINUS, + 186: KEYS.SEMICOLON, + 187: KEYS.EQUAL, + 188: KEYS.COMMA, + 189: KEYS.MINUS, + 190: KEYS.PERIOD, + 191: KEYS.SLASH, + 192: KEYS.BACKQUOTE, + 219: KEYS.BRACKET_LEFT, + 220: KEYS.BACKSLASH, + 221: KEYS.BRACKET_RIGHT, + 222: KEYS.QUOTE, + 224: KEYS.META, + 229: KEYS.SEMICOLON +}; + +/** + * Generate codes for digit keys (0-9) + */ +for(let i = 0; i < 10; i++) { + keyCodeToKey[i + 48] = `${i}`; +} + +/** + * Generate codes for letter keys (a-z) + */ +for(let i = 0; i < 26; i++) { + let keyCode = i + 65; + keyCodeToKey[keyCode] = String.fromCharCode(keyCode).toLowerCase(); +} + +/** + * Returns key associated with the keyCode from the passed event. + * @param {KeyboardEvent} event the event + * @returns {KEYS} the key on the keyboard. + */ +export function keyboardEventToKey(event) { + return keyCodeToKey[event.which]; +} diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js new file mode 100644 index 000000000..5650b4c7f --- /dev/null +++ b/modules/remotecontrol/Controller.js @@ -0,0 +1,136 @@ +/* global $, APP */ +import * as KeyCodes from "../keycode/keycode"; + +/** + * Extract the keyboard key from the keyboard event. + * @param event {KeyboardEvent} the event. + * @returns {KEYS} the key that is pressed or undefined. + */ +function getKey(event) { + return KeyCodes.keyboardEventToKey(event); +} + +/** + * Extract the modifiers from the keyboard event. + * @param event {KeyboardEvent} the event. + * @returns {Array} with possible values: "shift", "control", "alt", "command". + */ +function getModifiers(event) { + let modifiers = []; + if(event.shiftKey) { + modifiers.push("shift"); + } + + if(event.ctrlKey) { + modifiers.push("control"); + } + + + if(event.altKey) { + modifiers.push("alt"); + } + + if(event.metaKey) { + modifiers.push("command"); + } + + return modifiers; +} + +/** + * This class represents the controller party for a remote controller session. + * It listens for mouse and keyboard events and sends them to the receiver + * party of the remote control session. + */ +class Controller { + /** + * Creates new instance. + */ + constructor() {} + + /** + * Starts processing the mouse and keyboard events. + * @param {JQuery.selector} area the selector which will be used for + * attaching the listeners on. + */ + start(area) { + this.area = area; + this.area.mousemove(event => { + const position = this.area.position(); + this._sendEvent({ + type: "mousemove", + x: (event.pageX - position.left)/this.area.width(), + y: (event.pageY - position.top)/this.area.height() + }); + }); + this.area.mousedown(this._onMouseClickHandler.bind(this, "mousedown")); + this.area.mouseup(this._onMouseClickHandler.bind(this, "mouseup")); + this.area.dblclick( + this._onMouseClickHandler.bind(this, "mousedblclick")); + this.area.contextmenu(() => false); + this.area[0].onmousewheel = event => { + this._sendEvent({ + type: "mousescroll", + x: event.deltaX, + y: event.deltaY + }); + }; + $(window).keydown(this._onKeyPessHandler.bind(this, "keydown")); + $(window).keyup(this._onKeyPessHandler.bind(this, "keyup")); + } + + /** + * Stops processing the mouse and keyboard events. + */ + stop() { + this.area.off( "mousemove" ); + this.area.off( "mousedown" ); + this.area.off( "mouseup" ); + this.area.off( "contextmenu" ); + this.area.off( "dblclick" ); + $(window).off( "keydown"); + $(window).off( "keyup"); + this.area[0].onmousewheel = undefined; + } + + /** + * Handler for mouse click events. + * @param {String} type the type of event ("mousedown"/"mouseup") + * @param {Event} event the mouse event. + */ + _onMouseClickHandler(type, event) { + this._sendEvent({ + type: type, + button: event.which + }); + } + + /** + * Handler for key press events. + * @param {String} type the type of event ("keydown"/"keyup") + * @param {Event} event the key event. + */ + _onKeyPessHandler(type, event) { + this._sendEvent({ + type: type, + key: getKey(event), + modifiers: getModifiers(event), + }); + } + + /** + * Sends remote control event to the controlled participant. + * @param {Object} event the remote control event. + */ + _sendRemoteControlEvent(event) { + try{ + APP.conference.sendEndpointMessage("", + {type: "remote-control-event", event}); + } catch (e) { + // failed to send the event. + } + } +} + + +export default new Controller(); diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js new file mode 100644 index 000000000..144d5b57e --- /dev/null +++ b/modules/remotecontrol/Receiver.js @@ -0,0 +1,47 @@ +/* global APP, JitsiMeetJS */ +const ConferenceEvents = JitsiMeetJS.events.conference; + +/** + * This class represents the receiver party for a remote controller session. + * It handles "remote-control-event" events and sends them to the + * API module. From there the events can be received from wrapper application + * and executed. + */ +class Receiver { + /** + * Creates new instance. + * @constructor + */ + constructor() {} + + /** + * Attaches listener for ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED events. + */ + start() { + APP.conference.addConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + this._onRemoteControlEvent); + } + + /** + * Removes the listener for ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED + * events. + */ + stop() { + APP.conference.removeConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + this._onRemoteControlEvent); + } + + /** + * Sends "remote-control-event" events to to the API module. + * @param {JitsiParticipant} participant the controller participant + * @param {Object} event the remote control event. + */ + _onRemoteControlEvent(participant, event) { + if(event.type === "remote-control-event") + APP.API.sendRemoteControlEvent(event.event); + } +} + +export default new Receiver(); From 0f33e59e4d7ad3d2395147f167d125ded08f2c25 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Tue, 20 Dec 2016 16:15:13 -0600 Subject: [PATCH 02/18] feat(remotecontrol): announce remotecontrol support --- app.js | 6 +-- conference.js | 1 + modules/API/API.js | 45 ++++++++++++++++------- modules/remotecontrol/Controller.js | 50 ++++++++++++++++--------- modules/remotecontrol/Receiver.js | 33 +++++++++++++---- modules/remotecontrol/RemoteControl.js | 51 ++++++++++++++++++++++++++ modules/tokendata/TokenData.js | 4 -- react/features/app/functions.web.js | 2 +- service/remotecontrol/Constants.js | 23 ++++++++++++ 9 files changed, 168 insertions(+), 47 deletions(-) create mode 100644 modules/remotecontrol/RemoteControl.js create mode 100644 service/remotecontrol/Constants.js diff --git a/app.js b/app.js index d8e73ae53..9207cf6ca 100644 --- a/app.js +++ b/app.js @@ -22,8 +22,7 @@ import conference from './conference'; import API from './modules/API/API'; import translation from "./modules/translation/translation"; -// For remote control testing: -// import remoteControlController from "./modules/remotecontrol/Controller"; +import remoteControl from "./modules/remotecontrol/remotecontrol"; const APP = { // Used by do_external_connect.js if we receive the attach data after @@ -61,7 +60,8 @@ const APP = { */ ConferenceUrl : null, connection: null, - API + API, + remoteControl }; // TODO The execution of the mobile app starts from react/index.native.js. diff --git a/conference.js b/conference.js index 73e762d3d..a51192de9 100644 --- a/conference.js +++ b/conference.js @@ -488,6 +488,7 @@ export default { }).then(([tracks, con]) => { logger.log('initialized with %s local tracks', tracks.length); APP.connection = connection = con; + APP.remoteControl.init(); this._bindConnectionFailedHandler(con); this._createRoom(tracks); this.isDesktopSharingEnabled = diff --git a/modules/API/API.js b/modules/API/API.js index b988c089f..b9860d09b 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -55,7 +55,9 @@ function initCommands() { APP.conference.toggleScreenSharing.bind(APP.conference), "video-hangup": () => APP.conference.hangup(), "email": APP.conference.changeLocalEmail, - "avatar-url": APP.conference.changeLocalAvatarUrl + "avatar-url": APP.conference.changeLocalAvatarUrl, + "remote-control-supported": isSupported => + APP.remoteControl.onRemoteControlSupported(isSupported) }; Object.keys(commands).forEach(function (key) { postis.listen(key, args => commands[key](...args)); @@ -94,7 +96,13 @@ function triggerEvent (name, object) { } } -export default { +class API { + /** + * Constructs new instance + * @constructor + */ + constructor() { } + /** * Initializes the APIConnector. Setups message event listeners that will * receive information from external applications that embed Jitsi Meet. @@ -108,6 +116,13 @@ export default { return; enabled = true; + } + + /** + * initializes postis library. + * @private + */ + _initPostis() { let postisOptions = { window: target }; @@ -116,7 +131,7 @@ export default { = "jitsi_meet_external_api_" + jitsi_meet_external_api_id; postis = postisInit(postisOptions); initCommands(); - }, + } /** * Notify external application (if API is enabled) that message was sent. @@ -124,7 +139,7 @@ export default { */ notifySendingChatMessage (body) { triggerEvent("outgoing-message", {"message": body}); - }, + } /** * Notify external application (if API is enabled) that @@ -143,7 +158,7 @@ export default { "incoming-message", {"from": id, "nick": nick, "message": body, "stamp": ts} ); - }, + } /** * Notify external application (if API is enabled) that @@ -152,7 +167,7 @@ export default { */ notifyUserJoined (id) { triggerEvent("participant-joined", {id}); - }, + } /** * Notify external application (if API is enabled) that @@ -161,7 +176,7 @@ export default { */ notifyUserLeft (id) { triggerEvent("participant-left", {id}); - }, + } /** * Notify external application (if API is enabled) that @@ -171,7 +186,7 @@ export default { */ notifyDisplayNameChanged (id, displayName) { triggerEvent("display-name-change", {id, displayname: displayName}); - }, + } /** * Notify external application (if API is enabled) that @@ -181,7 +196,7 @@ export default { */ notifyConferenceJoined (room) { triggerEvent("video-conference-joined", {roomName: room}); - }, + } /** * Notify external application (if API is enabled) that @@ -191,7 +206,7 @@ export default { */ notifyConferenceLeft (room) { triggerEvent("video-conference-left", {roomName: room}); - }, + } /** * Notify external application (if API is enabled) that @@ -199,7 +214,7 @@ export default { */ notifyReadyToClose () { triggerEvent("video-ready-to-close", {}); - }, + } /** * Sends remote control event. @@ -207,13 +222,15 @@ export default { */ sendRemoteControlEvent(event) { sendMessage({method: "remote-control-event", params: event}); - }, + } /** * Removes the listeners. */ - dispose: function () { + dispose () { if(enabled) postis.destroy(); } -}; +} + +export default new API(); diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index 5650b4c7f..56b739f11 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -1,5 +1,7 @@ /* global $, APP */ import * as KeyCodes from "../keycode/keycode"; +import {EVENT_TYPES, API_EVENT_TYPE} + from "../../service/remotecontrol/Constants"; /** * Extract the keyboard key from the keyboard event. @@ -42,11 +44,21 @@ function getModifiers(event) { * It listens for mouse and keyboard events and sends them to the receiver * party of the remote control session. */ -class Controller { +export default class Controller { /** * Creates new instance. */ - constructor() {} + constructor() { + this.enabled = false; + } + + /** + * Enables / Disables the remote control + * @param {boolean} enabled the new state. + */ + enable(enabled) { + this.enabled = enabled; + } /** * Starts processing the mouse and keyboard events. @@ -54,29 +66,34 @@ class Controller { * attaching the listeners on. */ start(area) { + if(!this.enabled) + return; this.area = area; this.area.mousemove(event => { const position = this.area.position(); - this._sendEvent({ - type: "mousemove", + this._sendRemoteControlEvent({ + type: EVENT_TYPES.mousemove, x: (event.pageX - position.left)/this.area.width(), y: (event.pageY - position.top)/this.area.height() }); }); - this.area.mousedown(this._onMouseClickHandler.bind(this, "mousedown")); - this.area.mouseup(this._onMouseClickHandler.bind(this, "mouseup")); + this.area.mousedown(this._onMouseClickHandler.bind(this, + EVENT_TYPES.mousedown)); + this.area.mouseup(this._onMouseClickHandler.bind(this, + EVENT_TYPES.mouseup)); this.area.dblclick( - this._onMouseClickHandler.bind(this, "mousedblclick")); + this._onMouseClickHandler.bind(this, EVENT_TYPES.mousedblclick)); this.area.contextmenu(() => false); this.area[0].onmousewheel = event => { - this._sendEvent({ - type: "mousescroll", + this._sendRemoteControlEvent({ + type: EVENT_TYPES.mousescroll, x: event.deltaX, y: event.deltaY }); }; - $(window).keydown(this._onKeyPessHandler.bind(this, "keydown")); - $(window).keyup(this._onKeyPessHandler.bind(this, "keyup")); + $(window).keydown(this._onKeyPessHandler.bind(this, + EVENT_TYPES.keydown)); + $(window).keyup(this._onKeyPessHandler.bind(this, EVENT_TYPES.keyup)); } /** @@ -99,7 +116,7 @@ class Controller { * @param {Event} event the mouse event. */ _onMouseClickHandler(type, event) { - this._sendEvent({ + this._sendRemoteControlEvent({ type: type, button: event.which }); @@ -111,7 +128,7 @@ class Controller { * @param {Event} event the key event. */ _onKeyPessHandler(type, event) { - this._sendEvent({ + this._sendRemoteControlEvent({ type: type, key: getKey(event), modifiers: getModifiers(event), @@ -123,14 +140,13 @@ class Controller { * @param {Object} event the remote control event. */ _sendRemoteControlEvent(event) { + if(!this.enabled) + return; try{ APP.conference.sendEndpointMessage("", - {type: "remote-control-event", event}); + {type: API_EVENT_TYPE, event}); } catch (e) { // failed to send the event. } } } - - -export default new Controller(); diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index 144d5b57e..7785d6324 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -1,4 +1,7 @@ /* global APP, JitsiMeetJS */ +import {DISCO_REMOTE_CONTROL_FEATURE, API_EVENT_TYPE} + from "../../service/remotecontrol/Constants"; + const ConferenceEvents = JitsiMeetJS.events.conference; /** @@ -7,20 +10,36 @@ const ConferenceEvents = JitsiMeetJS.events.conference; * API module. From there the events can be received from wrapper application * and executed. */ -class Receiver { +export default class Receiver { /** * Creates new instance. * @constructor */ - constructor() {} + constructor() { + this.enabled = false; + } + + /** + * Enables / Disables the remote control + * @param {boolean} enabled the new state. + */ + enable(enabled) { + if(this.enabled !== enabled && enabled === true) { + this.enabled = enabled; + // Announce remote control support. + APP.connection.addFeature(DISCO_REMOTE_CONTROL_FEATURE, true); + } + } /** * Attaches listener for ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED events. */ start() { - APP.conference.addConferenceListener( - ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - this._onRemoteControlEvent); + if(this.enabled) { + APP.conference.addConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + this._onRemoteControlEvent); + } } /** @@ -39,9 +58,7 @@ class Receiver { * @param {Object} event the remote control event. */ _onRemoteControlEvent(participant, event) { - if(event.type === "remote-control-event") + if(event.type === API_EVENT_TYPE && this.enabled) APP.API.sendRemoteControlEvent(event.event); } } - -export default new Receiver(); diff --git a/modules/remotecontrol/RemoteControl.js b/modules/remotecontrol/RemoteControl.js new file mode 100644 index 000000000..2da4680dd --- /dev/null +++ b/modules/remotecontrol/RemoteControl.js @@ -0,0 +1,51 @@ +/* global APP, config */ +import Controller from "./Controller"; +import Receiver from "./Receiver"; + +/** + * Implements the remote control functionality. + */ +class RemoteControl { + /** + * Constructs new instance. Creates controller and receiver properties. + * @constructor + */ + constructor() { + this.controller = new Controller(); + this.receiver = new Receiver(); + this.enabled = false; + this.initialized = false; + } + + /** + * Initializes the remote control - checks if the remote control should be + * enabled or not, initializes the API module. + */ + init() { + if(config.disableRemoteControl || this.initialized) { + return; + } + this.initialized = true; + APP.API.init({ + forceEnable: true, + }); + this.controller.enable(true); + } + + /** + * Handles API event for support for executing remote control events into + * the wrapper application. + * @param {boolean} isSupported true if the receiver side is supported by + * the wrapper application. + */ + onRemoteControlSupported(isSupported) { + if(isSupported && !config.disableRemoteControl) { + this.enabled = true; + if(this.initialized) { + this.receiver.enable(true); + } + } + } +} + +export default new RemoteControl(); diff --git a/modules/tokendata/TokenData.js b/modules/tokendata/TokenData.js index 6e1856ddf..86438937b 100644 --- a/modules/tokendata/TokenData.js +++ b/modules/tokendata/TokenData.js @@ -73,10 +73,6 @@ class TokenData{ this.jwt = jwt; - //External API settings - this.externalAPISettings = { - forceEnable: true - }; this._decode(); // Use JWT param as token if there is not other token set and if the // iss field is not anonymous. If you want to pass data with JWT token diff --git a/react/features/app/functions.web.js b/react/features/app/functions.web.js index ec4bbee3a..77319fde8 100644 --- a/react/features/app/functions.web.js +++ b/react/features/app/functions.web.js @@ -23,7 +23,7 @@ export function init() { APP.keyboardshortcut = KeyboardShortcut; APP.tokenData = getTokenData(); - APP.API.init(APP.tokenData.externalAPISettings); + APP.API.init(APP.tokenData.jwt ? {forceEnable: true} : undefined); APP.translation.init(settings.getLanguage()); } diff --git a/service/remotecontrol/Constants.js b/service/remotecontrol/Constants.js new file mode 100644 index 000000000..34e776ea9 --- /dev/null +++ b/service/remotecontrol/Constants.js @@ -0,0 +1,23 @@ +/** + * The value for the "var" attribute of feature tag in disco-info packets. + */ +export const DISCO_REMOTE_CONTROL_FEATURE + = "http://jitsi.org/meet/remotecontrol"; + +/** + * Types of remote-control-event events. + */ +export const EVENT_TYPES = { + mousemove: "mousemove", + mousedown: "mousedown", + mouseup: "mouseup", + mousedblclick: "mousedblclick", + mousescroll: "mousescroll", + keydown: "keydown", + keyup: "keyup" +}; + +/** + * The type of remote control events sent trough the API module. + */ +export const API_EVENT_TYPE = "remote-control-event"; From 846fb9abb07438866aa1088292e9cab7822537c6 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Tue, 27 Dec 2016 18:46:55 -0600 Subject: [PATCH 03/18] feat(remotecontrol): Implement requesting remote control permissions --- conference.js | 5 +- modules/API/API.js | 4 +- modules/remotecontrol/Controller.js | 135 +++++++++++++----- modules/remotecontrol/Receiver.js | 61 +++++--- modules/remotecontrol/RemoteControl.js | 24 +++- .../remotecontrol/RemoteControlParticipant.js | 36 +++++ service/remotecontrol/Constants.js | 16 ++- 7 files changed, 221 insertions(+), 60 deletions(-) create mode 100644 modules/remotecontrol/RemoteControlParticipant.js diff --git a/conference.js b/conference.js index a51192de9..36309b9cd 100644 --- a/conference.js +++ b/conference.js @@ -985,8 +985,6 @@ export default { let externalInstallation = false; if (shareScreen) { - // For remote control testing: - // remoteControlReceiver.start(); createLocalTracks({ devices: ['desktop'], desktopSharingExtensionExternalInstallation: { @@ -1076,8 +1074,7 @@ export default { dialogTitleKey, dialogTxt, false); }); } else { - // For remote control testing: - // remoteControlReceiver.stop(); + APP.remoteControl.receiver.stop(); createLocalTracks({ devices: ['video'] }).then( ([stream]) => this.useVideoStream(stream) ).then(() => { diff --git a/modules/API/API.js b/modules/API/API.js index b9860d09b..471ccfe48 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -56,8 +56,8 @@ function initCommands() { "video-hangup": () => APP.conference.hangup(), "email": APP.conference.changeLocalEmail, "avatar-url": APP.conference.changeLocalAvatarUrl, - "remote-control-supported": isSupported => - APP.remoteControl.onRemoteControlSupported(isSupported) + "remote-control-event": event => + APP.remoteControl.onRemoteControlAPIEvent(event) }; Object.keys(commands).forEach(function (key) { postis.listen(key, args => commands[key](...args)); diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index 56b739f11..6e60e1189 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -1,7 +1,10 @@ -/* global $, APP */ +/* global $, JitsiMeetJS, APP */ import * as KeyCodes from "../keycode/keycode"; -import {EVENT_TYPES, API_EVENT_TYPE} +import {EVENT_TYPES, REMOTE_CONTROL_EVENT_TYPE, PERMISSIONS_ACTIONS} from "../../service/remotecontrol/Constants"; +import RemoteControlParticipant from "./RemoteControlParticipant"; + +const ConferenceEvents = JitsiMeetJS.events.conference; /** * Extract the keyboard key from the keyboard event. @@ -44,34 +47,113 @@ function getModifiers(event) { * It listens for mouse and keyboard events and sends them to the receiver * party of the remote control session. */ -export default class Controller { +export default class Controller extends RemoteControlParticipant { /** * Creates new instance. */ constructor() { - this.enabled = false; + super(); + this.controlledParticipant = null; + this.requestedParticipant = null; + this.stopListener = this._handleRemoteControlStoppedEvent.bind(this); } /** - * Enables / Disables the remote control - * @param {boolean} enabled the new state. + * Requests permissions from the remote control receiver side. + * @param {string} userId the user id of the participant that will be + * requested. */ - enable(enabled) { - this.enabled = enabled; + requestPermissions(userId) { + if(!this.enabled) { + return Promise.reject(new Error("Remote control is disabled!")); + } + return new Promise((resolve, reject) => { + let permissionsReplyListener = (participant, event) => { + let result = null; + try { + result = this._handleReply(participant, event); + } catch (e) { + reject(e); + } + if(result !== null) { + this.requestedParticipant = null; + APP.conference.removeConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + permissionsReplyListener); + resolve(result); + } + }; + APP.conference.addConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + permissionsReplyListener); + this.requestedParticipant = userId; + this._sendRemoteControlEvent(userId, { + type: EVENT_TYPES.permissions, + action: PERMISSIONS_ACTIONS.request + }, e => { + APP.conference.removeConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + permissionsReplyListener); + this.requestedParticipant = null; + reject(e); + }); + }); + } + + /** + * Handles the reply of the permissions request. + * @param {JitsiParticipant} participant the participant that has sent the + * reply + * @param {object} event the remote control event. + */ + _handleReply(participant, event) { + const remoteControlEvent = event.event; + const userId = participant.getId(); + if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE + && remoteControlEvent.type === EVENT_TYPES.permissions + && userId === this.requestedParticipant) { + if(remoteControlEvent.action === PERMISSIONS_ACTIONS.grant) { + this.controlledParticipant = userId; + this._start(); + return true; + } else if(remoteControlEvent.action === PERMISSIONS_ACTIONS.deny) { + return false; + } else { + throw new Error("Unknown reply received!"); + } + } else { + //different message type or another user -> ignoring the message + return null; + } + } + + /** + * Handles remote control stopped. + * @param {JitsiParticipant} participant the participant that has sent the + * event + * @param {object} event the the remote control event. + */ + _handleRemoteControlStoppedEvent(participant, event) { + if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE + && event.event.type === EVENT_TYPES.stop + && participant.getId() === this.controlledParticipant) { + this._stop(); + } } /** * Starts processing the mouse and keyboard events. - * @param {JQuery.selector} area the selector which will be used for - * attaching the listeners on. */ - start(area) { + _start() { if(!this.enabled) return; - this.area = area; + APP.conference.addConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + this.stopListener); + this.area = $("#largeVideoWrapper"); this.area.mousemove(event => { const position = this.area.position(); - this._sendRemoteControlEvent({ + this._sendRemoteControlEvent(this.controlledParticipant, { type: EVENT_TYPES.mousemove, x: (event.pageX - position.left)/this.area.width(), y: (event.pageY - position.top)/this.area.height() @@ -85,7 +167,7 @@ export default class Controller { this._onMouseClickHandler.bind(this, EVENT_TYPES.mousedblclick)); this.area.contextmenu(() => false); this.area[0].onmousewheel = event => { - this._sendRemoteControlEvent({ + this._sendRemoteControlEvent(this.controlledParticipant, { type: EVENT_TYPES.mousescroll, x: event.deltaX, y: event.deltaY @@ -99,7 +181,11 @@ export default class Controller { /** * Stops processing the mouse and keyboard events. */ - stop() { + _stop() { + APP.conference.removeConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + this.stopListener); + this.controlledParticipant = null; this.area.off( "mousemove" ); this.area.off( "mousedown" ); this.area.off( "mouseup" ); @@ -116,7 +202,7 @@ export default class Controller { * @param {Event} event the mouse event. */ _onMouseClickHandler(type, event) { - this._sendRemoteControlEvent({ + this._sendRemoteControlEvent(this.controlledParticipant, { type: type, button: event.which }); @@ -128,25 +214,10 @@ export default class Controller { * @param {Event} event the key event. */ _onKeyPessHandler(type, event) { - this._sendRemoteControlEvent({ + this._sendRemoteControlEvent(this.controlledParticipant, { type: type, key: getKey(event), modifiers: getModifiers(event), }); } - - /** - * Sends remote control event to the controlled participant. - * @param {Object} event the remote control event. - */ - _sendRemoteControlEvent(event) { - if(!this.enabled) - return; - try{ - APP.conference.sendEndpointMessage("", - {type: API_EVENT_TYPE, event}); - } catch (e) { - // failed to send the event. - } - } } diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index 7785d6324..e23017b7e 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -1,6 +1,7 @@ /* global APP, JitsiMeetJS */ -import {DISCO_REMOTE_CONTROL_FEATURE, API_EVENT_TYPE} - from "../../service/remotecontrol/Constants"; +import {DISCO_REMOTE_CONTROL_FEATURE, REMOTE_CONTROL_EVENT_TYPE, EVENT_TYPES, + PERMISSIONS_ACTIONS} from "../../service/remotecontrol/Constants"; +import RemoteControlParticipant from "./RemoteControlParticipant"; const ConferenceEvents = JitsiMeetJS.events.conference; @@ -10,13 +11,16 @@ const ConferenceEvents = JitsiMeetJS.events.conference; * API module. From there the events can be received from wrapper application * and executed. */ -export default class Receiver { +export default class Receiver extends RemoteControlParticipant { /** * Creates new instance. * @constructor */ constructor() { - this.enabled = false; + super(); + this.controller = null; + this._remoteControlEventsListener + = this._onRemoteControlEvent.bind(this); } /** @@ -28,17 +32,9 @@ export default class Receiver { this.enabled = enabled; // Announce remote control support. APP.connection.addFeature(DISCO_REMOTE_CONTROL_FEATURE, true); - } - } - - /** - * Attaches listener for ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED events. - */ - start() { - if(this.enabled) { APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - this._onRemoteControlEvent); + this._remoteControlEventsListener); } } @@ -49,7 +45,13 @@ export default class Receiver { stop() { APP.conference.removeConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - this._onRemoteControlEvent); + this._remoteControlEventsListener); + const event = { + type: EVENT_TYPES.stop + }; + this._sendRemoteControlEvent(this.controller, event); + this.controller = null; + APP.API.sendRemoteControlEvent(event); } /** @@ -58,7 +60,34 @@ export default class Receiver { * @param {Object} event the remote control event. */ _onRemoteControlEvent(participant, event) { - if(event.type === API_EVENT_TYPE && this.enabled) - APP.API.sendRemoteControlEvent(event.event); + if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE) { + const remoteControlEvent = event.event; + if(this.controller === null + && remoteControlEvent.type === EVENT_TYPES.permissions + && remoteControlEvent.action === PERMISSIONS_ACTIONS.request) { + remoteControlEvent.userId = participant.getId(); + remoteControlEvent.userJID = participant.getJid(); + remoteControlEvent.displayName = participant.getDisplayName(); + } else if(this.controller !== participant.getId()) { + return; + } + APP.API.sendRemoteControlEvent(remoteControlEvent); + } + } + + /** + * Handles remote control permission events received from the API module. + * @param {String} userId the user id of the participant related to the + * event. + * @param {PERMISSIONS_ACTIONS} action the action related to the event. + */ + _onRemoteControlPermissionsEvent(userId, action) { + if(action === PERMISSIONS_ACTIONS.grant) { + this.controller = userId; + } + this._sendRemoteControlEvent(userId, { + type: EVENT_TYPES.permissions, + action: action + }); } } diff --git a/modules/remotecontrol/RemoteControl.js b/modules/remotecontrol/RemoteControl.js index 2da4680dd..76d06a8ec 100644 --- a/modules/remotecontrol/RemoteControl.js +++ b/modules/remotecontrol/RemoteControl.js @@ -1,6 +1,8 @@ /* global APP, config */ import Controller from "./Controller"; import Receiver from "./Receiver"; +import {EVENT_TYPES} + from "../../service/remotecontrol/Constants"; /** * Implements the remote control functionality. @@ -32,14 +34,28 @@ class RemoteControl { this.controller.enable(true); } + /** + * Handles remote control events from the API module. + * @param {object} event the remote control event + */ + onRemoteControlAPIEvent(event) { + switch(event.type) { + case EVENT_TYPES.supported: + this._onRemoteControlSupported(); + break; + case EVENT_TYPES.permissions: + this.receiver._onRemoteControlPermissionsEvent( + event.userId, event.action); + break; + } + } + /** * Handles API event for support for executing remote control events into * the wrapper application. - * @param {boolean} isSupported true if the receiver side is supported by - * the wrapper application. */ - onRemoteControlSupported(isSupported) { - if(isSupported && !config.disableRemoteControl) { + _onRemoteControlSupported() { + if(!config.disableRemoteControl) { this.enabled = true; if(this.initialized) { this.receiver.enable(true); diff --git a/modules/remotecontrol/RemoteControlParticipant.js b/modules/remotecontrol/RemoteControlParticipant.js new file mode 100644 index 000000000..75323d7f6 --- /dev/null +++ b/modules/remotecontrol/RemoteControlParticipant.js @@ -0,0 +1,36 @@ +/* global APP */ +import {REMOTE_CONTROL_EVENT_TYPE} + from "../../service/remotecontrol/Constants"; + +export default class RemoteControlParticipant { + /** + * Creates new instance. + */ + constructor() { + this.enabled = false; + } + + /** + * Enables / Disables the remote control + * @param {boolean} enabled the new state. + */ + enable(enabled) { + this.enabled = enabled; + } + + /** + * Sends remote control event to other participant trough data channel. + * @param {Object} event the remote control event. + * @param {Function} onDataChannelFail handler for data channel failure. + */ + _sendRemoteControlEvent(to, event, onDataChannelFail = () => {}) { + if(!this.enabled || !to) + return; + try{ + APP.conference.sendEndpointMessage(to, + {type: REMOTE_CONTROL_EVENT_TYPE, event}); + } catch (e) { + onDataChannelFail(e); + } + } +} diff --git a/service/remotecontrol/Constants.js b/service/remotecontrol/Constants.js index 34e776ea9..810e57f11 100644 --- a/service/remotecontrol/Constants.js +++ b/service/remotecontrol/Constants.js @@ -14,10 +14,22 @@ export const EVENT_TYPES = { mousedblclick: "mousedblclick", mousescroll: "mousescroll", keydown: "keydown", - keyup: "keyup" + keyup: "keyup", + permissions: "permissions", + stop: "stop", + supported: "supported" +}; + +/** + * Actions for the remote control permission events. + */ +export const PERMISSIONS_ACTIONS = { + request: "request", + grant: "grant", + deny: "deny" }; /** * The type of remote control events sent trough the API module. */ -export const API_EVENT_TYPE = "remote-control-event"; +export const REMOTE_CONTROL_EVENT_TYPE = "remote-control-event"; From a4d5c41b3ab103207b18d84b350e38d3023596b5 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Thu, 5 Jan 2017 19:18:07 -0600 Subject: [PATCH 04/18] feat(remotecontrol): UI for requesting permissions --- conference.js | 13 +- css/_filmstrip.scss | 6 +- css/_popup_menu.scss | 8 +- lang/main.json | 11 +- modules/UI/UI.js | 7 ++ modules/UI/videolayout/RemoteVideo.js | 162 +++++++++++++++++++------ modules/UI/videolayout/VideoLayout.js | 31 ++++- modules/remotecontrol/Controller.js | 112 +++++++++++++---- modules/remotecontrol/Receiver.js | 95 +++++++++++++-- modules/remotecontrol/RemoteControl.js | 20 ++- service/remotecontrol/Constants.js | 3 +- 11 files changed, 378 insertions(+), 90 deletions(-) diff --git a/conference.js b/conference.js index 36309b9cd..d198b354e 100644 --- a/conference.js +++ b/conference.js @@ -488,11 +488,11 @@ export default { }).then(([tracks, con]) => { logger.log('initialized with %s local tracks', tracks.length); APP.connection = connection = con; + this.isDesktopSharingEnabled = + JitsiMeetJS.isDesktopSharingEnabled(); APP.remoteControl.init(); this._bindConnectionFailedHandler(con); this._createRoom(tracks); - this.isDesktopSharingEnabled = - JitsiMeetJS.isDesktopSharingEnabled(); if (UIUtil.isButtonEnabled('contacts') && !interfaceConfig.filmStripOnly) { @@ -985,7 +985,7 @@ export default { let externalInstallation = false; if (shareScreen) { - createLocalTracks({ + this.screenSharingPromise = createLocalTracks({ devices: ['desktop'], desktopSharingExtensionExternalInstallation: { interval: 500, @@ -1075,7 +1075,9 @@ export default { }); } else { APP.remoteControl.receiver.stop(); - createLocalTracks({ devices: ['video'] }).then( + this.screenSharingPromise = createLocalTracks( + { devices: ['video'] }) + .then( ([stream]) => this.useVideoStream(stream) ).then(() => { this.videoSwitchInProgress = false; @@ -1107,6 +1109,8 @@ export default { } ); + room.on(ConferenceEvents.PARTCIPANT_FEATURES_CHANGED, + user => APP.UI.onUserFeaturesChanged(user)); room.on(ConferenceEvents.USER_JOINED, (id, user) => { if (user.isHidden()) return; @@ -1780,6 +1784,7 @@ export default { */ hangup (requestFeedback = false) { APP.UI.hideRingOverLay(); + APP.remoteControl.receiver.enable(false); let requestFeedbackPromise = requestFeedback ? APP.UI.requestFeedbackOnHangup() // false - because the thank you dialog shouldn't be displayed diff --git a/css/_filmstrip.scss b/css/_filmstrip.scss index dddbc886e..1b6dd089e 100644 --- a/css/_filmstrip.scss +++ b/css/_filmstrip.scss @@ -92,7 +92,7 @@ 0 0 3px $videoThumbnailSelected !important; } - .remotevideomenu { + .remotevideomenu > .icon-menu { display: none; } @@ -105,7 +105,7 @@ box-shadow: inset 0 0 3px $videoThumbnailHovered, 0 0 3px $videoThumbnailHovered; - .remotevideomenu { + .remotevideomenu > .icon-menu { display: inline-block; } } @@ -121,4 +121,4 @@ } } } -} \ No newline at end of file +} diff --git a/css/_popup_menu.scss b/css/_popup_menu.scss index f129321e0..2283926d4 100644 --- a/css/_popup_menu.scss +++ b/css/_popup_menu.scss @@ -6,7 +6,6 @@ padding: 0; margin: 2px 0; bottom: 0; - width: 100px; height: auto; &:first-child { @@ -66,4 +65,9 @@ span.remotevideomenu:hover ul.popupmenu, ul.popupmenu:hover { display:block !important; -} \ No newline at end of file +} + +.remote-control-spinner { + top: 6px; + left: 2px; +} diff --git a/lang/main.json b/lang/main.json index 2dc1487fd..3e398633a 100644 --- a/lang/main.json +++ b/lang/main.json @@ -156,8 +156,8 @@ "kick": "Kick out", "muted": "Muted", "domute": "Mute", - "flip": "Flip" - + "flip": "Flip", + "remoteControl": "Remote control" }, "connectionindicator": { @@ -316,7 +316,12 @@ "externalInstallationMsg": "You need to install our desktop sharing extension.", "muteParticipantTitle": "Mute this participant?", "muteParticipantBody": "You won't be able to unmute them, but they can unmute themselves at any time.", - "muteParticipantButton": "Mute" + "muteParticipantButton": "Mute", + "remoteControlTitle": "Remote Control", + "remoteControlDeniedMessage": "__user__ rejected your remote control request!", + "remoteControlAllowedMessage": "__user__ accepted your remote control request!", + "remoteControlErrorMessage": "An error occurred while trying to request remote control permissions from __user__!", + "remoteControlStopMessage": "The remote control session ended!" }, "email": { diff --git a/modules/UI/UI.js b/modules/UI/UI.js index 75752268f..624b5c1ba 100644 --- a/modules/UI/UI.js +++ b/modules/UI/UI.js @@ -1441,4 +1441,11 @@ UI.hideUserMediaPermissionsGuidanceOverlay = function () { GumPermissionsOverlay.hide(); }; +/** + * Handles user's features changes. + */ +UI.onUserFeaturesChanged = function (user) { + VideoLayout.onUserFeaturesChanged(user); +}; + module.exports = UI; diff --git a/modules/UI/videolayout/RemoteVideo.js b/modules/UI/videolayout/RemoteVideo.js index ac156ddbd..aae794885 100644 --- a/modules/UI/videolayout/RemoteVideo.js +++ b/modules/UI/videolayout/RemoteVideo.js @@ -29,6 +29,7 @@ function RemoteVideo(user, VideoLayout, emitter) { this.videoSpanId = `participant_${this.id}`; SmallVideo.call(this, VideoLayout); this.hasRemoteVideoMenu = false; + this._supportsRemoteControl = false; this.addRemoteVideoContainer(); this.connectionIndicator = new ConnectionIndicator(this, this.id); this.setDisplayName(); @@ -64,7 +65,7 @@ RemoteVideo.prototype.addRemoteVideoContainer = function() { this.initBrowserSpecificProperties(); - if (APP.conference.isModerator) { + if (APP.conference.isModerator || this._supportsRemoteControl) { this.addRemoteVideoMenu(); } @@ -106,14 +107,6 @@ RemoteVideo.prototype._initPopupMenu = function (popupMenuElement) { // call the original show, passing its actual this origShowFunc.call(this.popover); }.bind(this); - - // override popover hide method so we can cleanup click handlers - let origHideFunc = this.popover.forceHide; - this.popover.forceHide = function () { - $(document).off("click", '#mutelink_' + this.id); - $(document).off("click", '#ejectlink_' + this.id); - origHideFunc.call(this.popover); - }.bind(this); }; /** @@ -139,38 +132,68 @@ RemoteVideo.prototype._generatePopupContent = function () { let popupmenuElement = document.createElement('ul'); popupmenuElement.className = 'popupmenu'; popupmenuElement.id = `remote_popupmenu_${this.id}`; + let menuItems = []; - let muteTranslationKey; - let muteClassName; - if (this.isAudioMuted) { - muteTranslationKey = 'videothumbnail.muted'; - muteClassName = 'mutelink disabled'; - } else { - muteTranslationKey = 'videothumbnail.domute'; - muteClassName = 'mutelink'; + if(APP.conference.isModerator) { + let muteTranslationKey; + let muteClassName; + if (this.isAudioMuted) { + muteTranslationKey = 'videothumbnail.muted'; + muteClassName = 'mutelink disabled'; + } else { + muteTranslationKey = 'videothumbnail.domute'; + muteClassName = 'mutelink'; + } + + let muteHandler = this._muteHandler.bind(this); + let kickHandler = this._kickHandler.bind(this); + + menuItems = [ + { + id: 'mutelink_' + this.id, + handler: muteHandler, + icon: 'icon-mic-disabled', + className: muteClassName, + data: { + i18n: muteTranslationKey + } + }, { + id: 'ejectlink_' + this.id, + handler: kickHandler, + icon: 'icon-kick', + data: { + i18n: 'videothumbnail.kick' + } + } + ]; } - let muteHandler = this._muteHandler.bind(this); - let kickHandler = this._kickHandler.bind(this); - - let menuItems = [ - { - id: 'mutelink_' + this.id, - handler: muteHandler, - icon: 'icon-mic-disabled', - className: muteClassName, - data: { - i18n: muteTranslationKey - } - }, { - id: 'ejectlink_' + this.id, - handler: kickHandler, - icon: 'icon-kick', - data: { - i18n: 'videothumbnail.kick' - } + if(this._supportsRemoteControl) { + let icon, handler, className; + if(APP.remoteControl.controller.getRequestedParticipant() + === this.id) { + handler = () => {}; + className = "requestRemoteControlLink disabled"; + icon = "remote-control-spinner fa fa-spinner fa-spin"; + } else if(!APP.remoteControl.controller.isStarted()) { + handler = this._requestRemoteControlPermissions.bind(this); + icon = "fa fa-play"; + className = "requestRemoteControlLink"; + } else { + handler = this._stopRemoteControl.bind(this); + icon = "fa fa-stop"; + className = "requestRemoteControlLink"; } - ]; + menuItems.push({ + id: 'remoteControl_' + this.id, + handler, + icon, + className, + data: { + i18n: 'videothumbnail.remoteControl' + } + }); + } menuItems.forEach(el => { let menuItem = this._generatePopupMenuItem(el); @@ -182,6 +205,68 @@ RemoteVideo.prototype._generatePopupContent = function () { return popupmenuElement; }; +/** + * Sets the remote control supported value and initializes or updates the menu + * depending on the remote control is supported or not. + * @param {boolean} isSupported + */ +RemoteVideo.prototype.setRemoteControlSupport = function(isSupported = false) { + if(this._supportsRemoteControl === isSupported) { + return; + } + this._supportsRemoteControl = isSupported; + if(!isSupported) { + return; + } + + if(!this.hasRemoteVideoMenu) { + //create menu + this.addRemoteVideoMenu(); + } else { + //update the content + this.updateRemoteVideoMenu(this.isAudioMuted, true); + } + +}; + +/** + * Requests permissions for remote control session. + */ +RemoteVideo.prototype._requestRemoteControlPermissions = function () { + APP.remoteControl.controller.requestPermissions(this.id).then(result => { + if(result === null) { + return; + } + this.updateRemoteVideoMenu(this.isAudioMuted, true); + APP.UI.messageHandler.openMessageDialog( + "dialog.remoteControlTitle", + (result === false) ? "dialog.remoteControlDeniedMessage" + : "dialog.remoteControlAllowedMessage", + {user: this.user.getDisplayName() + || interfaceConfig.DEFAULT_REMOTE_DISPLAY_NAME} + ); + }, error => { + logger.error(error); + this.updateRemoteVideoMenu(this.isAudioMuted, true); + APP.UI.messageHandler.openMessageDialog( + "dialog.remoteControlTitle", + "dialog.remoteControlErrorMessage", + {user: this.user.getDisplayName() + || interfaceConfig.DEFAULT_REMOTE_DISPLAY_NAME} + ); + }); + this.updateRemoteVideoMenu(this.isAudioMuted, true); +}; + +/** + * Stops remote control session. + */ +RemoteVideo.prototype._stopRemoteControl = function () { + // send message about stopping + APP.remoteControl.controller.stop(); + this.updateRemoteVideoMenu(this.isAudioMuted, true); +}; + RemoteVideo.prototype._muteHandler = function () { if (this.isAudioMuted) return; @@ -244,8 +329,7 @@ RemoteVideo.prototype._generatePopupMenuItem = function (opts = {}) { linkItem.appendChild(textContent); linkItem.id = id; - // Delegate event to the document. - $(document).on("click", `#${id}`, handler); + linkItem.onclick = handler; menuItem.appendChild(linkItem); return menuItem; diff --git a/modules/UI/videolayout/VideoLayout.js b/modules/UI/videolayout/VideoLayout.js index 9f80bc017..97c273f67 100644 --- a/modules/UI/videolayout/VideoLayout.js +++ b/modules/UI/videolayout/VideoLayout.js @@ -406,6 +406,7 @@ var VideoLayout = { remoteVideo = smallVideo; else remoteVideo = new RemoteVideo(user, VideoLayout, eventEmitter); + this._setRemoteControlProperties(user, remoteVideo); this.addRemoteVideoContainer(id, remoteVideo); }, @@ -1158,12 +1159,36 @@ var VideoLayout = { * Sets the flipX state of the local video. * @param {boolean} true for flipped otherwise false; */ - setLocalFlipX: function (val) { + setLocalFlipX (val) { this.localFlipX = val; - }, - getEventEmitter: () => {return eventEmitter;} + getEventEmitter() {return eventEmitter;}, + + /** + * Handles user's features changes. + */ + onUserFeaturesChanged (user) { + let video = this.getSmallVideo(user.getId()); + + if (!video) { + return; + } + this._setRemoteControlProperties(user, video); + }, + + /** + * Sets the remote control properties (checks whether remote control + * is supported and executes remoteVideo.setRemoteControlSupport). + * @param {JitsiParticipant} user the user that will be checked for remote + * control support. + * @param {RemoteVideo} remoteVideo the remoteVideo on which the properties + * will be set. + */ + _setRemoteControlProperties (user, remoteVideo) { + APP.remoteControl.checkUserRemoteControlSupport(user).then(result => + remoteVideo.setRemoteControlSupport(result)); + } }; export default VideoLayout; diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index 6e60e1189..e10356a01 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -55,20 +55,34 @@ export default class Controller extends RemoteControlParticipant { super(); this.controlledParticipant = null; this.requestedParticipant = null; - this.stopListener = this._handleRemoteControlStoppedEvent.bind(this); + this._stopListener = this._handleRemoteControlStoppedEvent.bind(this); + this._userLeftListener = this._onUserLeft.bind(this); } /** * Requests permissions from the remote control receiver side. * @param {string} userId the user id of the participant that will be * requested. + * @returns {Promise} - resolve values: + * true - accept + * false - deny + * null - the participant has left. */ requestPermissions(userId) { if(!this.enabled) { return Promise.reject(new Error("Remote control is disabled!")); } return new Promise((resolve, reject) => { - let permissionsReplyListener = (participant, event) => { + const clearRequest = () => { + this.requestedParticipant = null; + APP.conference.removeConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + permissionsReplyListener); + APP.conference.removeConferenceListener( + ConferenceEvents.USER_LEFT, + onUserLeft); + }; + const permissionsReplyListener = (participant, event) => { let result = null; try { result = this._handleReply(participant, event); @@ -76,25 +90,27 @@ export default class Controller extends RemoteControlParticipant { reject(e); } if(result !== null) { - this.requestedParticipant = null; - APP.conference.removeConferenceListener( - ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - permissionsReplyListener); + clearRequest(); resolve(result); } }; + const onUserLeft = (id) => { + if(id === this.requestedParticipant) { + clearRequest(); + resolve(null); + } + }; APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, permissionsReplyListener); + APP.conference.addConferenceListener(ConferenceEvents.USER_LEFT, + onUserLeft); this.requestedParticipant = userId; this._sendRemoteControlEvent(userId, { type: EVENT_TYPES.permissions, action: PERMISSIONS_ACTIONS.request }, e => { - APP.conference.removeConferenceListener( - ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - permissionsReplyListener); - this.requestedParticipant = null; + clearRequest(); reject(e); }); }); @@ -112,14 +128,18 @@ export default class Controller extends RemoteControlParticipant { if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE && remoteControlEvent.type === EVENT_TYPES.permissions && userId === this.requestedParticipant) { - if(remoteControlEvent.action === PERMISSIONS_ACTIONS.grant) { - this.controlledParticipant = userId; - this._start(); - return true; - } else if(remoteControlEvent.action === PERMISSIONS_ACTIONS.deny) { - return false; - } else { - throw new Error("Unknown reply received!"); + switch(remoteControlEvent.action) { + case PERMISSIONS_ACTIONS.grant: { + this.controlledParticipant = userId; + this._start(); + return true; + } + case PERMISSIONS_ACTIONS.deny: + return false; + case PERMISSIONS_ACTIONS.error: + throw new Error("Error occurred on receiver side"); + default: + throw new Error("Unknown reply received!"); } } else { //different message type or another user -> ignoring the message @@ -149,7 +169,9 @@ export default class Controller extends RemoteControlParticipant { return; APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - this.stopListener); + this._stopListener); + APP.conference.addConferenceListener(ConferenceEvents.USER_LEFT, + this._userLeftListener); this.area = $("#largeVideoWrapper"); this.area.mousemove(event => { const position = this.area.position(); @@ -179,12 +201,17 @@ export default class Controller extends RemoteControlParticipant { } /** - * Stops processing the mouse and keyboard events. + * Stops processing the mouse and keyboard events. Removes added listeners. */ _stop() { + if(!this.controlledParticipant) { + return; + } APP.conference.removeConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - this.stopListener); + this._stopListener); + APP.conference.removeConferenceListener(ConferenceEvents.USER_LEFT, + this._userLeftListener); this.controlledParticipant = null; this.area.off( "mousemove" ); this.area.off( "mousedown" ); @@ -194,6 +221,23 @@ export default class Controller extends RemoteControlParticipant { $(window).off( "keydown"); $(window).off( "keyup"); this.area[0].onmousewheel = undefined; + APP.UI.messageHandler.openMessageDialog( + "dialog.remoteControlTitle", + "dialog.remoteControlStopMessage" + ); + } + + /** + * Calls this._stop() and sends stop message to the controlled participant. + */ + stop() { + if(!this.controlledParticipant) { + return; + } + this._sendRemoteControlEvent(this.controlledParticipant, { + type: EVENT_TYPES.stop + }); + this._stop(); } /** @@ -208,6 +252,22 @@ export default class Controller extends RemoteControlParticipant { }); } + /** + * Returns true if the remote control session is started. + * @returns {boolean} + */ + isStarted() { + return this.controlledParticipant !== null; + } + + /** + * Returns the id of the requested participant + * @returns {string} this.requestedParticipant + */ + getRequestedParticipant() { + return this.requestedParticipant; + } + /** * Handler for key press events. * @param {String} type the type of event ("keydown"/"keyup") @@ -220,4 +280,14 @@ export default class Controller extends RemoteControlParticipant { modifiers: getModifiers(event), }); } + + /** + * Calls the stop method if the other side have left. + * @param {string} id - the user id for the participant that have left + */ + _onUserLeft(id) { + if(this.controlledParticipant === id) { + this._stop(); + } + } } diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index e23017b7e..eeb1f1763 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -1,4 +1,4 @@ -/* global APP, JitsiMeetJS */ +/* global APP, JitsiMeetJS, interfaceConfig */ import {DISCO_REMOTE_CONTROL_FEATURE, REMOTE_CONTROL_EVENT_TYPE, EVENT_TYPES, PERMISSIONS_ACTIONS} from "../../service/remotecontrol/Constants"; import RemoteControlParticipant from "./RemoteControlParticipant"; @@ -21,6 +21,7 @@ export default class Receiver extends RemoteControlParticipant { this.controller = null; this._remoteControlEventsListener = this._onRemoteControlEvent.bind(this); + this._userLeftListener = this._onUserLeft.bind(this); } /** @@ -28,30 +29,60 @@ export default class Receiver extends RemoteControlParticipant { * @param {boolean} enabled the new state. */ enable(enabled) { - if(this.enabled !== enabled && enabled === true) { + if(this.enabled !== enabled) { this.enabled = enabled; + } + if(enabled === true) { // Announce remote control support. APP.connection.addFeature(DISCO_REMOTE_CONTROL_FEATURE, true); APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._remoteControlEventsListener); + } else { + this._stop(true); + APP.connection.removeFeature(DISCO_REMOTE_CONTROL_FEATURE); + APP.conference.removeConferenceListener( + ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + this._remoteControlEventsListener); } } /** * Removes the listener for ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED - * events. + * events. Sends stop message to the wrapper application. Optionally + * displays dialog for informing the user that remote control session + * ended. + * @param {boolean} dontShowDialog - if true the dialog won't be displayed. + */ + _stop(dontShowDialog = false) { + if(!this.controller) { + return; + } + this.controller = null; + APP.conference.removeConferenceListener(ConferenceEvents.USER_LEFT, + this._userLeftListener); + APP.API.sendRemoteControlEvent({ + type: EVENT_TYPES.stop + }); + if(!dontShowDialog) { + APP.UI.messageHandler.openMessageDialog( + "dialog.remoteControlTitle", + "dialog.remoteControlStopMessage" + ); + } + } + + /** + * Calls this._stop() and sends stop message to the controller participant */ stop() { - APP.conference.removeConferenceListener( - ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, - this._remoteControlEventsListener); - const event = { + if(!this.controller) { + return; + } + this._sendRemoteControlEvent(this.controller, { type: EVENT_TYPES.stop - }; - this._sendRemoteControlEvent(this.controller, event); - this.controller = null; - APP.API.sendRemoteControlEvent(event); + }); + this._stop(); } /** @@ -67,9 +98,15 @@ export default class Receiver extends RemoteControlParticipant { && remoteControlEvent.action === PERMISSIONS_ACTIONS.request) { remoteControlEvent.userId = participant.getId(); remoteControlEvent.userJID = participant.getJid(); - remoteControlEvent.displayName = participant.getDisplayName(); + remoteControlEvent.displayName = participant.getDisplayName() + || interfaceConfig.DEFAULT_REMOTE_DISPLAY_NAME; + remoteControlEvent.screenSharing + = APP.conference.isSharingScreen; } else if(this.controller !== participant.getId()) { return; + } else if(remoteControlEvent.type === EVENT_TYPES.stop) { + this._stop(); + return; } APP.API.sendRemoteControlEvent(remoteControlEvent); } @@ -83,11 +120,45 @@ export default class Receiver extends RemoteControlParticipant { */ _onRemoteControlPermissionsEvent(userId, action) { if(action === PERMISSIONS_ACTIONS.grant) { + APP.conference.addConferenceListener(ConferenceEvents.USER_LEFT, + this._userLeftListener); this.controller = userId; + if(!APP.conference.isSharingScreen) { + APP.conference.toggleScreenSharing(); + APP.conference.screenSharingPromise.then(() => { + if(APP.conference.isSharingScreen) { + this._sendRemoteControlEvent(userId, { + type: EVENT_TYPES.permissions, + action: action + }); + } else { + this._sendRemoteControlEvent(userId, { + type: EVENT_TYPES.permissions, + action: PERMISSIONS_ACTIONS.error + }); + } + }).catch(() => { + this._sendRemoteControlEvent(userId, { + type: EVENT_TYPES.permissions, + action: PERMISSIONS_ACTIONS.error + }); + }); + return; + } } this._sendRemoteControlEvent(userId, { type: EVENT_TYPES.permissions, action: action }); } + + /** + * Calls the stop method if the other side have left. + * @param {string} id - the user id for the participant that have left + */ + _onUserLeft(id) { + if(this.controller === id) { + this._stop(); + } + } } diff --git a/modules/remotecontrol/RemoteControl.js b/modules/remotecontrol/RemoteControl.js index 76d06a8ec..e00f8f1af 100644 --- a/modules/remotecontrol/RemoteControl.js +++ b/modules/remotecontrol/RemoteControl.js @@ -1,7 +1,7 @@ /* global APP, config */ import Controller from "./Controller"; import Receiver from "./Receiver"; -import {EVENT_TYPES} +import {EVENT_TYPES, DISCO_REMOTE_CONTROL_FEATURE} from "../../service/remotecontrol/Constants"; /** @@ -24,7 +24,8 @@ class RemoteControl { * enabled or not, initializes the API module. */ init() { - if(config.disableRemoteControl || this.initialized) { + if(config.disableRemoteControl || this.initialized + || !APP.conference.isDesktopSharingEnabled) { return; } this.initialized = true; @@ -32,6 +33,9 @@ class RemoteControl { forceEnable: true, }); this.controller.enable(true); + if(this.enabled) { // supported message came before init. + this._onRemoteControlSupported(); + } } /** @@ -62,6 +66,18 @@ class RemoteControl { } } } + + /** + * Checks whether the passed user supports remote control or not + * @param {JitsiParticipant} user the user to be tested + * @returns {Promise} the promise will be resolved with true if + * the user supports remote control and with false if not. + */ + checkUserRemoteControlSupport(user) { + return user.getFeatures().then(features => + features.has(DISCO_REMOTE_CONTROL_FEATURE), () => false + ); + } } export default new RemoteControl(); diff --git a/service/remotecontrol/Constants.js b/service/remotecontrol/Constants.js index 810e57f11..afc8cc15e 100644 --- a/service/remotecontrol/Constants.js +++ b/service/remotecontrol/Constants.js @@ -26,7 +26,8 @@ export const EVENT_TYPES = { export const PERMISSIONS_ACTIONS = { request: "request", grant: "grant", - deny: "deny" + deny: "deny", + error: "error" }; /** From 5d269ad0aadb4419df3b89b6d100b8b4c9aef3a7 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Fri, 6 Jan 2017 16:03:54 -0600 Subject: [PATCH 05/18] fix(remotecontrol): Disable the keyboard shortcuts during remote control --- modules/keyboardshortcut/keyboardshortcut.js | 20 ++++++++++++++++++++ modules/remotecontrol/Controller.js | 2 ++ 2 files changed, 22 insertions(+) diff --git a/modules/keyboardshortcut/keyboardshortcut.js b/modules/keyboardshortcut/keyboardshortcut.js index 53497347a..1eac7b6bf 100644 --- a/modules/keyboardshortcut/keyboardshortcut.js +++ b/modules/keyboardshortcut/keyboardshortcut.js @@ -65,6 +65,12 @@ function showKeyboardShortcutsPanel(show) { */ let _shortcuts = {}; +/** + * True if the keyboard shortcuts are enabled and false if not. + * @type {boolean} + */ +let enabled = true; + /** * Maps keycode to character, id of popover for given function and function. */ @@ -74,6 +80,9 @@ var KeyboardShortcut = { var self = this; window.onkeyup = function(e) { + if(!enabled) { + return; + } var key = self._getKeyboardKey(e).toUpperCase(); var num = parseInt(key, 10); if(!($(":focus").is("input[type=text]") || @@ -93,6 +102,9 @@ var KeyboardShortcut = { }; window.onkeydown = function(e) { + if(!enabled) { + return; + } if(!($(":focus").is("input[type=text]") || $(":focus").is("input[type=password]") || $(":focus").is("textarea"))) { @@ -105,6 +117,14 @@ var KeyboardShortcut = { }; }, + /** + * Enables/Disables the keyboard shortcuts. + * @param {boolean} value - the new value. + */ + enable: function (value) { + enabled = value; + }, + /** * Registers a new shortcut. * diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index e10356a01..72f478976 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -167,6 +167,7 @@ export default class Controller extends RemoteControlParticipant { _start() { if(!this.enabled) return; + APP.keyboardshortcut.enable(false); APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._stopListener); @@ -207,6 +208,7 @@ export default class Controller extends RemoteControlParticipant { if(!this.controlledParticipant) { return; } + APP.keyboardshortcut.enable(true); APP.conference.removeConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._stopListener); From 84be7fd73910dec64ef05d22e8247dab38ecc612 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Mon, 9 Jan 2017 14:10:58 -0600 Subject: [PATCH 06/18] fix(remotecontrol): import of remote control --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index 9207cf6ca..f892d3e15 100644 --- a/app.js +++ b/app.js @@ -22,7 +22,7 @@ import conference from './conference'; import API from './modules/API/API'; import translation from "./modules/translation/translation"; -import remoteControl from "./modules/remotecontrol/remotecontrol"; +import remoteControl from "./modules/remotecontrol/RemoteControl"; const APP = { // Used by do_external_connect.js if we receive the attach data after From 5d22061c0a18f72f9ddd49ad5396743bd78a0fc0 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Tue, 10 Jan 2017 12:51:25 -0600 Subject: [PATCH 07/18] fix(remotecontrol): Handle on-stage participant changes --- modules/UI/videolayout/LargeVideoManager.js | 3 + modules/UI/videolayout/RemoteVideo.js | 4 + modules/remotecontrol/Controller.js | 90 ++++++++++++++++++--- service/UI/UIEvents.js | 5 ++ 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/modules/UI/videolayout/LargeVideoManager.js b/modules/UI/videolayout/LargeVideoManager.js index 225370ddf..0a52afb01 100644 --- a/modules/UI/videolayout/LargeVideoManager.js +++ b/modules/UI/videolayout/LargeVideoManager.js @@ -3,6 +3,7 @@ const logger = require("jitsi-meet-logger").getLogger(__filename); import Avatar from "../avatar/Avatar"; import {createDeferred} from '../../util/helpers'; +import UIEvents from "../../../service/UI/UIEvents"; import UIUtil from "../util/UIUtil"; import {VideoContainer, VIDEO_CONTAINER_TYPE} from "./VideoContainer"; @@ -19,6 +20,7 @@ export default class LargeVideoManager { * @type {Object.} */ this.containers = {}; + this.eventEmitter = emitter; this.state = VIDEO_CONTAINER_TYPE; this.videoContainer = new VideoContainer( @@ -164,6 +166,7 @@ export default class LargeVideoManager { // after everything is done check again if there are any pending // new streams. this.updateInProcess = false; + this.eventEmitter.emit(UIEvents.LARGE_VIDEO_ID_CHANGED, this.id); this.scheduleLargeVideoUpdate(); }); } diff --git a/modules/UI/videolayout/RemoteVideo.js b/modules/UI/videolayout/RemoteVideo.js index aae794885..b4dd66678 100644 --- a/modules/UI/videolayout/RemoteVideo.js +++ b/modules/UI/videolayout/RemoteVideo.js @@ -245,6 +245,10 @@ RemoteVideo.prototype._requestRemoteControlPermissions = function () { {user: this.user.getDisplayName() || interfaceConfig.DEFAULT_REMOTE_DISPLAY_NAME} ); + let pinnedId = this.VideoLayout.getPinnedId(); + if(pinnedId !== this.id) { + this.VideoLayout.handleVideoThumbClicked(this.id); + } }, error => { logger.error(error); this.updateRemoteVideoMenu(this.isAudioMuted, true); diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index 72f478976..aadcc0cbc 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -3,6 +3,7 @@ import * as KeyCodes from "../keycode/keycode"; import {EVENT_TYPES, REMOTE_CONTROL_EVENT_TYPE, PERMISSIONS_ACTIONS} from "../../service/remotecontrol/Constants"; import RemoteControlParticipant from "./RemoteControlParticipant"; +import UIEvents from "../../service/UI/UIEvents"; const ConferenceEvents = JitsiMeetJS.events.conference; @@ -53,10 +54,13 @@ export default class Controller extends RemoteControlParticipant { */ constructor() { super(); + this.isCollectingEvents = false; this.controlledParticipant = null; this.requestedParticipant = null; this._stopListener = this._handleRemoteControlStoppedEvent.bind(this); this._userLeftListener = this._onUserLeft.bind(this); + this._largeVideoChangedListener + = this._onLargeVideoIdChanged.bind(this); } /** @@ -162,17 +166,36 @@ export default class Controller extends RemoteControlParticipant { } /** - * Starts processing the mouse and keyboard events. + * Starts processing the mouse and keyboard events. Sets conference + * listeners. Disables keyboard events. */ _start() { - if(!this.enabled) + if(!this.enabled) { return; - APP.keyboardshortcut.enable(false); + } + APP.UI.addListener(UIEvents.LARGE_VIDEO_ID_CHANGED, + this._largeVideoChangedListener); APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._stopListener); APP.conference.addConferenceListener(ConferenceEvents.USER_LEFT, this._userLeftListener); + this.resume(); + } + + /** + * Disables the keyboatd shortcuts. Starts collecting remote control + * events. + * + * It can be used to resume an active remote control session wchich was + * paused with this.pause(). + */ + resume() { + if(!this.enabled || this.isCollectingEvents) { + return; + } + this.isCollectingEvents = true; + APP.keyboardshortcut.enable(false); this.area = $("#largeVideoWrapper"); this.area.mousemove(event => { const position = this.area.position(); @@ -203,26 +226,22 @@ export default class Controller extends RemoteControlParticipant { /** * Stops processing the mouse and keyboard events. Removes added listeners. + * Enables the keyboard shortcuts. Displays dialog to notify the user that + * remote control session has ended. */ _stop() { if(!this.controlledParticipant) { return; } - APP.keyboardshortcut.enable(true); + APP.UI.removeListener(UIEvents.LARGE_VIDEO_ID_CHANGED, + this._largeVideoChangedListener); APP.conference.removeConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._stopListener); APP.conference.removeConferenceListener(ConferenceEvents.USER_LEFT, this._userLeftListener); this.controlledParticipant = null; - this.area.off( "mousemove" ); - this.area.off( "mousedown" ); - this.area.off( "mouseup" ); - this.area.off( "contextmenu" ); - this.area.off( "dblclick" ); - $(window).off( "keydown"); - $(window).off( "keyup"); - this.area[0].onmousewheel = undefined; + this.pause(); APP.UI.messageHandler.openMessageDialog( "dialog.remoteControlTitle", "dialog.remoteControlStopMessage" @@ -230,7 +249,13 @@ export default class Controller extends RemoteControlParticipant { } /** - * Calls this._stop() and sends stop message to the controlled participant. + * Executes this._stop() mehtod: + * Stops processing the mouse and keyboard events. Removes added listeners. + * Enables the keyboard shortcuts. Displays dialog to notify the user that + * remote control session has ended. + * + * In addition: + * Sends stop message to the controlled participant. */ stop() { if(!this.controlledParticipant) { @@ -242,6 +267,30 @@ export default class Controller extends RemoteControlParticipant { this._stop(); } + /** + * Pauses the collecting of events and enables the keyboard shortcus. But + * it doesn't removes any other listeners. Basically the remote control + * session will be still active after this.pause(), but no events from the + * controller side will be captured and sent. + * + * You can resume the collecting of the events with this.resume(). + */ + pause() { + if(!this.controlledParticipant) { + return; + } + this.isCollectingEvents = false; + APP.keyboardshortcut.enable(true); + this.area.off( "mousemove" ); + this.area.off( "mousedown" ); + this.area.off( "mouseup" ); + this.area.off( "contextmenu" ); + this.area.off( "dblclick" ); + $(window).off( "keydown"); + $(window).off( "keyup"); + this.area[0].onmousewheel = undefined; + } + /** * Handler for mouse click events. * @param {String} type the type of event ("mousedown"/"mouseup") @@ -292,4 +341,19 @@ export default class Controller extends RemoteControlParticipant { this._stop(); } } + + /** + * Handles changes of the participant displayed on the large video. + * @param {string} id - the user id for the participant that is displayed. + */ + _onLargeVideoIdChanged(id) { + if (!this.controlledParticipant) { + return; + } + if(this.controlledParticipant == id) { + this.resume(); + } else { + this.pause(); + } + } } diff --git a/service/UI/UIEvents.js b/service/UI/UIEvents.js index 28ba05535..30cac66b6 100644 --- a/service/UI/UIEvents.js +++ b/service/UI/UIEvents.js @@ -120,6 +120,11 @@ export default { */ LARGE_VIDEO_AVATAR_VISIBLE: "UI.large_video_avatar_visible", + /** + * Notifies that the displayed particpant id on the largeVideo is changed. + */ + LARGE_VIDEO_ID_CHANGED: "UI.large_video_id_changed", + /** * Toggling room lock */ From 0efca9a9a89f97514a79906eb5d0527933a06b7a Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Tue, 17 Jan 2017 18:16:18 -0600 Subject: [PATCH 08/18] fix(remotecontrol): Fixing issues after peer review. --- conference.js | 6 +++--- modules/API/API.js | 13 ++++++++++++- modules/keycode/keycode.js | 1 + modules/remotecontrol/Controller.js | 3 --- modules/remotecontrol/Receiver.js | 9 +++++++-- modules/remotecontrol/RemoteControl.js | 12 ++++++++++-- react/features/app/functions.web.js | 6 +++++- 7 files changed, 38 insertions(+), 12 deletions(-) diff --git a/conference.js b/conference.js index d198b354e..eb3472dd1 100644 --- a/conference.js +++ b/conference.js @@ -1838,9 +1838,9 @@ export default { /** * Sends a message via the data channel. - * @param to {string} the id of the endpoint that should receive the - * message. If "" the message will be sent to all participants. - * @param payload {object} the payload of the message. + * @param {string} to the id of the endpoint that should receive the + * message. If "" - the message will be sent to all participants. + * @param {object} payload the payload of the message. * @throws NetworkError or InvalidStateError or Error if the operation * fails. */ diff --git a/modules/API/API.js b/modules/API/API.js index 471ccfe48..fd53ed695 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -218,7 +218,18 @@ class API { /** * Sends remote control event. - * @param {object} event the event. + * @param {object} event the remote control event. The remote control event + * has one mandatory property - type which is string from EVENT_TYPES enum + * defined in /service/remotecontrol/constants. Depending on the event type + * the event can have other properties or not. + * mousemove - will also have {int} properties x and y + * mousedown, mouseup and mousedblclick - will have {int} property button + * with values - 1(left), 2(middle) or 3 (right) + * mousescroll - will have {int} property y + * keydown and keyup - will have {string} property key and array of strings + * property modifiers. + * stop - won't have any other properties + * permissions - will have {PERMISSIONS_ACTIONS} property action. */ sendRemoteControlEvent(event) { sendMessage({method: "remote-control-event", params: event}); diff --git a/modules/keycode/keycode.js b/modules/keycode/keycode.js index 94a81ba4b..b6f5c8d71 100644 --- a/modules/keycode/keycode.js +++ b/modules/keycode/keycode.js @@ -1,5 +1,6 @@ /** * Enumerates the supported keys. + * NOTE: The maps represents actual keys on the keyboard not chars. */ export const KEYS = { BACKSPACE: "backspace" , diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index aadcc0cbc..e5e885c68 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -170,9 +170,6 @@ export default class Controller extends RemoteControlParticipant { * listeners. Disables keyboard events. */ _start() { - if(!this.enabled) { - return; - } APP.UI.addListener(UIEvents.LARGE_VIDEO_ID_CHANGED, this._largeVideoChangedListener); APP.conference.addConferenceListener( diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index eeb1f1763..c1ccef848 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -86,9 +86,14 @@ export default class Receiver extends RemoteControlParticipant { } /** - * Sends "remote-control-event" events to to the API module. + * Listens for data channel EndpointMessage events. Handles only events of + * type remote control. Sends "remote-control-event" events to the API + * module. * @param {JitsiParticipant} participant the controller participant - * @param {Object} event the remote control event. + * @param {Object} event EndpointMessage event from the data channels. It + * has {string} type property. The function process only events of type + * REMOTE_CONTROL_EVENT_TYPE which will have event property(the remote + * control event) */ _onRemoteControlEvent(participant, event) { if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE) { diff --git a/modules/remotecontrol/RemoteControl.js b/modules/remotecontrol/RemoteControl.js index e00f8f1af..ed906a465 100644 --- a/modules/remotecontrol/RemoteControl.js +++ b/modules/remotecontrol/RemoteControl.js @@ -39,8 +39,16 @@ class RemoteControl { } /** - * Handles remote control events from the API module. - * @param {object} event the remote control event + * Handles remote control events from the API module. Currently only events + * with type = EVENT_TYPES.supported or EVENT_TYPES.permissions + * @param {object} event the remote control event. The remote control event + * has one mandatory property - type which is string from EVENT_TYPES enum + * defined in /service/remotecontrol/constants. If the event type is + * "supported" it won't have any other properties. If the event type is + * "permissions" the method will expect also the following properties: + * {string} userId - the user id of the participant that made the request + * for permissions + * {PERMISSIONS_ACTIONS} action the action related to the event. */ onRemoteControlAPIEvent(event) { switch(event.type) { diff --git a/react/features/app/functions.web.js b/react/features/app/functions.web.js index 77319fde8..22d161843 100644 --- a/react/features/app/functions.web.js +++ b/react/features/app/functions.web.js @@ -23,7 +23,11 @@ export function init() { APP.keyboardshortcut = KeyboardShortcut; APP.tokenData = getTokenData(); - APP.API.init(APP.tokenData.jwt ? {forceEnable: true} : undefined); + + // Force enable the API if jwt token is passed because most probably + // jitsi meet is displayed inside of wrapper that will need to communicate + // with jitsi meet. + APP.API.init(APP.tokenData.jwt ? { forceEnable: true } : undefined); APP.translation.init(settings.getLanguage()); } From e6935549610105b4ec96701a994b94ee33715bcc Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Thu, 19 Jan 2017 15:13:05 -0600 Subject: [PATCH 09/18] fix(remotecontrol): Logging --- modules/remotecontrol/Controller.js | 8 ++++++++ modules/remotecontrol/Receiver.js | 8 ++++++++ modules/remotecontrol/RemoteControl.js | 5 +++++ modules/remotecontrol/RemoteControlParticipant.js | 8 +++++++- 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index e5e885c68..4f4ebcb8d 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -1,4 +1,5 @@ /* global $, JitsiMeetJS, APP */ +const logger = require("jitsi-meet-logger").getLogger(__filename); import * as KeyCodes from "../keycode/keycode"; import {EVENT_TYPES, REMOTE_CONTROL_EVENT_TYPE, PERMISSIONS_ACTIONS} from "../../service/remotecontrol/Constants"; @@ -76,6 +77,7 @@ export default class Controller extends RemoteControlParticipant { if(!this.enabled) { return Promise.reject(new Error("Remote control is disabled!")); } + logger.debug("Requsting remote control permissions from: " + userId); return new Promise((resolve, reject) => { const clearRequest = () => { this.requestedParticipant = null; @@ -135,6 +137,8 @@ export default class Controller extends RemoteControlParticipant { switch(remoteControlEvent.action) { case PERMISSIONS_ACTIONS.grant: { this.controlledParticipant = userId; + logger.debug("Remote control permissions granted to: " + + userId); this._start(); return true; } @@ -170,6 +174,7 @@ export default class Controller extends RemoteControlParticipant { * listeners. Disables keyboard events. */ _start() { + logger.log("Starting remote control controller."); APP.UI.addListener(UIEvents.LARGE_VIDEO_ID_CHANGED, this._largeVideoChangedListener); APP.conference.addConferenceListener( @@ -191,6 +196,7 @@ export default class Controller extends RemoteControlParticipant { if(!this.enabled || this.isCollectingEvents) { return; } + logger.log("Resuming remote control controller."); this.isCollectingEvents = true; APP.keyboardshortcut.enable(false); this.area = $("#largeVideoWrapper"); @@ -230,6 +236,7 @@ export default class Controller extends RemoteControlParticipant { if(!this.controlledParticipant) { return; } + logger.log("Stopping remote control controller."); APP.UI.removeListener(UIEvents.LARGE_VIDEO_ID_CHANGED, this._largeVideoChangedListener); APP.conference.removeConferenceListener( @@ -276,6 +283,7 @@ export default class Controller extends RemoteControlParticipant { if(!this.controlledParticipant) { return; } + logger.log("Pausing remote control controller."); this.isCollectingEvents = false; APP.keyboardshortcut.enable(true); this.area.off( "mousemove" ); diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index c1ccef848..ea55bca80 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -1,4 +1,5 @@ /* global APP, JitsiMeetJS, interfaceConfig */ +const logger = require("jitsi-meet-logger").getLogger(__filename); import {DISCO_REMOTE_CONTROL_FEATURE, REMOTE_CONTROL_EVENT_TYPE, EVENT_TYPES, PERMISSIONS_ACTIONS} from "../../service/remotecontrol/Constants"; import RemoteControlParticipant from "./RemoteControlParticipant"; @@ -33,12 +34,14 @@ export default class Receiver extends RemoteControlParticipant { this.enabled = enabled; } if(enabled === true) { + logger.log("Remote control receiver enabled."); // Announce remote control support. APP.connection.addFeature(DISCO_REMOTE_CONTROL_FEATURE, true); APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._remoteControlEventsListener); } else { + logger.log("Remote control receiver disabled."); this._stop(true); APP.connection.removeFeature(DISCO_REMOTE_CONTROL_FEATURE); APP.conference.removeConferenceListener( @@ -58,6 +61,7 @@ export default class Receiver extends RemoteControlParticipant { if(!this.controller) { return; } + logger.log("Remote control receiver stop."); this.controller = null; APP.conference.removeConferenceListener(ConferenceEvents.USER_LEFT, this._userLeftListener); @@ -114,6 +118,9 @@ export default class Receiver extends RemoteControlParticipant { return; } APP.API.sendRemoteControlEvent(remoteControlEvent); + } else if(event.type === REMOTE_CONTROL_EVENT_TYPE) { + logger.debug("Remote control event is ignored because remote " + + "control is disabled", event); } } @@ -128,6 +135,7 @@ export default class Receiver extends RemoteControlParticipant { APP.conference.addConferenceListener(ConferenceEvents.USER_LEFT, this._userLeftListener); this.controller = userId; + logger.debug("Remote control permissions granted to: " + userId); if(!APP.conference.isSharingScreen) { APP.conference.toggleScreenSharing(); APP.conference.screenSharingPromise.then(() => { diff --git a/modules/remotecontrol/RemoteControl.js b/modules/remotecontrol/RemoteControl.js index ed906a465..f97b52754 100644 --- a/modules/remotecontrol/RemoteControl.js +++ b/modules/remotecontrol/RemoteControl.js @@ -1,4 +1,5 @@ /* global APP, config */ +const logger = require("jitsi-meet-logger").getLogger(__filename); import Controller from "./Controller"; import Receiver from "./Receiver"; import {EVENT_TYPES, DISCO_REMOTE_CONTROL_FEATURE} @@ -28,6 +29,7 @@ class RemoteControl { || !APP.conference.isDesktopSharingEnabled) { return; } + logger.log("Initializing remote control."); this.initialized = true; APP.API.init({ forceEnable: true, @@ -67,11 +69,14 @@ class RemoteControl { * the wrapper application. */ _onRemoteControlSupported() { + logger.log("Remote Control supported."); if(!config.disableRemoteControl) { this.enabled = true; if(this.initialized) { this.receiver.enable(true); } + } else { + logger.log("Remote Control disabled."); } } diff --git a/modules/remotecontrol/RemoteControlParticipant.js b/modules/remotecontrol/RemoteControlParticipant.js index 75323d7f6..b74f73fad 100644 --- a/modules/remotecontrol/RemoteControlParticipant.js +++ b/modules/remotecontrol/RemoteControlParticipant.js @@ -1,4 +1,5 @@ /* global APP */ +const logger = require("jitsi-meet-logger").getLogger(__filename); import {REMOTE_CONTROL_EVENT_TYPE} from "../../service/remotecontrol/Constants"; @@ -24,12 +25,17 @@ export default class RemoteControlParticipant { * @param {Function} onDataChannelFail handler for data channel failure. */ _sendRemoteControlEvent(to, event, onDataChannelFail = () => {}) { - if(!this.enabled || !to) + if(!this.enabled || !to) { + logger.warn("Remote control: Skip sending remote control event." + + " Params:", this.enable, to); return; + } try{ APP.conference.sendEndpointMessage(to, {type: REMOTE_CONTROL_EVENT_TYPE, event}); } catch (e) { + logger.error("Failed to send EndpointMessage via the datachannels", + e); onDataChannelFail(e); } } From b22e3ee25379785ad441fa1185f4cd3529f2414b Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Thu, 19 Jan 2017 17:19:58 -0600 Subject: [PATCH 10/18] style(remotecontrol): Fix JSDoc for RemoteControlEvent --- modules/API/API.js | 13 +------- modules/keycode/keycode.js | 4 ++- modules/remotecontrol/Controller.js | 7 ++-- modules/remotecontrol/Receiver.js | 8 ++--- modules/remotecontrol/RemoteControl.js | 9 +---- .../remotecontrol/RemoteControlParticipant.js | 2 +- service/remotecontrol/Constants.js | 33 +++++++++++++++++++ 7 files changed, 48 insertions(+), 28 deletions(-) diff --git a/modules/API/API.js b/modules/API/API.js index fd53ed695..5fb7423b5 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -218,18 +218,7 @@ class API { /** * Sends remote control event. - * @param {object} event the remote control event. The remote control event - * has one mandatory property - type which is string from EVENT_TYPES enum - * defined in /service/remotecontrol/constants. Depending on the event type - * the event can have other properties or not. - * mousemove - will also have {int} properties x and y - * mousedown, mouseup and mousedblclick - will have {int} property button - * with values - 1(left), 2(middle) or 3 (right) - * mousescroll - will have {int} property y - * keydown and keyup - will have {string} property key and array of strings - * property modifiers. - * stop - won't have any other properties - * permissions - will have {PERMISSIONS_ACTIONS} property action. + * @param {RemoteControlEvent} event the remote control event. */ sendRemoteControlEvent(event) { sendMessage({method: "remote-control-event", params: event}); diff --git a/modules/keycode/keycode.js b/modules/keycode/keycode.js index b6f5c8d71..2bbc2debc 100644 --- a/modules/keycode/keycode.js +++ b/modules/keycode/keycode.js @@ -1,6 +1,8 @@ /** * Enumerates the supported keys. - * NOTE: The maps represents actual keys on the keyboard not chars. + * NOTE: The maps represents actual keys on the keyboard not chars. + * @readonly + * @enum {string} */ export const KEYS = { BACKSPACE: "backspace" , diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index 4f4ebcb8d..ad4521a96 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -126,7 +126,7 @@ export default class Controller extends RemoteControlParticipant { * Handles the reply of the permissions request. * @param {JitsiParticipant} participant the participant that has sent the * reply - * @param {object} event the remote control event. + * @param {RemoteControlEvent} event the remote control event. */ _handleReply(participant, event) { const remoteControlEvent = event.event; @@ -159,7 +159,10 @@ export default class Controller extends RemoteControlParticipant { * Handles remote control stopped. * @param {JitsiParticipant} participant the participant that has sent the * event - * @param {object} event the the remote control event. + * @param {Object} event EndpointMessage event from the data channels. + * @property {string} type property. The function process only events of + * type REMOTE_CONTROL_EVENT_TYPE + * @property {RemoteControlEvent} event - the remote control event. */ _handleRemoteControlStoppedEvent(participant, event) { if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index ea55bca80..2ffdc5afd 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -94,10 +94,10 @@ export default class Receiver extends RemoteControlParticipant { * type remote control. Sends "remote-control-event" events to the API * module. * @param {JitsiParticipant} participant the controller participant - * @param {Object} event EndpointMessage event from the data channels. It - * has {string} type property. The function process only events of type - * REMOTE_CONTROL_EVENT_TYPE which will have event property(the remote - * control event) + * @param {Object} event EndpointMessage event from the data channels. + * @property {string} type property. The function process only events of + * type REMOTE_CONTROL_EVENT_TYPE + * @property {RemoteControlEvent} event - the remote control event. */ _onRemoteControlEvent(participant, event) { if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE) { diff --git a/modules/remotecontrol/RemoteControl.js b/modules/remotecontrol/RemoteControl.js index f97b52754..86f1f51e5 100644 --- a/modules/remotecontrol/RemoteControl.js +++ b/modules/remotecontrol/RemoteControl.js @@ -43,14 +43,7 @@ class RemoteControl { /** * Handles remote control events from the API module. Currently only events * with type = EVENT_TYPES.supported or EVENT_TYPES.permissions - * @param {object} event the remote control event. The remote control event - * has one mandatory property - type which is string from EVENT_TYPES enum - * defined in /service/remotecontrol/constants. If the event type is - * "supported" it won't have any other properties. If the event type is - * "permissions" the method will expect also the following properties: - * {string} userId - the user id of the participant that made the request - * for permissions - * {PERMISSIONS_ACTIONS} action the action related to the event. + * @param {RemoteControlEvent} event the remote control event. */ onRemoteControlAPIEvent(event) { switch(event.type) { diff --git a/modules/remotecontrol/RemoteControlParticipant.js b/modules/remotecontrol/RemoteControlParticipant.js index b74f73fad..0f283ba9f 100644 --- a/modules/remotecontrol/RemoteControlParticipant.js +++ b/modules/remotecontrol/RemoteControlParticipant.js @@ -21,7 +21,7 @@ export default class RemoteControlParticipant { /** * Sends remote control event to other participant trough data channel. - * @param {Object} event the remote control event. + * @param {RemoteControlEvent} event the remote control event. * @param {Function} onDataChannelFail handler for data channel failure. */ _sendRemoteControlEvent(to, event, onDataChannelFail = () => {}) { diff --git a/service/remotecontrol/Constants.js b/service/remotecontrol/Constants.js index afc8cc15e..c856185ac 100644 --- a/service/remotecontrol/Constants.js +++ b/service/remotecontrol/Constants.js @@ -6,6 +6,8 @@ export const DISCO_REMOTE_CONTROL_FEATURE /** * Types of remote-control-event events. + * @readonly + * @enum {string} */ export const EVENT_TYPES = { mousemove: "mousemove", @@ -22,6 +24,8 @@ export const EVENT_TYPES = { /** * Actions for the remote control permission events. + * @readonly + * @enum {string} */ export const PERMISSIONS_ACTIONS = { request: "request", @@ -34,3 +38,32 @@ export const PERMISSIONS_ACTIONS = { * The type of remote control events sent trough the API module. */ export const REMOTE_CONTROL_EVENT_TYPE = "remote-control-event"; + +/** + * The remote control event. + * @typedef {object} RemoteControlEvent + * @property {EVENT_TYPES} type - the type of the event + * @property {int} x - avaibale for type === mousemove only. The new x + * coordinate of the mouse + * @property {int} y - For mousemove type - the new y + * coordinate of the mouse and for mousescroll - represents the vertical + * scrolling diff value + * @property {int} button - 1(left), 2(middle) or 3 (right). Supported by + * mousedown, mouseup and mousedblclick types. + * @property {KEYS} key - Represents the key related to the event. Supported by + * keydown and keyup types. + * @property {KEYS[]} modifiers - Represents the modifier related to the event. + * Supported by keydown and keyup types. + * @property {PERMISSIONS_ACTIONS} action - Supported by type === permissions. + * Represents the action related to the permissions event. + * + * Optional properties. Supported for permissions event for action === request: + * @property {string} userId - The user id of the participant that has sent the + * request. + * @property {string} userJID - The full JID in the MUC of the user that has + * sent the request. + * @property {string} displayName - the displayName of the participant that has + * sent the request. + * @property {boolean} screenSharing - true if the SS is started for the local + * participant and false if not. + */ From 15090243d03c2365287b6e839ac52977a4b47453 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Fri, 20 Jan 2017 09:53:09 -0600 Subject: [PATCH 11/18] fix(API): JS errors caused by remote control merge --- modules/API/API.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/API/API.js b/modules/API/API.js index 5fb7423b5..8417e7db2 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -116,6 +116,10 @@ class API { return; enabled = true; + + if(!postis) { + this._initPostis(); + } } /** @@ -218,7 +222,7 @@ class API { /** * Sends remote control event. - * @param {RemoteControlEvent} event the remote control event. + * @param {RemoteControlEvent} event the remote control event. */ sendRemoteControlEvent(event) { sendMessage({method: "remote-control-event", params: event}); From 0453346cf44d32ed5321a2742d681fe0c51e078f Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Fri, 20 Jan 2017 14:26:25 -0600 Subject: [PATCH 12/18] ref(remotecontrol): Pass the largeVideoWrapper as parameter to remote control module --- modules/UI/videolayout/RemoteVideo.js | 3 ++- modules/UI/videolayout/VideoLayout.js | 8 ++++++++ modules/remotecontrol/Controller.js | 10 ++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/modules/UI/videolayout/RemoteVideo.js b/modules/UI/videolayout/RemoteVideo.js index b4dd66678..4248a6bad 100644 --- a/modules/UI/videolayout/RemoteVideo.js +++ b/modules/UI/videolayout/RemoteVideo.js @@ -233,7 +233,8 @@ RemoteVideo.prototype.setRemoteControlSupport = function(isSupported = false) { * Requests permissions for remote control session. */ RemoteVideo.prototype._requestRemoteControlPermissions = function () { - APP.remoteControl.controller.requestPermissions(this.id).then(result => { + APP.remoteControl.controller.requestPermissions( + this.id, this.VideoLayout.getLargeVideoWrapper()).then(result => { if(result === null) { return; } diff --git a/modules/UI/videolayout/VideoLayout.js b/modules/UI/videolayout/VideoLayout.js index 97c273f67..cc4911deb 100644 --- a/modules/UI/videolayout/VideoLayout.js +++ b/modules/UI/videolayout/VideoLayout.js @@ -1188,6 +1188,14 @@ var VideoLayout = { _setRemoteControlProperties (user, remoteVideo) { APP.remoteControl.checkUserRemoteControlSupport(user).then(result => remoteVideo.setRemoteControlSupport(result)); + }, + + /** + * Returns the wrapper jquery selector for the largeVideo + * @returns {JQuerySelector} the wrapper jquery selector for the largeVideo + */ + getLargeVideoWrapper() { + return this.getCurrentlyOnLargeContainer().$wrapper; } }; diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index ad4521a96..17dd6b3e1 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -68,15 +68,18 @@ export default class Controller extends RemoteControlParticipant { * Requests permissions from the remote control receiver side. * @param {string} userId the user id of the participant that will be * requested. + * @param {JQuerySelector} eventCaptureArea the area that is going to be + * used mouse and keyboard event capture. * @returns {Promise} - resolve values: * true - accept * false - deny * null - the participant has left. */ - requestPermissions(userId) { + requestPermissions(userId, eventCaptureArea) { if(!this.enabled) { return Promise.reject(new Error("Remote control is disabled!")); } + this.area = eventCaptureArea;// $("#largeVideoWrapper") logger.debug("Requsting remote control permissions from: " + userId); return new Promise((resolve, reject) => { const clearRequest = () => { @@ -134,6 +137,9 @@ export default class Controller extends RemoteControlParticipant { if(this.enabled && event.type === REMOTE_CONTROL_EVENT_TYPE && remoteControlEvent.type === EVENT_TYPES.permissions && userId === this.requestedParticipant) { + if(remoteControlEvent.action !== PERMISSIONS_ACTIONS.grant) { + this.area = null; + } switch(remoteControlEvent.action) { case PERMISSIONS_ACTIONS.grant: { this.controlledParticipant = userId; @@ -202,7 +208,6 @@ export default class Controller extends RemoteControlParticipant { logger.log("Resuming remote control controller."); this.isCollectingEvents = true; APP.keyboardshortcut.enable(false); - this.area = $("#largeVideoWrapper"); this.area.mousemove(event => { const position = this.area.position(); this._sendRemoteControlEvent(this.controlledParticipant, { @@ -249,6 +254,7 @@ export default class Controller extends RemoteControlParticipant { this._userLeftListener); this.controlledParticipant = null; this.pause(); + this.area = null; APP.UI.messageHandler.openMessageDialog( "dialog.remoteControlTitle", "dialog.remoteControlStopMessage" From 1f7c5529e9c9e488cd32a560daad53af61e9db06 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Fri, 20 Jan 2017 14:32:30 -0600 Subject: [PATCH 13/18] fix(remotecontrol): Pin the controlled participant only on remote control permissions granted --- modules/UI/videolayout/RemoteVideo.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/UI/videolayout/RemoteVideo.js b/modules/UI/videolayout/RemoteVideo.js index 4248a6bad..21b780f3b 100644 --- a/modules/UI/videolayout/RemoteVideo.js +++ b/modules/UI/videolayout/RemoteVideo.js @@ -246,9 +246,12 @@ RemoteVideo.prototype._requestRemoteControlPermissions = function () { {user: this.user.getDisplayName() || interfaceConfig.DEFAULT_REMOTE_DISPLAY_NAME} ); - let pinnedId = this.VideoLayout.getPinnedId(); - if(pinnedId !== this.id) { - this.VideoLayout.handleVideoThumbClicked(this.id); + if(result === true) {//the remote control permissions has been granted + // pin the controlled participant + let pinnedId = this.VideoLayout.getPinnedId(); + if(pinnedId !== this.id) { + this.VideoLayout.handleVideoThumbClicked(this.id); + } } }, error => { logger.error(error); From bd98d661d3cc8e2a2c8edb44c7a65b15a736660a Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Mon, 23 Jan 2017 12:07:08 -0600 Subject: [PATCH 14/18] ref(remotecontrol): Moves hangup logic to remote control module --- ConferenceEvents.js | 4 ++++ conference.js | 27 ++++++++++++++++++++++++--- modules/remotecontrol/Receiver.js | 14 ++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 ConferenceEvents.js diff --git a/ConferenceEvents.js b/ConferenceEvents.js new file mode 100644 index 000000000..2dfa4cd87 --- /dev/null +++ b/ConferenceEvents.js @@ -0,0 +1,4 @@ +/** + * Notifies interested parties that hangup procedure will start. + */ +export const BEFORE_HANGUP = "conference.before_hangup"; diff --git a/conference.js b/conference.js index eb3472dd1..ffda92fb2 100644 --- a/conference.js +++ b/conference.js @@ -14,11 +14,11 @@ import {reportError} from './modules/util/helpers'; import UIEvents from './service/UI/UIEvents'; import UIUtil from './modules/UI/util/UIUtil'; +import * as JitsiMeetConferenceEvents from './ConferenceEvents'; import analytics from './modules/analytics/analytics'; -// For remote control testing: -// import remoteControlReceiver from './modules/remotecontrol/Receiver'; +import EventEmitter from "events"; const ConnectionEvents = JitsiMeetJS.events.connection; const ConnectionErrors = JitsiMeetJS.errors.connection; @@ -31,6 +31,8 @@ const TrackErrors = JitsiMeetJS.errors.track; const ConnectionQualityEvents = JitsiMeetJS.events.connectionQuality; +const eventEmitter = new EventEmitter(); + let room, connection, localAudio, localVideo; /** @@ -1783,8 +1785,8 @@ export default { * requested */ hangup (requestFeedback = false) { + eventEmitter.emit(JitsiMeetConferenceEvents.BEFORE_HANGUP); APP.UI.hideRingOverLay(); - APP.remoteControl.receiver.enable(false); let requestFeedbackPromise = requestFeedback ? APP.UI.requestFeedbackOnHangup() // false - because the thank you dialog shouldn't be displayed @@ -1846,5 +1848,24 @@ export default { */ sendEndpointMessage (to, payload) { room.sendEndpointMessage(to, payload); + }, + + /** + * Adds new listener. + * @param {String} eventName the name of the event + * @param {Function} listener the listener. + */ + addListener (eventName, listener) { + eventEmitter.addListener(eventName, listener); + }, + + /** + * Removes listener. + * @param {String} eventName the name of the event that triggers the + * listener + * @param {Function} listener the listener. + */ + removeListener (eventName, listener) { + eventEmitter.removeListener(eventName, listener); } }; diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index 2ffdc5afd..9b856e0d1 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -3,6 +3,7 @@ const logger = require("jitsi-meet-logger").getLogger(__filename); import {DISCO_REMOTE_CONTROL_FEATURE, REMOTE_CONTROL_EVENT_TYPE, EVENT_TYPES, PERMISSIONS_ACTIONS} from "../../service/remotecontrol/Constants"; import RemoteControlParticipant from "./RemoteControlParticipant"; +import * as JitsiMeetConferenceEvents from '../../ConferenceEvents'; const ConferenceEvents = JitsiMeetJS.events.conference; @@ -23,6 +24,7 @@ export default class Receiver extends RemoteControlParticipant { this._remoteControlEventsListener = this._onRemoteControlEvent.bind(this); this._userLeftListener = this._onUserLeft.bind(this); + this._hangupListener = this._onHangup.bind(this); } /** @@ -40,6 +42,8 @@ export default class Receiver extends RemoteControlParticipant { APP.conference.addConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._remoteControlEventsListener); + APP.conference.addListener(JitsiMeetConferenceEvents.BEFORE_HANGUP, + this._hangupListener); } else { logger.log("Remote control receiver disabled."); this._stop(true); @@ -47,6 +51,9 @@ export default class Receiver extends RemoteControlParticipant { APP.conference.removeConferenceListener( ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, this._remoteControlEventsListener); + APP.conference.removeListener( + JitsiMeetConferenceEvents.BEFORE_HANGUP, + this._hangupListener); } } @@ -174,4 +181,11 @@ export default class Receiver extends RemoteControlParticipant { this._stop(); } } + + /** + * Handles hangup events. Disables the receiver. + */ + _onHangup() { + this.enable(false); + } } From b62e4d5ee9733b8a2e8d836bac6bb30ac5c3eeb3 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Mon, 23 Jan 2017 14:58:45 -0600 Subject: [PATCH 15/18] fix(remotecontrol): Log level --- modules/remotecontrol/Controller.js | 4 ++-- modules/remotecontrol/Receiver.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index 17dd6b3e1..8068a1a5c 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -80,7 +80,7 @@ export default class Controller extends RemoteControlParticipant { return Promise.reject(new Error("Remote control is disabled!")); } this.area = eventCaptureArea;// $("#largeVideoWrapper") - logger.debug("Requsting remote control permissions from: " + userId); + logger.log("Requsting remote control permissions from: " + userId); return new Promise((resolve, reject) => { const clearRequest = () => { this.requestedParticipant = null; @@ -143,7 +143,7 @@ export default class Controller extends RemoteControlParticipant { switch(remoteControlEvent.action) { case PERMISSIONS_ACTIONS.grant: { this.controlledParticipant = userId; - logger.debug("Remote control permissions granted to: " + logger.log("Remote control permissions granted to: " + userId); this._start(); return true; diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index 9b856e0d1..88c33eafc 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -126,7 +126,7 @@ export default class Receiver extends RemoteControlParticipant { } APP.API.sendRemoteControlEvent(remoteControlEvent); } else if(event.type === REMOTE_CONTROL_EVENT_TYPE) { - logger.debug("Remote control event is ignored because remote " + logger.log("Remote control event is ignored because remote " + "control is disabled", event); } } @@ -142,7 +142,7 @@ export default class Receiver extends RemoteControlParticipant { APP.conference.addConferenceListener(ConferenceEvents.USER_LEFT, this._userLeftListener); this.controller = userId; - logger.debug("Remote control permissions granted to: " + userId); + logger.log("Remote control permissions granted to: " + userId); if(!APP.conference.isSharingScreen) { APP.conference.toggleScreenSharing(); APP.conference.screenSharingPromise.then(() => { From 4af706bd8322d2da03ec2a556d467c96b12c9eef Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Mon, 23 Jan 2017 14:59:51 -0600 Subject: [PATCH 16/18] style(keycode): Comment --- modules/keycode/keycode.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/keycode/keycode.js b/modules/keycode/keycode.js index 2bbc2debc..38c6e0348 100644 --- a/modules/keycode/keycode.js +++ b/modules/keycode/keycode.js @@ -1,6 +1,6 @@ /** * Enumerates the supported keys. - * NOTE: The maps represents actual keys on the keyboard not chars. + * NOTE: The maps represents physical keys on the keyboard, not chars. * @readonly * @enum {string} */ From 05bfbf5620229be609f7c27daee57872874a7134 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Mon, 23 Jan 2017 15:01:54 -0600 Subject: [PATCH 17/18] fix(remotecontrol): Execute Reciever.enable only when the value is changed --- modules/remotecontrol/Receiver.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/remotecontrol/Receiver.js b/modules/remotecontrol/Receiver.js index 88c33eafc..cf6063617 100644 --- a/modules/remotecontrol/Receiver.js +++ b/modules/remotecontrol/Receiver.js @@ -32,9 +32,10 @@ export default class Receiver extends RemoteControlParticipant { * @param {boolean} enabled the new state. */ enable(enabled) { - if(this.enabled !== enabled) { - this.enabled = enabled; + if(this.enabled === enabled) { + return; } + this.enabled = enabled; if(enabled === true) { logger.log("Remote control receiver enabled."); // Announce remote control support. From 2b1176df5305214cc95a787c6deff8a857bed193 Mon Sep 17 00:00:00 2001 From: hristoterezov Date: Mon, 23 Jan 2017 16:06:51 -0600 Subject: [PATCH 18/18] style(remotecontrol): getRequestedParticipant method comments --- modules/remotecontrol/Controller.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/remotecontrol/Controller.js b/modules/remotecontrol/Controller.js index 8068a1a5c..44f338198 100644 --- a/modules/remotecontrol/Controller.js +++ b/modules/remotecontrol/Controller.js @@ -327,7 +327,8 @@ export default class Controller extends RemoteControlParticipant { /** * Returns the id of the requested participant - * @returns {string} this.requestedParticipant + * @returns {string} this.requestedParticipant. + * NOTE: This id should be the result of JitsiParticipant.getId() call. */ getRequestedParticipant() { return this.requestedParticipant;