feat: show no audio signal notification

conference.js

@@ -479,14 +479,18 @@ export default {
             audioOnlyError,
             screenSharingError,
             videoOnlyError;
-        const initialDevices = [];
+        const initialDevices = [ 'audio' ];
-        let requestedAudio = false;
+        const requestedAudio = true;
         let requestedVideo = false;
 
-        if (!options.startWithAudioMuted) {
+        // Always get a handle on the audio input device so that we have statistics even if the user joins the
-            initialDevices.push('audio');
+        // conference muted. Previous implementation would only acquire the handle when the user first unmuted,
-            requestedAudio = true;
+        // which would results in statistics ( such as "No audio input" or "Are you trying to speak?") being available
+        // only after that point.
+        if (options.startWithAudioMuted) {
+            this.muteAudio(true, true);
         }
+
         if (!options.startWithVideoMuted
                 && !options.startAudioOnly
                 && !options.startScreenSharing) {

config.js

@@ -470,6 +470,7 @@ var config = {
      disableNS
      enableLipSync
      enableTalkWhileMuted
+     enableNoAudioDetection
      forceJVB121Ratio
      hiddenDomain
      ignoreStartMuted

lang/main.json

@@ -630,6 +630,10 @@
         "lowerYourHand": "Lower your hand",
         "moreActions": "More actions",
         "mute": "Mute / Unmute",
+
+        "noAudioSignalTitle": "There is no input coming from your mic!",
+        "noAudioSignalDesc": "If you did not purposely mute it from system settings or hardware, consider changing the device.",
+        "noAudioSignalDescSuggestion": "If you did not purposely mute it from system settings or hardware, consider using the following device:",
         "openChat": "Open chat",
         "pip": "Enter Picture-in-Picture mode",
         "privateMessage": "Send private message",

react/features/app/components/App.web.js

@@ -9,6 +9,7 @@ import '../../chat';
 import '../../external-api';
 import '../../power-monitor';
 import '../../room-lock';
+import '../../no-audio-signal';
 import '../../talk-while-muted';
 import '../../video-layout';
 

react/features/base/devices/functions.js

@@ -139,6 +139,39 @@ export function groupDevicesByKind(devices: Object[]): Object {
     };
 }
 
+/**
+ * Filters audio devices from a list of MediaDeviceInfo objects.
+ *
+ * @param {Array<MediaDeviceInfo>} devices - Unfiltered media devices.
+ * @private
+ * @returns {Array<MediaDeviceInfo>} Filtered audio devices.
+ */
+export function filterAudioDevices(devices: Object[]): Object {
+    return devices.filter(device => device.kind === 'audioinput');
+}
+
+/**
+ * We want to strip any device details that are not very user friendly, like usb ids put in brackets at the end.
+ *
+ * @param {string} label - Device label to format.
+ *
+ * @returns {string} - Formatted string.
+ */
+export function formatDeviceLabel(label: string) {
+
+    let formattedLabel = label;
+
+    // Remove braked description at the end as it contains non user friendly strings i.e.
+    // Microsoft® LifeCam HD-3000 (045e:0779:31dg:d1231)
+    const ix = formattedLabel.lastIndexOf('(');
+
+    if (ix !== -1) {
+        formattedLabel = formattedLabel.substr(0, ix);
+    }
+
+    return formattedLabel;
+}
+
 /**
  * Set device id of the audio output device which is currently in use.
  * Empty string stands for default device.

react/features/base/devices/index.js

@@ -1,6 +1,7 @@
 export * from './actions';
 export * from './actionTypes';
 export * from './functions';
+export * from './services';
 
 import './middleware';
 import './reducer';

react/features/base/devices/middleware.js

@@ -20,7 +20,7 @@ import {
 } from './actionTypes';
 import { showNotification, showWarningNotification } from '../../notifications';
 import { updateSettings } from '../settings';
-import { setAudioOutputDeviceId } from './functions';
+import { formatDeviceLabel, setAudioOutputDeviceId } from './functions';
 import logger from './logger';
 
 const JITSI_TRACK_ERROR_TO_MESSAGE_KEY_MAP = {
@@ -186,12 +186,7 @@ function _checkAndNotifyForNewDevice(store, newDevices, oldDevices) {
 
         // we want to strip any device details that are not very
         // user friendly, like usb ids put in brackets at the end
-        let description = newDevice.label;
+        const description = formatDeviceLabel(newDevice.label);
-        const ix = description.lastIndexOf('(');
-
-        if (ix !== -1) {
-            description = description.substr(0, ix);
-        }
 
         let titleKey;
 

react/features/base/devices/services/device-detect/ActiveDeviceDetector.js

@@ -0,0 +1,143 @@
+// @flow
+import EventEmitter from 'events';
+import { ACTIVE_DEVICE_DETECTED } from './Events';
+import logger from '../../logger';
+import JitsiMeetJS from '../../../lib-jitsi-meet';
+
+const JitsiTrackEvents = JitsiMeetJS.events.track;
+
+// If after 3000 ms the detector did not find any active devices consider that there aren't any usable ones available
+// i.e. audioLevel > 0.008
+const DETECTION_TIMEOUT = 3000;
+
+/**
+ * Detect active input devices based on their audio levels, currently this is very simplistic. It works by simply
+ * checking all monitored devices for TRACK_AUDIO_LEVEL_CHANGED if a device has a audio level > 0.008 ( 0.008 is
+ * no input from the perspective of a JitsiLocalTrack ), at which point it triggers a ACTIVE_DEVICE_DETECTED event.
+ * If there are no devices that meet that criteria for DETECTION_TIMEOUT an event with empty deviceLabel parameter
+ * will be triggered,
+ * signaling that no active device was detected.
+ * TODO Potentially improve the active device detection using rnnoise VAD scoring.
+ */
+export class ActiveDeviceDetector extends EventEmitter {
+
+    /**
+     * Currently monitored devices.
+     */
+    _availableDevices: Array<Object>;
+
+    /**
+     * State flag, check if the instance was destroyed.
+     */
+    _destroyed: boolean = false;
+
+    /**
+     * Create active device detector.
+     *
+     * @param {Array<MediaDeviceInfo>} micDeviceList - Device list that is monitored inside the service.
+     *
+     * @returns {ActiveDeviceDetector}
+     */
+    static async create(micDeviceList: Array<MediaDeviceInfo>) {
+        const availableDevices = [];
+
+        try {
+            for (const micDevice of micDeviceList) {
+                const localTrack = await JitsiMeetJS.createLocalTracks({
+                    devices: [ 'audio' ],
+                    micDeviceId: micDevice.deviceId
+                });
+
+                // We provide a specific deviceId thus we expect a single JitsiLocalTrack to be returned.
+                availableDevices.push(localTrack[0]);
+            }
+
+            return new ActiveDeviceDetector(availableDevices);
+        } catch (error) {
+            logger.error('Cleaning up remaining JitsiLocalTrack, due to ActiveDeviceDetector create fail!');
+
+            for (const device of availableDevices) {
+                device.stopStream();
+            }
+
+            throw error;
+        }
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param {Array<Object>} availableDevices - Device list that is monitored inside the service.
+     */
+    constructor(availableDevices: Array<Object>) {
+        super();
+
+        this._availableDevices = availableDevices;
+
+        // Setup event handlers for monitored devices.
+        for (const device of this._availableDevices) {
+            device.on(JitsiTrackEvents.TRACK_AUDIO_LEVEL_CHANGED, audioLevel => {
+                this._handleAudioLevelEvent(device, audioLevel);
+            });
+        }
+
+        // Cancel the detection in case no devices was found with audioLevel > 0 in te set timeout.
+        setTimeout(this._handleDetectionTimeout.bind(this), DETECTION_TIMEOUT);
+    }
+
+    /**
+     * Handle what happens if no device publishes a score in the defined time frame, i.e. Emit an event with empty
+     * deviceLabel.
+     *
+     * @returns {void}
+     */
+    _handleDetectionTimeout() {
+        if (!this._destroyed) {
+            this.emit(ACTIVE_DEVICE_DETECTED, { deviceLabel: '',
+                audioLevel: 0 });
+            this.destroy();
+        }
+    }
+
+    /**
+     * Handles audio level event generated by JitsiLocalTracks.
+     *
+     * @param {Object} device - Label of the emitting track.
+     * @param {number} audioLevel - Audio level generated by device.
+     *
+     * @returns {void}
+     */
+    _handleAudioLevelEvent(device, audioLevel) {
+        if (!this._destroyed) {
+            // This is a very naive approach but works is most, a more accurate approach would ne to use rnnoise
+            // in order to limit the number of false positives.
+            // The 0.008 constant is due to how LocalStatsCollector from lib-jitsi-meet publishes audio-levels, in this
+            // case 0.008 denotes no input.
+            // TODO potentially refactor lib-jitsi-meet to expose this constant as a function. i.e. getSilenceLevel.
+            if (audioLevel > 0.008) {
+                this.emit(ACTIVE_DEVICE_DETECTED, { deviceId: device.deviceId,
+                    deviceLabel: device.track.label,
+                    audioLevel });
+                this.destroy();
+            }
+        }
+    }
+
+    /**
+     * Destroy the ActiveDeviceDetector, clean up the currently monitored devices associated JitsiLocalTracks.
+     *
+     * @returns {void}.
+     */
+    destroy() {
+        if (this._destroyed) {
+            return;
+        }
+
+        for (const device of this._availableDevices) {
+            device.removeAllListeners();
+            device.stopStream();
+        }
+
+        this._destroyed = true;
+    }
+}

react/features/base/devices/services/device-detect/Events.js

@@ -0,0 +1,8 @@
+// Event triggered when the ActiveDeviceDetector finds an audio device that has audio input.
+// Note it does not check if the input is valid or not it simply checks for intensity > 0.008.
+// Event structure:
+// { deviceId: string,
+//   deviceLabel: string,
+//   audioLevel: number }
+// TO DO. Potentially use rnnoise service to get a more accurate reading.
+export const ACTIVE_DEVICE_DETECTED = 'active_device_detected';

react/features/base/devices/services/index.js

@@ -0,0 +1,5 @@
+export * from './device-detect/ActiveDeviceDetector';
+export * from './device-detect/Events';
+export * from './vad-reporter/Events';
+export * from './vad-reporter/TrackVADEmitter';
+export * from './vad-reporter/VADReportingService';

react/features/base/devices/services/vad-reporter/TrackVADEmitter.js

@@ -1,10 +1,10 @@
 // @flow
 
-import { createRnnoiseProcessorPromise, getSampleLength } from '../rnnoise/';
+import { createRnnoiseProcessorPromise, getSampleLength } from '../../../../rnnoise';
 import EventEmitter from 'events';
-import JitsiMeetJS from '../base/lib-jitsi-meet';
+import JitsiMeetJS from '../../../lib-jitsi-meet';
-import logger from './logger';
+import logger from '../../logger';
-import { VAD_SCORE_PUBLISHED } from './VADEvents';
+import { VAD_SCORE_PUBLISHED } from './Events';
 
 /**
  * The structure used by TrackVADEmitter to relay a score

react/features/base/devices/services/vad-reporter/VADReportingService.js

@@ -1,9 +1,9 @@
 // @flow
 
 import EventEmitter from 'events';
-import logger from './logger';
+import logger from '../../logger';
 import TrackVADEmitter from './TrackVADEmitter';
-import { VAD_SCORE_PUBLISHED, VAD_REPORT_PUBLISHED } from './VADEvents';
+import { VAD_SCORE_PUBLISHED, VAD_REPORT_PUBLISHED } from './Events';
 import type { VADScore } from './TrackVADEmitter';
 export type { VADScore };
 

react/features/base/tracks/actionTypes.js

@@ -1,3 +1,14 @@
+/**
+ * The type of Redux action which sets the noSrcDataNotificationUid state representing the UID of the previous
+ * no data from source notification. Used to check if such a notification was previously displayed.
+ *
+ * {
+ *     type: SET_NO_SRC_DATA_NOTI_UID,
+ *     uid: ?number
+ * }
+ */
+export const SET_NO_SRC_DATA_NOTI_UID = 'SET_NO_SRC_DATA_NOTI_UID';
+
 /**
  * The type of redux action dispatched to disable screensharing or to start the
  * flow for enabling screenshare.

react/features/base/tracks/actions.js

@@ -14,6 +14,7 @@ import {
 import { getLocalParticipant } from '../participants';
 
 import {
+    SET_NO_SRC_DATA_NOTI_UID,
     TOGGLE_SCREENSHARING,
     TRACK_ADDED,
     TRACK_CREATE_CANCELED,
@@ -342,6 +343,9 @@ export function trackAdded(track) {
         let isReceivingData, noDataFromSourceNotificationInfo, participantId;
 
         if (local) {
+            // Reset the no data from src notification state when we change the track, as it's context is set
+            // on a per device basis.
+            dispatch(setNoSrcDataNotificationUid());
             const participant = getLocalParticipant(getState);
 
             if (participant) {
@@ -358,6 +362,12 @@ export function trackAdded(track) {
                     });
 
                     dispatch(notificationAction);
+
+                    // Set the notification ID so that other parts of the application know that this was
+                    // displayed in the context of the current device.
+                    // I.E. The no-audio-signal notification shouldn't be displayed if this was already shown.
+                    dispatch(setNoSrcDataNotificationUid(notificationAction.uid));
+
                     noDataFromSourceNotificationInfo = { uid: notificationAction.uid };
                 } else {
                     const timeout = setTimeout(() => dispatch(showNoDataFromSourceVideoError(track)), 5000);
@@ -638,3 +648,20 @@ function _trackCreateCanceled(mediaType) {
         trackType: mediaType
     };
 }
+
+/**
+ * Sets UID of the displayed no data from source notification. Used to track
+ * if the notification was previously displayed in this context.
+ *
+ * @param {number} uid - Notification UID.
+ * @returns {{
+    *     type: SET_NO_AUDIO_SIGNAL_UID,
+    *     uid: number
+    * }}
+    */
+export function setNoSrcDataNotificationUid(uid) {
+    return {
+        type: SET_NO_SRC_DATA_NOTI_UID,
+        uid
+    };
+}

react/features/base/tracks/reducer.js

@@ -1,7 +1,8 @@
 import { PARTICIPANT_ID_CHANGED } from '../participants';
-import { ReducerRegistry } from '../redux';
+import { ReducerRegistry, set } from '../redux';
 
 import {
+    SET_NO_SRC_DATA_NOTI_UID,
     TRACK_ADDED,
     TRACK_CREATE_CANCELED,
     TRACK_CREATE_ERROR,
@@ -133,3 +134,17 @@ ReducerRegistry.register('features/base/tracks', (state = [], action) => {
         return state;
     }
 });
+
+/**
+ * Listen for actions that mutate the no-src-data state, like the current notification id
+ */
+ReducerRegistry.register('features/base/no-src-data', (state = {}, action) => {
+    switch (action.type) {
+    case SET_NO_SRC_DATA_NOTI_UID:
+        return set(state, 'noSrcDataNotiUid', action.uid);
+
+    default:
+        return state;
+    }
+});
+

react/features/no-audio-signal/actionTypes.js

@@ -0,0 +1,11 @@
+/**
+ * The type of Redux action which sets the pending notification UID
+ * to use it when hiding the notification is necessary, or unset it when
+ * undefined (or no param) is passed.
+ *
+ * {
+ *     type: SET_CURRENT_NOTIFICATION_UID,
+ *     uid: ?number
+ * }
+ */
+export const SET_NO_AUDIO_SIGNAL_NOTI_UID = 'SET_NO_AUDIO_SIGNAL_NOTI_UID';

react/features/no-audio-signal/actions.js

@@ -0,0 +1,21 @@
+// @flow
+
+import { SET_NO_AUDIO_SIGNAL_NOTI_UID } from './actionTypes';
+
+/**
+ * Sets UID of the the pending notification to use it when hiding
+ * the notification is necessary, or unset it when undefined (or no param) is
+ * passed.
+ *
+ * @param {?number} uid - The UID of the notification.
+ * @returns {{
+ *     type: SET_NO_AUDIO_SIGNAL_NOTI_UID,
+ *     uid: number
+ * }}
+ */
+export function setNoAudioSignalNotificationUid(uid: ?number) {
+    return {
+        type: SET_NO_AUDIO_SIGNAL_NOTI_UID,
+        uid
+    };
+}

react/features/no-audio-signal/constants.js

@@ -0,0 +1,6 @@
+/**
+ * The identifier of the sound to be played when we got an event for no audio signal.
+ *
+ * @type {string}
+ */
+export const NO_AUDIO_SIGNAL_SOUND_ID = 'NO_AUDIO_SIGNAL_SOUND_ID';

react/features/no-audio-signal/index.js

@@ -0,0 +1,4 @@
+// @flow
+
+import './middleware';
+import './reducer';

react/features/no-audio-signal/middleware.js

@@ -0,0 +1,120 @@
+// @flow
+import { setNoAudioSignalNotificationUid } from './actions';
+import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from '../base/app';
+import { CONFERENCE_JOINED } from '../base/conference';
+import {
+    ACTIVE_DEVICE_DETECTED,
+    ActiveDeviceDetector,
+    filterAudioDevices,
+    formatDeviceLabel,
+    getAvailableDevices,
+    setAudioInputDevice
+} from '../base/devices';
+import { JitsiConferenceEvents } from '../base/lib-jitsi-meet';
+import { MiddlewareRegistry } from '../base/redux';
+import { updateSettings } from '../base/settings';
+import { playSound, registerSound, unregisterSound } from '../base/sounds';
+import { NO_AUDIO_SIGNAL_SOUND_ID } from './constants';
+import { hideNotification, showNotification } from '../notifications';
+import { NO_AUDIO_SIGNAL_SOUND_FILE } from './sounds';
+
+MiddlewareRegistry.register(store => next => async action => {
+    const result = next(action);
+    const { dispatch, getState } = store;
+    const { conference } = action;
+    let audioDetectService = null;
+
+    switch (action.type) {
+    case APP_WILL_MOUNT:
+        dispatch(registerSound(NO_AUDIO_SIGNAL_SOUND_ID, NO_AUDIO_SIGNAL_SOUND_FILE));
+        break;
+    case APP_WILL_UNMOUNT:
+        dispatch(unregisterSound(NO_AUDIO_SIGNAL_SOUND_ID));
+        break;
+
+    case CONFERENCE_JOINED: {
+        conference.on(JitsiConferenceEvents.TRACK_ADDED, track => {
+            const { noAudioSignalNotificationUid } = getState()['features/no-audio-signal'];
+
+            if (track.isAudioTrack() && track.isLocal()) {
+                // In case the device is switched attempt to destroy, this should prevent the notification firing
+                // when the device was switched, however it is possible that a user switches the device and the
+                // notification from the previous devices pops up, but this will probably happen very rarely and even
+                // if it does it's not that disruptive to the ux.
+                if (audioDetectService) {
+                    audioDetectService.destroy();
+                    audioDetectService = null;
+                }
+
+                // When a new track is added hide the current notification is one is displayed, and reset the redux
+                // state so that we begin monitoring on the new device as well.
+                if (noAudioSignalNotificationUid) {
+                    dispatch(hideNotification(noAudioSignalNotificationUid));
+                    dispatch(setNoAudioSignalNotificationUid());
+                }
+            }
+        });
+        conference.on(JitsiConferenceEvents.NO_AUDIO_INPUT, async () => {
+            const { noSrcDataNotiUid } = getState()['features/base/no-src-data'];
+
+            // In case the 'no data detected from source' notification was already shown, we prevent the
+            // no audio signal notification as it's redundant i.e. it's clear that the users microphone is
+            // muted from system settings.
+            if (noSrcDataNotiUid) {
+                return;
+            }
+
+            const devices = await dispatch(getAvailableDevices());
+            const audioDevices = filterAudioDevices(devices);
+
+            audioDetectService = await ActiveDeviceDetector.create(audioDevices);
+
+            audioDetectService.on(ACTIVE_DEVICE_DETECTED, detectEvent => {
+                let descriptionKey = 'toolbar.noAudioSignalDesc';
+                let customActionNameKey = null;
+                let customActionHandler = null;
+
+                // In case the detector picked up a device show a notification with a device suggestion
+                if (detectEvent.deviceLabel !== '') {
+                    descriptionKey = 'toolbar.noAudioSignalDescSuggestion';
+
+                    // Preferably the label should be passed as an argument paired with a i18next string, however
+                    // at the point of the implementation the showNotification function only supports doing that for
+                    // the description.
+                    // TODO Add support for arguments to showNotification title and customAction strings.
+                    customActionNameKey = `Use ${formatDeviceLabel(detectEvent.deviceLabel)}`;
+                    customActionHandler = () => {
+                        // Select device callback
+                        dispatch(
+                                updateSettings({
+                                    userSelectedMicDeviceId: detectEvent.deviceId,
+                                    userSelectedMicDeviceLabel: detectEvent.deviceLabel
+                                })
+                        );
+
+                        dispatch(setAudioInputDevice(detectEvent.deviceId));
+                    };
+                }
+
+                const notification = showNotification({
+                    titleKey: 'toolbar.noAudioSignalTitle',
+                    descriptionKey,
+                    customActionNameKey,
+                    customActionHandler
+                });
+
+                dispatch(notification);
+
+                dispatch(playSound(NO_AUDIO_SIGNAL_SOUND_ID));
+
+                // Store the current notification uid so we can check for this state and hide it in case
+                // a new track was added, thus changing the context of the notification
+                dispatch(setNoAudioSignalNotificationUid(notification.uid));
+            });
+        });
+        break;
+    }
+    }
+
+    return result;
+});

react/features/no-audio-signal/reducer.js

@@ -0,0 +1,17 @@
+// @flow
+
+import { ReducerRegistry, set } from '../base/redux';
+
+import { SET_NO_AUDIO_SIGNAL_NOTI_UID } from './actionTypes';
+
+/**
+ * Reduces the redux actions of the feature no audio signal
+ */
+ReducerRegistry.register('features/no-audio-signal', (state = {}, action) => {
+    switch (action.type) {
+    case SET_NO_AUDIO_SIGNAL_NOTI_UID:
+        return set(state, 'noAudioSignalNotificationUid', action.uid);
+    }
+
+    return state;
+});

react/features/no-audio-signal/sounds.js

@@ -0,0 +1,6 @@
+/**
+ * The file used for the no audio signal sound notification.
+ *
+ * @type {string}
+ */
+export const NO_AUDIO_SIGNAL_SOUND_FILE = 'noAudioSignal.mp3';

react/features/vad-reporter/logger.js

@@ -1,5 +0,0 @@
-// @flow
-
-import { getLogger } from '../base/logging/functions';
-
-export default getLogger('features/vad-reporter');