DeforaNetworks/jitsi-meet
3
0
Fork 0
Code Releases Activity

feat(RN): displays transcription subtitles

Browse Source
This commit is contained in:
paweldomas
2018-08-23 14:57:12 -05:00
committed by Любомир Маринов
parent eac069c930
commit 26d906fa46
11 changed files with 400 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
react/features/subtitles/middleware.js
View File

@@ -2,14 +2,14 @@
import { MiddlewareRegistry } from '../base/redux';
import {
ENDPOINT_MESSAGE_RECEIVED,
TOGGLE_REQUESTING_SUBTITLES
} from './actionTypes';
import {
removeTranscriptMessage,
updateTranscriptMessage
} from './actions';
import {
ENDPOINT_MESSAGE_RECEIVED,
TOGGLE_REQUESTING_SUBTITLES
} from './actionTypes';
const logger = require('jitsi-meet-logger').getLogger(__filename);
@@ -25,34 +25,35 @@ const JSON_TYPE_TRANSCRIPTION_RESULT = 'transcription-result';
*/
const JSON_TYPE_TRANSLATION_RESULT = 'translation-result';
/**
* The local participant property which is used to store the language
* preference for translation for a participant.
*/
const P_NAME_TRANSLATION_LANGUAGE = 'translation_language';
/**
* The local participant property which is used to set whether the local
* participant wants to have a transcriber in the room.
*/
const P_NAME_REQUESTING_TRANSCRIPTION = 'requestingTranscription';
/**
* The local participant property which is used to store the language
* preference for translation for a participant.
*/
const P_NAME_TRANSLATION_LANGUAGE = 'translation_language';
/**
* Time after which the rendered subtitles will be removed.
*/
const REMOVE_AFTER_MS = 3000;
/**
* Middleware that catches actions related to transcript messages
* to be rendered in {@link TranscriptionSubtitles }
* Middleware that catches actions related to transcript messages to be rendered
* in {@link Captions}.
*
* @param {Store} store - Redux store.
* @param {Store} store - The redux store.
* @returns {Function}
*/
MiddlewareRegistry.register(store => next => action => {
switch (action.type) {
case ENDPOINT_MESSAGE_RECEIVED:
return _endpointMessageReceived(store, next, action);
case TOGGLE_REQUESTING_SUBTITLES:
_requestingSubtitlesToggled(store);
break;
@@ -61,22 +62,6 @@ MiddlewareRegistry.register(store => next => action => {
return next(action);
});
/**
* Toggle the local property 'requestingTranscription'. This will cause Jicofo
* and Jigasi to decide whether the transcriber needs to be in the room.
*
* @param {Store} store - The redux store.
* @private
* @returns {void}
*/
function _requestingSubtitlesToggled({ getState }) {
const { _requestingSubtitles } = getState()['features/subtitles'];
const { conference } = getState()['features/base/conference'];
conference.setLocalParticipantProperty(P_NAME_REQUESTING_TRANSCRIPTION,
!_requestingSubtitles);
}
/**
* Notifies the feature transcription that the action
* {@code ENDPOINT_MESSAGE_RECEIVED} is being dispatched within a specific redux
@@ -92,84 +77,81 @@ function _requestingSubtitlesToggled({ getState }) {
* @returns {Object} The value returned by {@code next(action)}.
*/
function _endpointMessageReceived({ dispatch, getState }, next, action) {
if (!(action.json
&& (action.json.type === JSON_TYPE_TRANSCRIPTION_RESULT
|| action.json.type === JSON_TYPE_TRANSLATION_RESULT))) {
const { json } = action;
if (!(json
&& (json.type === JSON_TYPE_TRANSCRIPTION_RESULT
|| json.type === JSON_TYPE_TRANSLATION_RESULT))) {
return next(action);
}
const json = action.json;
const state = getState();
const translationLanguage
= getState()['features/base/conference'].conference
= state['features/base/conference'].conference
.getLocalParticipantProperty(P_NAME_TRANSLATION_LANGUAGE);
try {
const transcriptMessageID = json.message_id;
const participantName = json.participant.name;
const isInterim = json.is_interim;
const stability = json.stability;
if (json.type === JSON_TYPE_TRANSLATION_RESULT
&& json.language === translationLanguage) {
&& json.language === translationLanguage) {
// Displays final results in the target language if translation is
// enabled.
const newTranscriptMessage = {
participantName,
clearTimeOut: undefined,
final: json.text,
clearTimeOut: undefined
participantName
};
setClearerOnTranscriptMessage(dispatch,
_setClearerOnTranscriptMessage(dispatch,
transcriptMessageID, newTranscriptMessage);
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
} else if (json.type === JSON_TYPE_TRANSCRIPTION_RESULT
&& !translationLanguage) {
&& !translationLanguage) {
// Displays interim and final results without any translation if
// translations are disabled.
const text = json.transcript[0].text;
const { text } = json.transcript[0];
// We update the previous transcript message with the same
// message ID or adds a new transcript message if it does not
// exist in the map.
const newTranscriptMessage
= { ...getState()['features/subtitles']._transcriptMessages
.get(transcriptMessageID) || { participantName } };
const newTranscriptMessage = {
...state['features/subtitles']._transcriptMessages
.get(transcriptMessageID)
|| { participantName }
};
setClearerOnTranscriptMessage(dispatch,
_setClearerOnTranscriptMessage(dispatch,
transcriptMessageID, newTranscriptMessage);
// If this is final result, update the state as a final result
// and start a count down to remove the subtitle from the state
if (!isInterim) {
if (!json.is_interim) {
newTranscriptMessage.final = text;
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
} else if (stability > 0.85) {
} else if (json.stability > 0.85) {
// If the message has a high stability, we can update the
// stable field of the state and remove the previously
// unstable results
newTranscriptMessage.stable = text;
newTranscriptMessage.unstable = undefined;
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
} else {
// Otherwise, this result has an unstable result, which we
// add to the state. The unstable result will be appended
// after the stable part.
newTranscriptMessage.unstable = text;
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
}
dispatch(
updateTranscriptMessage(
transcriptMessageID,
newTranscriptMessage));
}
} catch (error) {
logger.error('Error occurred while updating transcriptions\n', error);
@@ -178,6 +160,24 @@ function _endpointMessageReceived({ dispatch, getState }, next, action) {
return next(action);
}
/**
* Toggle the local property 'requestingTranscription'. This will cause Jicofo
* and Jigasi to decide whether the transcriber needs to be in the room.
*
* @param {Store} store - The redux store.
* @private
* @returns {void}
*/
function _requestingSubtitlesToggled({ getState }) {
const state = getState();
const { _requestingSubtitles } = state['features/subtitles'];
const { conference } = state['features/base/conference'];
conference.setLocalParticipantProperty(
P_NAME_REQUESTING_TRANSCRIPTION,
!_requestingSubtitles);
}
/**
* Set a timeout on a TranscriptMessage object so it clears itself when it's not
* updated.
@@ -185,10 +185,9 @@ function _endpointMessageReceived({ dispatch, getState }, next, action) {
* @param {Function} dispatch - Dispatch remove action to store.
* @param {string} transcriptMessageID - The id of the message to remove.
* @param {Object} transcriptMessage - The message to remove.
*
* @returns {void}
*/
function setClearerOnTranscriptMessage(
function _setClearerOnTranscriptMessage(
dispatch,
transcriptMessageID,
transcriptMessage) {
@@ -196,7 +195,8 @@ function setClearerOnTranscriptMessage(
clearTimeout(transcriptMessage.clearTimeOut);
}
transcriptMessage.clearTimeOut = setTimeout(() => {
dispatch(removeTranscriptMessage(transcriptMessageID));
}, REMOVE_AFTER_MS);
transcriptMessage.clearTimeOut
= setTimeout(
() => dispatch(removeTranscriptMessage(transcriptMessageID)),
REMOVE_AFTER_MS);
}