diff --git a/_locales/en/messages.json b/_locales/en/messages.json index cc39f8a74..e60672bae 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -2725,5 +2725,25 @@ "example": "00:01" } } + }, + "callingDeviceSelection__settings": { + "message": "Settings", + "description": "Title for device selection settings" + }, + "callingDeviceSelection__label--video": { + "message": "Video", + "description": "Label for video input selector" + }, + "callingDeviceSelection__label--audio-input": { + "message": "Microphone", + "description": "Label for audio input selector" + }, + "callingDeviceSelection__label--audio-output": { + "message": "Speakers", + "description": "Label for audio output selector" + }, + "callingDeviceSelection__select--no-device": { + "message": "No devices available", + "description": "Message for when there are no available devices to select for input/output audio or video" } } diff --git a/package.json b/package.json index 6ce1db94f..480fa2725 100644 --- a/package.json +++ b/package.json @@ -129,7 +129,7 @@ "redux-ts-utils": "3.2.2", "reselect": "4.0.0", "rimraf": "2.6.2", - "ringrtc": "https://github.com/signalapp/signal-ringrtc-node.git#de96fa13f04e846bdcb0ae9be8e2dc80a932f2be", + "ringrtc": "https://github.com/signalapp/signal-ringrtc-node.git#0cd60f529d8f17734fe19e7195c9fd3c3f4271db", "sanitize-filename": "1.6.3", "sanitize.css": "11.0.0", "semver": "5.4.1", diff --git a/stylesheets/_modules.scss b/stylesheets/_modules.scss index ed2ae8654..036351606 100644 --- a/stylesheets/_modules.scss +++ b/stylesheets/_modules.scss @@ -5984,6 +5984,21 @@ button.module-image__border-overlay:focus { } } +.module-ongoing-call__settings { + position: absolute; + top: 25px; + right: 25px; + + &--button { + @include color-svg( + '../images/icons/v2/settings-solid-16.svg', + $color-white + ); + height: 22px; + width: 22px; + } +} + // Module: Left Pane .module-left-pane { @@ -8847,6 +8862,87 @@ button.module-image__border-overlay:focus { margin-right: auto; } +/* Calling: Device Selection */ + +.module-calling-device-selection { + position: relative; +} + +.module-calling-device-selection__close-button { + @include button-reset; + + @include light-theme { + @include color-svg('../images/x-shadow-16.svg', $color-gray-75); + } + + @include dark-theme { + @include color-svg('../images/x-shadow-16.svg', $color-white); + } + + height: 16px; + position: absolute; + right: 5px; + top: 0; + width: 16px; + z-index: 2; + + @include keyboard-mode { + &:focus { + outline: 2px solid $ultramarine-ui-light; + } + } +} + +.module-calling-device-selection__title { + @include font-title-2; + margin-top: 12px; + margin-bottom: 20px; +} + +.module-calling-device-selection__label { + @include font-body-1-bold; + display: block; + margin-bottom: 16px; +} + +.module-calling-device-selection__select { + margin-bottom: 20px; + position: relative; + + select { + @include font-body-1; + -webkit-appearance: none; + border-radius: 4px; + border: 1px solid $color-gray-45; + cursor: pointer; + height: 40px; + outline: 0; + padding: 10px; + padding-right: 32px; + text-overflow: ellipsis; + width: 100%; + } + + &::after { + border: 2px solid $color-gray-75; + + border-radius: 2px; + border-right: 0; + border-top: 0; + content: ' '; + display: block; + height: 10px; + pointer-events: none; + position: absolute; + right: 15px; + top: 16px; + transform-origin: center; + transform: rotate(-45deg); + width: 10px; + z-index: 2; + } +} + /* Third-party module: react-tooltip-lite */ .react-tooltip-lite { diff --git a/ts/components/CallManager.stories.tsx b/ts/components/CallManager.stories.tsx index 99c59875a..934955093 100644 --- a/ts/components/CallManager.stories.tsx +++ b/ts/components/CallManager.stories.tsx @@ -31,17 +31,18 @@ const defaultProps = { callDetails, callState: CallState.Accepted, declineCall: action('decline-call'), - getVideoCapturer: () => ({}), - getVideoRenderer: () => ({}), hangUp: action('hang-up'), hasLocalAudio: true, hasLocalVideo: true, hasRemoteVideo: true, i18n, + renderDeviceSelection: () =>
, setLocalAudio: action('set-local-audio'), + setLocalPreview: action('set-local-preview'), setLocalVideo: action('set-local-video'), - setVideoCapturer: action('set-video-capturer'), - setVideoRenderer: action('set-video-renderer'), + setRendererCanvas: action('set-renderer-canvas'), + settingsDialogOpen: false, + toggleSettings: action('toggle-settings'), }; const permutations = [ diff --git a/ts/components/CallManager.tsx b/ts/components/CallManager.tsx index e6ec9021f..494c9a9b8 100644 --- a/ts/components/CallManager.tsx +++ b/ts/components/CallManager.tsx @@ -10,7 +10,10 @@ import { CallDetailsType } from '../state/ducks/calling'; type CallManagerPropsType = { callDetails?: CallDetailsType; callState?: CallState; + renderDeviceSelection: () => JSX.Element; + settingsDialogOpen: boolean; }; + type PropsType = IncomingCallBarPropsType & CallScreenPropsType & CallManagerPropsType; @@ -20,17 +23,18 @@ export const CallManager = ({ callDetails, callState, declineCall, - getVideoCapturer, - getVideoRenderer, hangUp, hasLocalAudio, hasLocalVideo, hasRemoteVideo, i18n, + renderDeviceSelection, setLocalAudio, + setLocalPreview, setLocalVideo, - setVideoCapturer, - setVideoRenderer, + setRendererCanvas, + settingsDialogOpen, + toggleSettings, }: PropsType): JSX.Element | null => { if (!callDetails || !callState) { return null; @@ -43,21 +47,23 @@ export const CallManager = ({ if (outgoing || ongoing) { return ( - + <> + + {settingsDialogOpen && renderDeviceSelection()} + ); } diff --git a/ts/components/CallScreen.stories.tsx b/ts/components/CallScreen.stories.tsx index 9f847f8e1..7d0416dfa 100644 --- a/ts/components/CallScreen.stories.tsx +++ b/ts/components/CallScreen.stories.tsx @@ -30,17 +30,16 @@ const callDetails = { const defaultProps = { callDetails, callState: CallState.Accepted, - getVideoCapturer: () => ({}), - getVideoRenderer: () => ({}), hangUp: action('hang-up'), hasLocalAudio: true, hasLocalVideo: true, hasRemoteVideo: true, i18n, setLocalAudio: action('set-local-audio'), + setLocalPreview: action('set-local-preview'), setLocalVideo: action('set-local-video'), - setVideoCapturer: action('set-video-capturer'), - setVideoRenderer: action('set-video-renderer'), + setRendererCanvas: action('set-renderer-canvas'), + toggleSettings: action('toggle-settings'), }; const permutations = [ diff --git a/ts/components/CallScreen.tsx b/ts/components/CallScreen.tsx index d585232cb..59d7f5b36 100644 --- a/ts/components/CallScreen.tsx +++ b/ts/components/CallScreen.tsx @@ -4,14 +4,13 @@ import { CallDetailsType, HangUpType, SetLocalAudioType, + SetLocalPreviewType, SetLocalVideoType, - SetVideoCapturerType, - SetVideoRendererType, + SetRendererCanvasType, } from '../state/ducks/calling'; import { Avatar } from './Avatar'; import { CallState } from '../types/Calling'; import { LocalizerType } from '../types/Util'; -import { CanvasVideoRenderer, GumVideoCapturer } from '../window.d'; type CallingButtonProps = { classNameSuffix: string; @@ -37,12 +36,6 @@ const CallingButton = ({ export type PropsType = { callDetails?: CallDetailsType; callState?: CallState; - getVideoCapturer: ( - ref: React.RefObject - ) => GumVideoCapturer; - getVideoRenderer: ( - ref: React.RefObject - ) => CanvasVideoRenderer; hangUp: (_: HangUpType) => void; hasLocalAudio: boolean; hasLocalVideo: boolean; @@ -50,8 +43,9 @@ export type PropsType = { i18n: LocalizerType; setLocalAudio: (_: SetLocalAudioType) => void; setLocalVideo: (_: SetLocalVideoType) => void; - setVideoCapturer: (_: SetVideoCapturerType) => void; - setVideoRenderer: (_: SetVideoRendererType) => void; + setLocalPreview: (_: SetLocalPreviewType) => void; + setRendererCanvas: (_: SetRendererCanvasType) => void; + toggleSettings: () => void; }; type StateType = { @@ -79,11 +73,6 @@ export class CallScreen extends React.Component { this.controlsFadeTimer = null; this.localVideoRef = React.createRef(); this.remoteVideoRef = React.createRef(); - - this.setVideoCapturerAndRenderer( - props.getVideoCapturer(this.localVideoRef), - props.getVideoRenderer(this.remoteVideoRef) - ); } public componentDidMount() { @@ -92,6 +81,9 @@ export class CallScreen extends React.Component { this.fadeControls(); document.addEventListener('keydown', this.handleKeyDown); + + this.props.setLocalPreview({ element: this.localVideoRef }); + this.props.setRendererCanvas({ element: this.remoteVideoRef }); } public componentWillUnmount() { @@ -103,7 +95,8 @@ export class CallScreen extends React.Component { if (this.controlsFadeTimer) { clearTimeout(this.controlsFadeTimer); } - this.setVideoCapturerAndRenderer(null, null); + this.props.setLocalPreview({ element: undefined }); + this.props.setRendererCanvas({ element: undefined }); } updateAcceptedTimer = () => { @@ -203,6 +196,8 @@ export class CallScreen extends React.Component { hasLocalAudio, hasLocalVideo, hasRemoteVideo, + i18n, + toggleSettings, } = this.props; const { showControls } = this.state; const isAudioOnly = !hasLocalVideo && !hasRemoteVideo; @@ -241,6 +236,13 @@ export class CallScreen extends React.Component { {callDetails.title}
{this.renderMessage(callState)} +
+
{hasRemoteVideo ? this.renderRemoteVideo() @@ -356,27 +358,4 @@ export class CallScreen extends React.Component { } return `${mins}:${secs}`; } - - private setVideoCapturerAndRenderer( - capturer: GumVideoCapturer | null, - renderer: CanvasVideoRenderer | null - ) { - const { callDetails, setVideoCapturer, setVideoRenderer } = this.props; - - if (!callDetails) { - return; - } - - const { callId } = callDetails; - - setVideoCapturer({ - callId, - capturer, - }); - - setVideoRenderer({ - callId, - renderer, - }); - } } diff --git a/ts/components/CallingDeviceSelection.stories.tsx b/ts/components/CallingDeviceSelection.stories.tsx new file mode 100644 index 000000000..f8129250f --- /dev/null +++ b/ts/components/CallingDeviceSelection.stories.tsx @@ -0,0 +1,145 @@ +import * as React from 'react'; +import { CallingDeviceSelection, Props } from './CallingDeviceSelection'; + +// @ts-ignore +import { setup as setupI18n } from '../../js/modules/i18n'; +// @ts-ignore +import enMessages from '../../_locales/en/messages.json'; + +import { action } from '@storybook/addon-actions'; +import { storiesOf } from '@storybook/react'; + +const i18n = setupI18n('en', enMessages); + +const audioDevice = { + name: '', + index: 0, + same_name_index: 0, +}; + +const createProps = ({ + availableMicrophones = [], + availableSpeakers = [], + selectedMicrophone = audioDevice, + selectedSpeaker = audioDevice, + availableCameras = [], + selectedCamera = '', +}: Partial = {}): Props => ({ + availableCameras, + availableMicrophones, + availableSpeakers, + changeIODevice: action('change-io-device'), + i18n, + selectedCamera, + selectedMicrophone, + selectedSpeaker, + toggleSettings: action('toggle-settings'), +}); + +const stories = storiesOf('Components/CallingDeviceSelection', module); + +stories.add('Default', () => { + return ; +}); + +stories.add('Some Devices', () => { + const availableSpeakers = [ + { + name: 'Default - Internal Microphone', + index: 0, + same_name_index: 0, + }, + { + name: "Natalie's Airpods (Bluetooth)", + index: 1, + same_name_index: 1, + }, + { + name: 'UE Boom (Bluetooth)', + index: 2, + same_name_index: 2, + }, + ]; + const selectedSpeaker = availableSpeakers[0]; + + const props = createProps({ + availableSpeakers, + selectedSpeaker, + }); + + return ; +}); + +stories.add('All Devices', () => { + const availableSpeakers = [ + { + name: 'Default - Internal Speakers', + index: 0, + same_name_index: 0, + }, + { + name: "Natalie's Airpods (Bluetooth)", + index: 1, + same_name_index: 1, + }, + { + name: 'UE Boom (Bluetooth)', + index: 2, + same_name_index: 2, + }, + ]; + const selectedSpeaker = availableSpeakers[0]; + + const availableMicrophones = [ + { + name: 'Default - Internal Microphone', + index: 0, + same_name_index: 0, + }, + { + name: "Natalie's Airpods (Bluetooth)", + index: 1, + same_name_index: 1, + }, + ]; + const selectedMicrophone = availableMicrophones[0]; + + const availableCameras = [ + { + deviceId: + 'dfbe6effe70b0611ba0fdc2a9ea3f39f6cb110e6687948f7e5f016c111b7329c', + groupId: + '63ee218d2446869e40adfc958ff98263e51f74382b0143328ee4826f20a76f47', + kind: 'videoinput' as MediaDeviceKind, + label: 'FaceTime HD Camera (Built-in) (9fba:bced)', + toJSON() { + return ''; + }, + }, + { + deviceId: + 'e2db196a31d50ff9b135299dc0beea67f65b1a25a06d8a4ce76976751bb7a08d', + groupId: + '218ba7f00d7b1239cca15b9116769e5e7d30cc01104ebf84d667643661e0ecf9', + kind: 'videoinput' as MediaDeviceKind, + label: 'Logitech Webcam (4e72:9058)', + toJSON() { + return ''; + }, + }, + ]; + + const selectedCamera = + 'dfbe6effe70b0611ba0fdc2a9ea3f39f6cb110e6687948f7e5f016c111b7329c'; + + const props = createProps({ + availableCameras, + availableMicrophones, + availableSpeakers, + selectedCamera, + selectedMicrophone, + selectedSpeaker, + }); + + return ; +}); diff --git a/ts/components/CallingDeviceSelection.tsx b/ts/components/CallingDeviceSelection.tsx new file mode 100644 index 000000000..4f9effe02 --- /dev/null +++ b/ts/components/CallingDeviceSelection.tsx @@ -0,0 +1,192 @@ +import * as React from 'react'; + +import { ConfirmationModal } from './ConfirmationModal'; +import { LocalizerType } from '../types/Util'; +import { + AudioDevice, + CallingDeviceType, + ChangeIODevicePayloadType, + MediaDeviceSettings, +} from '../types/Calling'; + +export type Props = MediaDeviceSettings & { + changeIODevice: (payload: ChangeIODevicePayloadType) => void; + i18n: LocalizerType; + toggleSettings: () => void; +}; + +function renderAudioOptions( + devices: Array, + i18n: LocalizerType, + selectedDevice: AudioDevice | undefined +): JSX.Element { + if (!devices.length) { + return ( + + ); + } + + return ( + <> + {devices.map((device: AudioDevice) => { + const isSelected = + selectedDevice && selectedDevice.index === device.index; + return ( + + ); + })} + + ); +} + +function renderVideoOptions( + devices: Array, + i18n: LocalizerType, + selectedCamera: string | undefined +): JSX.Element { + if (!devices.length) { + return ( + + ); + } + + return ( + <> + {devices.map((device: MediaDeviceInfo) => { + const isSelected = selectedCamera === device.deviceId; + + return ( + + ); + })} + + ); +} + +function createAudioChangeHandler( + devices: Array, + changeIODevice: (payload: ChangeIODevicePayloadType) => void, + type: CallingDeviceType.SPEAKER | CallingDeviceType.MICROPHONE +) { + return (ev: React.FormEvent): void => { + changeIODevice({ + type, + selectedDevice: devices[Number(ev.currentTarget.value)], + }); + }; +} + +function createCameraChangeHandler( + changeIODevice: (payload: ChangeIODevicePayloadType) => void +) { + return (ev: React.FormEvent): void => { + changeIODevice({ + type: CallingDeviceType.CAMERA, + selectedDevice: String(ev.currentTarget.value), + }); + }; +} + +export const CallingDeviceSelection = ({ + availableCameras, + availableMicrophones, + availableSpeakers, + changeIODevice, + i18n, + selectedCamera, + selectedMicrophone, + selectedSpeaker, + toggleSettings, +}: Props): JSX.Element => { + const selectedMicrophoneIndex = selectedMicrophone + ? selectedMicrophone.index + : undefined; + const selectedSpeakerIndex = selectedSpeaker + ? selectedSpeaker.index + : undefined; + + return ( + +
+
+ +

+ {i18n('callingDeviceSelection__settings')} +

+ + +
+ +
+ + +
+ +
+ + +
+ +
+
+ ); +}; diff --git a/ts/services/calling.ts b/ts/services/calling.ts index 21d480566..1270e6b85 100644 --- a/ts/services/calling.ts +++ b/ts/services/calling.ts @@ -5,12 +5,13 @@ import { CallLogLevel, CallSettings, CallState, + CanvasVideoRenderer, DeviceId, + GumVideoCapturer, RingRTC, UserId, - VideoCapturer, - VideoRenderer, } from 'ringrtc'; + import { ActionsType as UxActionsType, CallDetailsType, @@ -18,6 +19,7 @@ import { import { CallingMessageClass, EnvelopeClass } from '../textsecure.d'; import { ConversationModelType } from '../model-types.d'; import is from '@sindresorhus/is'; +import { AudioDevice, MediaDeviceSettings } from '../types/Calling'; export { CallState, @@ -36,7 +38,16 @@ export type CallHistoryDetailsType = { }; export class CallingClass { + readonly videoCapturer: GumVideoCapturer; + readonly videoRenderer: CanvasVideoRenderer; private uxActions?: UxActionsType; + private lastMediaDeviceSettings?: MediaDeviceSettings; + private deviceReselectionTimer?: NodeJS.Timeout; + + constructor() { + this.videoCapturer = new GumVideoCapturer(640, 480, 30); + this.videoRenderer = new CanvasVideoRenderer(); + } initialize(uxActions: UxActionsType): void { this.uxActions = uxActions; @@ -80,11 +91,6 @@ export class CallingClass { return; } - if (RingRTC.call && RingRTC.call.state !== CallState.Ended) { - window.log.info('Call already in progress, new call not allowed.'); - return; - } - const remoteUserId = this.getRemoteUserIdFromConversation(conversation); if (!remoteUserId || !this.localDeviceId) { window.log.error('Missing identifier, new call not allowed.'); @@ -97,15 +103,26 @@ export class CallingClass { return; } + const callSettings = await this.getCallSettings(conversation); + + // Check state after awaiting to debounce call button. + if (RingRTC.call && RingRTC.call.state !== CallState.Ended) { + window.log.info('Call already in progress, new call not allowed.'); + return; + } + // We could make this faster by getting the call object // from the RingRTC before we lookup the ICE servers. const call = RingRTC.startOutgoingCall( remoteUserId, isVideoCall, this.localDeviceId, - await this.getCallSettings(conversation) + callSettings ); + await this.startDeviceReselectionTimer(); + RingRTC.setVideoCapturer(call.callId, this.videoCapturer); + RingRTC.setVideoRenderer(call.callId, this.videoRenderer); this.attachToCall(conversation, call); this.uxActions.outgoingCall({ @@ -116,6 +133,9 @@ export class CallingClass { async accept(callId: CallId, asVideoCall: boolean) { const haveMediaPermissions = await this.requestPermissions(asVideoCall); if (haveMediaPermissions) { + await this.startDeviceReselectionTimer(); + RingRTC.setVideoCapturer(callId, this.videoCapturer); + RingRTC.setVideoRenderer(callId, this.videoRenderer); RingRTC.accept(callId, asVideoCall); } else { window.log.info('Permissions were denied, call not allowed, hanging up.'); @@ -139,12 +159,230 @@ export class CallingClass { RingRTC.setOutgoingVideo(callId, enabled); } - setVideoCapturer(callId: CallId, capturer: VideoCapturer | null) { - RingRTC.setVideoCapturer(callId, capturer); + private async startDeviceReselectionTimer(): Promise { + // Poll once + await this.pollForMediaDevices(); + // Start the timer + if (!this.deviceReselectionTimer) { + this.deviceReselectionTimer = setInterval(async () => { + await this.pollForMediaDevices(); + }, 3000); + } } - setVideoRenderer(callId: CallId, renderer: VideoRenderer | null) { - RingRTC.setVideoRenderer(callId, renderer); + private stopDeviceReselectionTimer() { + if (this.deviceReselectionTimer) { + clearInterval(this.deviceReselectionTimer); + this.deviceReselectionTimer = undefined; + } + } + + // tslint:disable-next-line cyclomatic-complexity + private mediaDeviceSettingsEqual( + a?: MediaDeviceSettings, + b?: MediaDeviceSettings + ): boolean { + if (!a && !b) { + return true; + } + if (!a || !b) { + return false; + } + if ( + a.availableCameras.length !== b.availableCameras.length || + a.availableMicrophones.length !== b.availableMicrophones.length || + a.availableSpeakers.length !== b.availableSpeakers.length + ) { + return false; + } + for (let i = 0; i < a.availableCameras.length; i++) { + if ( + a.availableCameras[i].deviceId !== b.availableCameras[i].deviceId || + a.availableCameras[i].groupId !== b.availableCameras[i].groupId || + a.availableCameras[i].label !== b.availableCameras[i].label + ) { + return false; + } + } + for (let i = 0; i < a.availableMicrophones.length; i++) { + if ( + a.availableMicrophones[i].name !== b.availableMicrophones[i].name || + a.availableMicrophones[i].unique_id !== + b.availableMicrophones[i].unique_id || + a.availableMicrophones[i].same_name_index !== + b.availableMicrophones[i].same_name_index + ) { + return false; + } + } + for (let i = 0; i < a.availableSpeakers.length; i++) { + if ( + a.availableSpeakers[i].name !== b.availableSpeakers[i].name || + a.availableSpeakers[i].unique_id !== b.availableSpeakers[i].unique_id || + a.availableSpeakers[i].same_name_index !== + b.availableSpeakers[i].same_name_index + ) { + return false; + } + } + if ( + (a.selectedCamera && !b.selectedCamera) || + (!a.selectedCamera && b.selectedCamera) || + (a.selectedMicrophone && !b.selectedMicrophone) || + (!a.selectedMicrophone && b.selectedMicrophone) || + (a.selectedSpeaker && !b.selectedSpeaker) || + (!a.selectedSpeaker && b.selectedSpeaker) + ) { + return false; + } + if ( + a.selectedCamera && + b.selectedCamera && + a.selectedCamera !== b.selectedCamera + ) { + return false; + } + if ( + a.selectedMicrophone && + b.selectedMicrophone && + a.selectedMicrophone.index !== b.selectedMicrophone.index + ) { + return false; + } + if ( + a.selectedSpeaker && + b.selectedSpeaker && + a.selectedSpeaker.index !== b.selectedSpeaker.index + ) { + return false; + } + return true; + } + + private async pollForMediaDevices(): Promise { + const newSettings = await this.getMediaDeviceSettings(); + if ( + !this.mediaDeviceSettingsEqual(this.lastMediaDeviceSettings, newSettings) + ) { + window.log.info( + 'MediaDevice: available devices changed (from->to)', + this.lastMediaDeviceSettings, + newSettings + ); + await this.selectPreferredMediaDevices(newSettings); + this.lastMediaDeviceSettings = newSettings; + this.uxActions?.refreshIODevices(newSettings); + } + } + + async getMediaDeviceSettings(): Promise { + const availableMicrophones = RingRTC.getAudioInputs(); + const preferredMicrophone = window.storage.get( + 'preferred-audio-input-device' + ); + const selectedMicIndex = this.findBestMatchingDeviceIndex( + availableMicrophones, + preferredMicrophone + ); + const selectedMicrophone = + selectedMicIndex !== undefined + ? availableMicrophones[selectedMicIndex] + : undefined; + + const availableSpeakers = RingRTC.getAudioOutputs(); + const preferredSpeaker = window.storage.get( + 'preferred-audio-output-device' + ); + const selectedSpeakerIndex = this.findBestMatchingDeviceIndex( + availableSpeakers, + preferredSpeaker + ); + const selectedSpeaker = + selectedSpeakerIndex !== undefined + ? availableSpeakers[selectedSpeakerIndex] + : undefined; + + const availableCameras = await window.Signal.Services.calling.videoCapturer.enumerateDevices(); + const preferredCamera = window.storage.get('preferred-video-input-device'); + const selectedCamera = this.findBestMatchingCamera( + availableCameras, + preferredCamera + ); + + return { + availableMicrophones, + availableSpeakers, + selectedMicrophone, + selectedSpeaker, + availableCameras, + selectedCamera, + }; + } + + findBestMatchingDeviceIndex( + available: Array, + preferred: AudioDevice | undefined + ): number | undefined { + if (!preferred) { + // No preference stored + return undefined; + } + // Match by UUID first, if available + if (preferred.unique_id) { + const matchIndex = available.findIndex( + d => d.unique_id === preferred.unique_id + ); + if (matchIndex !== -1) { + return matchIndex; + } + } + + // Match by name second, and if there are multiple such names - by instance index. + const matchingNames = available.filter(d => d.name === preferred.name); + if (matchingNames.length > preferred.same_name_index) { + return matchingNames[preferred.same_name_index].index; + } + if (matchingNames.length > 0) { + return matchingNames[0].index; + } + + // Nothing matches; take the first device if there are any + return available.length > 0 ? 0 : undefined; + } + + findBestMatchingCamera( + available: Array, + preferred?: string + ): string | undefined { + const matchingId = available.filter(d => d.deviceId === preferred); + const nonInfrared = available.filter(d => !d.label.includes('IR Camera')); + + /// By default, pick the first non-IR camera (but allow the user to pick the infrared if they so desire) + if (matchingId.length > 0) { + return matchingId[0].deviceId; + } else if (nonInfrared.length > 0) { + return nonInfrared[0].deviceId; + } else { + return undefined; + } + } + + setPreferredMicrophone(device: AudioDevice) { + window.log.info('MediaDevice: setPreferredMicrophone', device); + window.storage.put('preferred-audio-input-device', device); + RingRTC.setAudioInput(device.index); + } + + setPreferredSpeaker(device: AudioDevice) { + window.log.info('MediaDevice: setPreferredSpeaker', device); + window.storage.put('preferred-audio-output-device', device); + RingRTC.setAudioOutput(device.index); + } + + async setPreferredCamera(device: string) { + window.log.info('MediaDevice: setPreferredCamera', device); + window.storage.put('preferred-video-input-device', device); + await this.videoCapturer.setPreferredDevice(device); } async handleCallingMessage( @@ -176,6 +414,30 @@ export class CallingClass { ); } + private async selectPreferredMediaDevices( + settings: MediaDeviceSettings + ): Promise { + // Assume that the MediaDeviceSettings have been obtained very recently and the index is still valid (no devices have been plugged in in between). + if (settings.selectedMicrophone) { + window.log.info( + 'MediaDevice: selecting microphone', + settings.selectedMicrophone + ); + RingRTC.setAudioInput(settings.selectedMicrophone.index); + } + if (settings.selectedSpeaker) { + window.log.info( + 'MediaDevice: selecting speaker', + settings.selectedMicrophone + ); + RingRTC.setAudioOutput(settings.selectedSpeaker.index); + } + if (settings.selectedCamera) { + window.log.info('MediaDevice: selecting camera', settings.selectedCamera); + await this.videoCapturer.setPreferredDevice(settings.selectedCamera); + } + } + private async requestCameraPermissions(): Promise { const cameraPermission = await window.getMediaCameraPermissions(); if (!cameraPermission) { @@ -325,6 +587,8 @@ export class CallingClass { acceptedTime = Date.now(); } else if (call.state === CallState.Ended) { this.addCallHistoryForEndedCall(conversation, call, acceptedTime); + this.stopDeviceReselectionTimer(); + this.lastMediaDeviceSettings = undefined; } uxActions.callStateChange({ callState: call.state, diff --git a/ts/state/ducks/calling.ts b/ts/state/ducks/calling.ts index 5b5a6fd38..d0eb95de6 100644 --- a/ts/state/ducks/calling.ts +++ b/ts/state/ducks/calling.ts @@ -1,7 +1,11 @@ import { notify } from '../../services/notify'; -import { calling, VideoCapturer, VideoRenderer } from '../../services/calling'; -import { CallState } from '../../types/Calling'; -import { CanvasVideoRenderer, GumVideoCapturer } from '../../window.d'; +import { calling } from '../../services/calling'; +import { + CallingDeviceType, + CallState, + ChangeIODevicePayloadType, + MediaDeviceSettings, +} from '../../types/Calling'; import { ColorType } from '../../types/Colors'; import { NoopActionType } from './noop'; import { callingTones } from '../../util/callingTones'; @@ -28,12 +32,13 @@ export type CallDetailsType = { title: string; }; -export type CallingStateType = { +export type CallingStateType = MediaDeviceSettings & { callDetails?: CallDetailsType; callState?: CallState; hasLocalAudio: boolean; hasLocalVideo: boolean; hasRemoteVideo: boolean; + settingsDialogOpen: boolean; }; export type AcceptCallType = { @@ -76,14 +81,12 @@ export type SetLocalVideoType = { enabled: boolean; }; -export type SetVideoCapturerType = { - callId: CallId; - capturer: CanvasVideoRenderer | null; +export type SetLocalPreviewType = { + element: React.RefObject | undefined; }; -export type SetVideoRendererType = { - callId: CallId; - renderer: GumVideoCapturer | null; +export type SetRendererCanvasType = { + element: React.RefObject | undefined; }; // Actions @@ -91,14 +94,18 @@ export type SetVideoRendererType = { const ACCEPT_CALL = 'calling/ACCEPT_CALL'; const CALL_STATE_CHANGE = 'calling/CALL_STATE_CHANGE'; const CALL_STATE_CHANGE_FULFILLED = 'calling/CALL_STATE_CHANGE_FULFILLED'; +const CHANGE_IO_DEVICE = 'calling/CHANGE_IO_DEVICE'; +const CHANGE_IO_DEVICE_FULFILLED = 'calling/CHANGE_IO_DEVICE_FULFILLED'; const DECLINE_CALL = 'calling/DECLINE_CALL'; const HANG_UP = 'calling/HANG_UP'; const INCOMING_CALL = 'calling/INCOMING_CALL'; const OUTGOING_CALL = 'calling/OUTGOING_CALL'; +const REFRESH_IO_DEVICES = 'calling/REFRESH_IO_DEVICES'; const REMOTE_VIDEO_CHANGE = 'calling/REMOTE_VIDEO_CHANGE'; const SET_LOCAL_AUDIO = 'calling/SET_LOCAL_AUDIO'; const SET_LOCAL_VIDEO = 'calling/SET_LOCAL_VIDEO'; const SET_LOCAL_VIDEO_FULFILLED = 'calling/SET_LOCAL_VIDEO_FULFILLED'; +const TOGGLE_SETTINGS = 'calling/TOGGLE_SETTINGS'; type AcceptCallActionType = { type: 'calling/ACCEPT_CALL'; @@ -115,6 +122,16 @@ type CallStateChangeFulfilledActionType = { payload: CallStateChangeType; }; +type ChangeIODeviceActionType = { + type: 'calling/CHANGE_IO_DEVICE'; + payload: Promise; +}; + +type ChangeIODeviceFulfilledActionType = { + type: 'calling/CHANGE_IO_DEVICE_FULFILLED'; + payload: ChangeIODevicePayloadType; +}; + type DeclineCallActionType = { type: 'calling/DECLINE_CALL'; payload: DeclineCallType; @@ -135,6 +152,11 @@ type OutgoingCallActionType = { payload: OutgoingCallType; }; +type RefreshIODevicesActionType = { + type: 'calling/REFRESH_IO_DEVICES'; + payload: MediaDeviceSettings; +}; + type RemoteVideoChangeActionType = { type: 'calling/REMOTE_VIDEO_CHANGE'; payload: RemoteVideoChangeType; @@ -155,18 +177,26 @@ type SetLocalVideoFulfilledActionType = { payload: SetLocalVideoType; }; +type ToggleSettingsActionType = { + type: 'calling/TOGGLE_SETTINGS'; +}; + export type CallingActionType = | AcceptCallActionType | CallStateChangeActionType | CallStateChangeFulfilledActionType + | ChangeIODeviceActionType + | ChangeIODeviceFulfilledActionType | DeclineCallActionType | HangUpActionType | IncomingCallActionType | OutgoingCallActionType + | RefreshIODevicesActionType | RemoteVideoChangeActionType | SetLocalAudioActionType | SetLocalVideoActionType - | SetLocalVideoFulfilledActionType; + | SetLocalVideoFulfilledActionType + | ToggleSettingsActionType; // Action Creators @@ -197,6 +227,29 @@ function callStateChange( }; } +function changeIODevice( + payload: ChangeIODevicePayloadType +): ChangeIODeviceActionType { + return { + type: CHANGE_IO_DEVICE, + payload: doChangeIODevice(payload), + }; +} + +async function doChangeIODevice( + payload: ChangeIODevicePayloadType +): Promise { + if (payload.type === CallingDeviceType.CAMERA) { + await calling.setPreferredCamera(payload.selectedDevice); + } else if (payload.type === CallingDeviceType.MICROPHONE) { + calling.setPreferredMicrophone(payload.selectedDevice); + } else if (payload.type === CallingDeviceType.SPEAKER) { + calling.setPreferredSpeaker(payload.selectedDevice); + } + + return payload; +} + async function doCallStateChange( payload: CallStateChangeType ): Promise { @@ -208,12 +261,11 @@ async function doCallStateChange( bounceAppIconStart(); } if (callState !== CallState.Ringing) { - callingTones.stopRingtone(); + await callingTones.stopRingtone(); bounceAppIconStop(); } if (callState === CallState.Ended) { - // tslint:disable-next-line no-floating-promises - callingTones.playEndCall(); + await callingTones.playEndCall(); } return payload; } @@ -275,6 +327,15 @@ function outgoingCall(payload: OutgoingCallType): OutgoingCallActionType { }; } +function refreshIODevices( + payload: MediaDeviceSettings +): RefreshIODevicesActionType { + return { + type: REFRESH_IO_DEVICES, + payload, + }; +} + function remoteVideoChange( payload: RemoteVideoChangeType ): RemoteVideoChangeActionType { @@ -284,8 +345,8 @@ function remoteVideoChange( }; } -function setVideoCapturer(payload: SetVideoCapturerType): NoopActionType { - calling.setVideoCapturer(payload.callId, payload.capturer as VideoCapturer); +function setLocalPreview(payload: SetLocalPreviewType): NoopActionType { + calling.videoCapturer.setLocalPreview(payload.element); return { type: 'NOOP', @@ -293,8 +354,8 @@ function setVideoCapturer(payload: SetVideoCapturerType): NoopActionType { }; } -function setVideoRenderer(payload: SetVideoRendererType): NoopActionType { - calling.setVideoRenderer(payload.callId, payload.renderer as VideoRenderer); +function setRendererCanvas(payload: SetRendererCanvasType): NoopActionType { + calling.videoRenderer.setCanvas(payload.element); return { type: 'NOOP', @@ -318,6 +379,12 @@ function setLocalVideo(payload: SetLocalVideoType): SetLocalVideoActionType { }; } +function toggleSettings(): ToggleSettingsActionType { + return { + type: TOGGLE_SETTINGS, + }; +} + async function doSetLocalVideo( payload: SetLocalVideoType ): Promise { @@ -335,15 +402,18 @@ async function doSetLocalVideo( export const actions = { acceptCall, callStateChange, + changeIODevice, declineCall, hangUp, incomingCall, outgoingCall, + refreshIODevices, remoteVideoChange, - setVideoCapturer, - setVideoRenderer, + setLocalPreview, + setRendererCanvas, setLocalAudio, setLocalVideo, + toggleSettings, }; export type ActionsType = typeof actions; @@ -352,14 +422,22 @@ export type ActionsType = typeof actions; function getEmptyState(): CallingStateType { return { + availableCameras: [], + availableMicrophones: [], + availableSpeakers: [], callDetails: undefined, callState: undefined, hasLocalAudio: false, hasLocalVideo: false, hasRemoteVideo: false, + selectedCamera: undefined, + selectedMicrophone: undefined, + selectedSpeaker: undefined, + settingsDialogOpen: false, }; } +// tslint:disable-next-line max-func-body-length export function reducer( state: CallingStateType = getEmptyState(), action: CallingActionType @@ -425,5 +503,51 @@ export function reducer( }; } + if (action.type === CHANGE_IO_DEVICE_FULFILLED) { + const { selectedDevice } = action.payload; + const nextState = Object.create(null); + + if (action.payload.type === CallingDeviceType.CAMERA) { + nextState.selectedCamera = selectedDevice; + } else if (action.payload.type === CallingDeviceType.MICROPHONE) { + nextState.selectedMicrophone = selectedDevice; + } else if (action.payload.type === CallingDeviceType.SPEAKER) { + nextState.selectedSpeaker = selectedDevice; + } + + return { + ...state, + ...nextState, + }; + } + + if (action.type === REFRESH_IO_DEVICES) { + const { + availableMicrophones, + selectedMicrophone, + availableSpeakers, + selectedSpeaker, + availableCameras, + selectedCamera, + } = action.payload; + + return { + ...state, + availableMicrophones, + selectedMicrophone, + availableSpeakers, + selectedSpeaker, + availableCameras, + selectedCamera, + }; + } + + if (action.type === TOGGLE_SETTINGS) { + return { + ...state, + settingsDialogOpen: !state.settingsDialogOpen, + }; + } + return state; } diff --git a/ts/state/smart/CallManager.tsx b/ts/state/smart/CallManager.tsx index 4d37d64cc..c995ca612 100644 --- a/ts/state/smart/CallManager.tsx +++ b/ts/state/smart/CallManager.tsx @@ -1,20 +1,26 @@ -import { RefObject } from 'react'; +import React from 'react'; import { connect } from 'react-redux'; -import { CanvasVideoRenderer, GumVideoCapturer } from 'ringrtc'; import { mapDispatchToProps } from '../actions'; import { CallManager } from '../../components/CallManager'; import { StateType } from '../reducer'; import { getIntl } from '../selectors/user'; +import { SmartCallingDeviceSelection } from './CallingDeviceSelection'; + +// Workaround: A react component's required properties are filtering up through connect() +// https://github.com/DefinitelyTyped/DefinitelyTyped/issues/31363 +const FilteredCallingDeviceSelection = SmartCallingDeviceSelection as any; + +function renderDeviceSelection(): JSX.Element { + return ; +} + const mapStateToProps = (state: StateType) => { return { ...state.calling, i18n: getIntl(state), - getVideoCapturer: (localVideoRef: RefObject) => - new GumVideoCapturer(640, 480, 30, localVideoRef), - getVideoRenderer: (remoteVideoRef: RefObject) => - new CanvasVideoRenderer(remoteVideoRef), + renderDeviceSelection, }; }; diff --git a/ts/state/smart/CallingDeviceSelection.tsx b/ts/state/smart/CallingDeviceSelection.tsx new file mode 100644 index 000000000..b2d2d96fa --- /dev/null +++ b/ts/state/smart/CallingDeviceSelection.tsx @@ -0,0 +1,31 @@ +import { connect } from 'react-redux'; +import { mapDispatchToProps } from '../actions'; +import { CallingDeviceSelection } from '../../components/CallingDeviceSelection'; +import { StateType } from '../reducer'; + +import { getIntl } from '../selectors/user'; + +const mapStateToProps = (state: StateType) => { + const { + availableMicrophones, + availableSpeakers, + selectedMicrophone, + selectedSpeaker, + availableCameras, + selectedCamera, + } = state.calling; + + return { + availableCameras, + availableMicrophones, + availableSpeakers, + i18n: getIntl(state), + selectedCamera, + selectedMicrophone, + selectedSpeaker, + }; +}; + +const smart = connect(mapStateToProps, mapDispatchToProps); + +export const SmartCallingDeviceSelection = smart(CallingDeviceSelection); diff --git a/ts/types/Calling.ts b/ts/types/Calling.ts index 975d1865e..c698a792c 100644 --- a/ts/types/Calling.ts +++ b/ts/types/Calling.ts @@ -1,3 +1,15 @@ +// Must be kept in sync with RingRTC.AudioDevice +export interface AudioDevice { + // Name, present on every platform. + name: string; + // Index of this device, starting from 0. + index: number; + // Index of this device out of all devices sharing the same name. + same_name_index: number; + // If present, a unique and stable identifier of this device. Only available on WIndows. + unique_id?: string; +} + // This must be kept in sync with RingRTC.CallState. export enum CallState { Prering = 'init', @@ -6,3 +18,23 @@ export enum CallState { Reconnecting = 'connecting', Ended = 'ended', } + +export enum CallingDeviceType { + CAMERA, + MICROPHONE, + SPEAKER, +} + +export type MediaDeviceSettings = { + availableMicrophones: Array; + selectedMicrophone: AudioDevice | undefined; + availableSpeakers: Array; + selectedSpeaker: AudioDevice | undefined; + availableCameras: Array; + selectedCamera: string | undefined; +}; + +export type ChangeIODevicePayloadType = + | { type: CallingDeviceType.CAMERA; selectedDevice: string } + | { type: CallingDeviceType.MICROPHONE; selectedDevice: AudioDevice } + | { type: CallingDeviceType.SPEAKER; selectedDevice: AudioDevice }; diff --git a/ts/util/callingTones.ts b/ts/util/callingTones.ts index 98c3fe1ae..95aec08f8 100644 --- a/ts/util/callingTones.ts +++ b/ts/util/callingTones.ts @@ -1,43 +1,51 @@ -import { Sound, SoundOpts } from './Sound'; +import { Sound } from './Sound'; +import PQueue from 'p-queue'; -async function playSound(howlProps: SoundOpts): Promise { - const canPlayTone = await window.getCallRingtoneNotification(); - - if (!canPlayTone) { - return; - } - - const tone = new Sound(howlProps); - await tone.play(); - - return tone; -} +const ringtoneEventQueue = new PQueue({ concurrency: 1 }); class CallingTones { private ringtone?: Sound; async playEndCall() { - await playSound({ + const canPlayTone = await window.getCallRingtoneNotification(); + if (!canPlayTone) { + return; + } + + const tone = new Sound({ src: 'sounds/navigation-cancel.ogg', }); + await tone.play(); } async playRingtone() { - if (this.ringtone) { - this.stopRingtone(); - } + await ringtoneEventQueue.add(async () => { + if (this.ringtone) { + this.ringtone.stop(); + this.ringtone = undefined; + } - this.ringtone = await playSound({ - loop: true, - src: 'sounds/ringtone_minimal.ogg', + const canPlayTone = await window.getCallRingtoneNotification(); + if (!canPlayTone) { + return; + } + + this.ringtone = new Sound({ + loop: true, + src: 'sounds/ringtone_minimal.ogg', + }); + + await this.ringtone.play(); }); } - stopRingtone() { - if (this.ringtone) { - this.ringtone.stop(); - this.ringtone = undefined; - } + async stopRingtone() { + await ringtoneEventQueue.add(async () => { + if (this.ringtone) { + this.ringtone.stop(); + this.ringtone = undefined; + } + }); } } diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index a52461bc0..8aa755f5e 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -11304,7 +11304,7 @@ "rule": "React-createRef", "path": "ts/components/CallScreen.tsx", "line": " this.localVideoRef = React.createRef();", - "lineNumber": 80, + "lineNumber": 74, "reasonCategory": "usageTrusted", "updated": "2020-06-02T21:51:34.813Z", "reasonDetail": "Used to render local preview video" diff --git a/yarn.lock b/yarn.lock index cea19b91c..c9a751daa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14923,9 +14923,9 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -"ringrtc@https://github.com/signalapp/signal-ringrtc-node.git#de96fa13f04e846bdcb0ae9be8e2dc80a932f2be": - version "2.4.2" - resolved "https://github.com/signalapp/signal-ringrtc-node.git#de96fa13f04e846bdcb0ae9be8e2dc80a932f2be" +"ringrtc@https://github.com/signalapp/signal-ringrtc-node.git#0cd60f529d8f17734fe19e7195c9fd3c3f4271db": + version "2.5.0" + resolved "https://github.com/signalapp/signal-ringrtc-node.git#0cd60f529d8f17734fe19e7195c9fd3c3f4271db" ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.1"