From fb9c10f126a59fcfb34f72e4480357b3af02b777 Mon Sep 17 00:00:00 2001 From: automated-signal <37887102+automated-signal@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:54:20 -0700 Subject: [PATCH] Blur participant videos when calls are reconnecting --- stylesheets/_modules.scss | 6 ++++++ ts/components/CallScreen.tsx | 16 +++++---------- ts/components/CallingPipRemoteVideo.tsx | 3 +++ ts/components/CallingToastManager.tsx | 8 +++----- ts/components/DirectCallRemoteParticipant.tsx | 9 ++++++++- .../GroupCallOverflowArea.stories.tsx | 1 + ts/components/GroupCallOverflowArea.tsx | 3 +++ .../GroupCallRemoteParticipant.stories.tsx | 1 + ts/components/GroupCallRemoteParticipant.tsx | 20 +++++++++++++++---- ts/components/GroupCallRemoteParticipants.tsx | 4 ++++ ts/util/callingIsReconnecting.ts | 18 +++++++++++++++++ 11 files changed, 68 insertions(+), 21 deletions(-) create mode 100644 ts/util/callingIsReconnecting.ts diff --git a/stylesheets/_modules.scss b/stylesheets/_modules.scss index f40c601a0..4f34caa52 100644 --- a/stylesheets/_modules.scss +++ b/stylesheets/_modules.scss @@ -3655,6 +3655,9 @@ button.module-image__border-overlay:focus { background-color: $color-gray-95; height: 100%; width: 100%; + &--reconnecting { + filter: blur(15px); + } } &__remote-video-disabled { @@ -3844,6 +3847,9 @@ button.module-image__border-overlay:focus { &__remote-video { // The background-color is seen while the video loads. background-color: $color-gray-75; + &--reconnecting { + filter: blur(15px); + } } &__blocked { diff --git a/ts/components/CallScreen.tsx b/ts/components/CallScreen.tsx index 60fa0a3d7..8a842630a 100644 --- a/ts/components/CallScreen.tsx +++ b/ts/components/CallScreen.tsx @@ -49,6 +49,7 @@ import { useKeyboardShortcuts, } from '../hooks/useKeyboardShortcuts'; import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate'; +import { isReconnecting } from '../util/callingIsReconnecting'; export type PropsType = { activeCall: ActiveCallType; @@ -113,9 +114,6 @@ function DirectCallHeaderMessage({ return clearInterval.bind(null, interval); }, [joinedAt]); - if (callState === CallState.Reconnecting) { - return <>{i18n('icu:callReconnecting')}; - } if (callState === CallState.Accepted && acceptedDuration) { return ( <> @@ -298,6 +296,7 @@ export function CallScreen({ conversation={conversation} hasRemoteVideo={hasRemoteVideo} i18n={i18n} + isReconnecting={isReconnecting(activeCall)} setRendererCanvas={setRendererCanvas} /> ) : ( @@ -333,6 +332,7 @@ export function CallScreen({ remoteParticipants={activeCall.remoteParticipants} setGroupCallVideoRequest={setGroupCallVideoRequest} remoteAudioLevels={activeCall.remoteAudioLevels} + isCallReconnecting={isReconnecting(activeCall)} /> ); break; @@ -343,15 +343,9 @@ export function CallScreen({ let lonelyInCallNode: ReactNode; let localPreviewNode: ReactNode; - const isLonelyInGroup = - activeCall.callMode === CallMode.Group && - !activeCall.remoteParticipants.length; + const isLonelyInCall = !activeCall.remoteParticipants.length; - const isLonelyInDirectCall = - activeCall.callMode === CallMode.Direct && - activeCall.callState !== CallState.Accepted; - - if (isLonelyInGroup || isLonelyInDirectCall) { + if (isLonelyInCall) { lonelyInCallNode = (
@@ -173,6 +175,7 @@ export function CallingPipRemoteVideo({ remoteParticipant={activeGroupCallSpeaker} remoteParticipantsCount={activeCall.remoteParticipants.length} isActiveSpeakerInSpeakerView={false} + isCallReconnecting={isReconnecting(activeCall)} /> ); diff --git a/ts/components/CallingToastManager.tsx b/ts/components/CallingToastManager.tsx index b5069501a..054f0e505 100644 --- a/ts/components/CallingToastManager.tsx +++ b/ts/components/CallingToastManager.tsx @@ -3,11 +3,12 @@ import React, { useCallback, useEffect, useRef, useState } from 'react'; import type { ActiveCallType } from '../types/Calling'; -import { CallMode, GroupCallConnectionState } from '../types/Calling'; +import { CallMode } from '../types/Calling'; import type { ConversationType } from '../state/ducks/conversations'; import type { LocalizerType } from '../types/Util'; import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary'; import { CallingToast, DEFAULT_LIFETIME } from './CallingToast'; +import { isReconnecting } from '../util/callingIsReconnecting'; type PropsType = { activeCall: ActiveCallType; @@ -22,10 +23,7 @@ type ToastType = | undefined; function getReconnectingToast({ activeCall, i18n }: PropsType): ToastType { - if ( - activeCall.callMode === CallMode.Group && - activeCall.connectionState === GroupCallConnectionState.Reconnecting - ) { + if (isReconnecting(activeCall)) { return { message: i18n('icu:callReconnecting'), type: 'static', diff --git a/ts/components/DirectCallRemoteParticipant.tsx b/ts/components/DirectCallRemoteParticipant.tsx index edc51e5d4..2d6201f40 100644 --- a/ts/components/DirectCallRemoteParticipant.tsx +++ b/ts/components/DirectCallRemoteParticipant.tsx @@ -2,6 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import React, { useRef, useEffect } from 'react'; +import classNames from 'classnames'; import type { SetRendererCanvasType } from '../state/ducks/calling'; import type { ConversationType } from '../state/ducks/conversations'; import type { LocalizerType } from '../types/Util'; @@ -12,6 +13,7 @@ type PropsType = { conversation: ConversationType; hasRemoteVideo: boolean; i18n: LocalizerType; + isReconnecting: boolean; setRendererCanvas: (_: SetRendererCanvasType) => void; }; @@ -19,6 +21,7 @@ export function DirectCallRemoteParticipant({ conversation, hasRemoteVideo, i18n, + isReconnecting, setRendererCanvas, }: PropsType): JSX.Element { const remoteVideoRef = useRef(null); @@ -32,7 +35,11 @@ export function DirectCallRemoteParticipant({ return hasRemoteVideo ? ( ) : ( diff --git a/ts/components/GroupCallOverflowArea.stories.tsx b/ts/components/GroupCallOverflowArea.stories.tsx index 17cf8eb16..bafeabb2d 100644 --- a/ts/components/GroupCallOverflowArea.stories.tsx +++ b/ts/components/GroupCallOverflowArea.stories.tsx @@ -42,6 +42,7 @@ const defaultProps = { getFrameBuffer: memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE)), getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource, i18n, + isCallReconnecting: false, onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'), remoteAudioLevels: new Map(), remoteParticipantsCount: 1, diff --git a/ts/components/GroupCallOverflowArea.tsx b/ts/components/GroupCallOverflowArea.tsx index 76edcebe6..7655e8d1a 100644 --- a/ts/components/GroupCallOverflowArea.tsx +++ b/ts/components/GroupCallOverflowArea.tsx @@ -19,6 +19,7 @@ export type PropsType = { getFrameBuffer: () => Buffer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; + isCallReconnecting: boolean; onParticipantVisibilityChanged: ( demuxId: number, isVisible: boolean @@ -32,6 +33,7 @@ export function GroupCallOverflowArea({ getFrameBuffer, getGroupCallVideoFrameSource, i18n, + isCallReconnecting, onParticipantVisibilityChanged, overflowedParticipants, remoteAudioLevels, @@ -127,6 +129,7 @@ export function GroupCallOverflowArea({ remoteParticipant={remoteParticipant} remoteParticipantsCount={remoteParticipantsCount} isActiveSpeakerInSpeakerView={false} + isCallReconnecting={isCallReconnecting} /> ))} diff --git a/ts/components/GroupCallRemoteParticipant.stories.tsx b/ts/components/GroupCallRemoteParticipant.stories.tsx index 3eb13477a..7a25a379c 100644 --- a/ts/components/GroupCallRemoteParticipant.stories.tsx +++ b/ts/components/GroupCallRemoteParticipant.stories.tsx @@ -67,6 +67,7 @@ const createProps = ( }, remoteParticipantsCount: 1, isActiveSpeakerInSpeakerView: false, + isCallReconnecting: false, ...overrideProps, }); diff --git a/ts/components/GroupCallRemoteParticipant.tsx b/ts/components/GroupCallRemoteParticipant.tsx index afac6d6fb..4f4b38ee0 100644 --- a/ts/components/GroupCallRemoteParticipant.tsx +++ b/ts/components/GroupCallRemoteParticipant.tsx @@ -28,7 +28,7 @@ import { useIntersectionObserver } from '../hooks/useIntersectionObserver'; import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants'; import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate'; -const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 5000; +const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 10000; const MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES = 60000; type BasePropsType = { @@ -36,6 +36,7 @@ type BasePropsType = { getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; isActiveSpeakerInSpeakerView: boolean; + isCallReconnecting: boolean; onVisibilityChanged?: (demuxId: number, isVisible: boolean) => unknown; remoteParticipant: GroupCallRemoteParticipantType; remoteParticipantsCount: number; @@ -69,6 +70,7 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( onVisibilityChanged, remoteParticipantsCount, isActiveSpeakerInSpeakerView, + isCallReconnecting, } = props; const { @@ -136,7 +138,13 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( ? MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES : MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES; if (frameAge > maxFrameAge) { - setHasReceivedVideoRecently(false); + // We consider that we have received video recently from a remote participant if + // we have received it recently relative to the last time we had a connection. If + // we lost their video due to our reconnecting, we still want to show the last + // frame of video (blurred out) until we have reconnected. + if (!isCallReconnecting) { + setHasReceivedVideoRecently(false); + } } const canvasEl = remoteVideoRef.current; @@ -191,7 +199,7 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( setHasReceivedVideoRecently(true); setIsWide(frameWidth > frameHeight); - }, [getFrameBuffer, videoFrameSource, sharingScreen]); + }, [getFrameBuffer, videoFrameSource, sharingScreen, isCallReconnecting]); useEffect(() => { if (!hasRemoteVideo) { @@ -310,7 +318,11 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( )} {wantsToShowVideo && ( VideoFrameSource; i18n: LocalizerType; + isCallReconnecting: boolean; isInSpeakerView: boolean; remoteParticipants: ReadonlyArray; setGroupCallVideoRequest: ( @@ -87,6 +88,7 @@ enum VideoRequestMode { export function GroupCallRemoteParticipants({ getGroupCallVideoFrameSource, i18n, + isCallReconnecting, isInSpeakerView, remoteParticipants, setGroupCallVideoRequest, @@ -297,6 +299,7 @@ export function GroupCallRemoteParticipants({ width={renderedWidth} remoteParticipantsCount={remoteParticipants.length} isActiveSpeakerInSpeakerView={isInSpeakerView} + isCallReconnecting={isCallReconnecting} /> ); }); @@ -424,6 +427,7 @@ export function GroupCallRemoteParticipants({ getFrameBuffer={getFrameBuffer} getGroupCallVideoFrameSource={getGroupCallVideoFrameSource} i18n={i18n} + isCallReconnecting={isCallReconnecting} onParticipantVisibilityChanged={onParticipantVisibilityChanged} overflowedParticipants={overflowedParticipants} remoteAudioLevels={remoteAudioLevels} diff --git a/ts/util/callingIsReconnecting.ts b/ts/util/callingIsReconnecting.ts new file mode 100644 index 000000000..9c44b4b64 --- /dev/null +++ b/ts/util/callingIsReconnecting.ts @@ -0,0 +1,18 @@ +// Copyright 2023 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { + CallMode, + CallState, + GroupCallConnectionState, +} from '../types/Calling'; +import type { ActiveCallType } from '../types/Calling'; + +export function isReconnecting(activeCall: ActiveCallType): boolean { + return ( + (activeCall.callMode === CallMode.Group && + activeCall.connectionState === GroupCallConnectionState.Reconnecting) || + (activeCall.callMode === CallMode.Direct && + activeCall.callState === CallState.Reconnecting) + ); +}