From cf4b068ab245a91d84ce2e6e309adec71228962d Mon Sep 17 00:00:00 2001 From: Scott Nonnenberg Date: Sat, 1 Mar 2025 05:42:08 +1000 Subject: [PATCH] Show speaking indicator in 1:1 calls --- stylesheets/_modules.scss | 6 ++ ts/components/CallManager.stories.tsx | 2 + ts/components/CallScreen.stories.tsx | 1 + ts/components/CallScreen.tsx | 9 ++ ts/components/CallingPip.stories.tsx | 1 + ts/services/calling.ts | 24 +++-- ts/state/createStore.ts | 3 + ts/state/ducks/calling.ts | 90 +++++++++++++++++-- ts/state/smart/CallManager.tsx | 1 + ts/test-electron/state/ducks/calling_test.ts | 5 ++ .../state/selectors/calling_test.ts | 4 + ts/types/Calling.ts | 1 + 12 files changed, 134 insertions(+), 13 deletions(-) diff --git a/stylesheets/_modules.scss b/stylesheets/_modules.scss index b0b040bb5..b871c2f52 100644 --- a/stylesheets/_modules.scss +++ b/stylesheets/_modules.scss @@ -4136,6 +4136,12 @@ button.module-image__border-overlay:focus { flex: 1; } + &__direct-call-speaking-indicator { + position: absolute; + inset-inline-end: 16px; + bottom: 112px; + } + &__participants { display: flex; flex: 1 1 0; diff --git a/ts/components/CallManager.stories.tsx b/ts/components/CallManager.stories.tsx index dfe10cec2..6b098a7f4 100644 --- a/ts/components/CallManager.stories.tsx +++ b/ts/components/CallManager.stories.tsx @@ -204,6 +204,7 @@ export function OngoingDirectCall(): JSX.Element { callMode: CallMode.Direct, callState: CallState.Accepted, peekedParticipants: [], + remoteAudioLevel: 0, remoteParticipants: [ { hasRemoteVideo: true, presenting: false, title: 'Remy' }, ], @@ -291,6 +292,7 @@ export function CallRequestNeeded(): JSX.Element { callMode: CallMode.Direct, callState: CallState.Accepted, peekedParticipants: [], + remoteAudioLevel: 0, remoteParticipants: [ { hasRemoteVideo: true, presenting: false, title: 'Mike' }, ], diff --git a/ts/components/CallScreen.stories.tsx b/ts/components/CallScreen.stories.tsx index 5dfd66a46..692ee6e0d 100644 --- a/ts/components/CallScreen.stories.tsx +++ b/ts/components/CallScreen.stories.tsx @@ -84,6 +84,7 @@ const createActiveDirectCallProp = ( conversation, callState: overrideProps.callState ?? CallState.Accepted, peekedParticipants: [] as [], + remoteAudioLevel: 0, remoteParticipants: [ { hasRemoteVideo: overrideProps.hasRemoteVideo ?? false, diff --git a/ts/components/CallScreen.tsx b/ts/components/CallScreen.tsx index 335dc5858..03474ebd2 100644 --- a/ts/components/CallScreen.tsx +++ b/ts/components/CallScreen.tsx @@ -876,6 +876,15 @@ export function CallScreen({ } /> ) : null} + {activeCall.callMode === CallMode.Direct && ( +
+ 0} + /> +
+ )} {/* We render the local preview first and set the footer flex direction to row-reverse to ensure the preview is visible at low viewport widths. */}
diff --git a/ts/components/CallingPip.stories.tsx b/ts/components/CallingPip.stories.tsx index c8dc7fa75..227bed58d 100644 --- a/ts/components/CallingPip.stories.tsx +++ b/ts/components/CallingPip.stories.tsx @@ -61,6 +61,7 @@ const getDefaultCall = (overrides: Overrides): ActiveDirectCallType => { callMode: CallMode.Direct as CallMode.Direct, callState: CallState.Accepted, peekedParticipants: [], + remoteAudioLevel: 0, remoteParticipants: [ { hasRemoteVideo: true, presenting: false, title: 'Arsene' }, ], diff --git a/ts/services/calling.ts b/ts/services/calling.ts index 204b2038f..ba12b64ba 100644 --- a/ts/services/calling.ts +++ b/ts/services/calling.ts @@ -195,6 +195,7 @@ type CallingReduxInterface = Pick< | 'callStateChange' | 'cancelIncomingGroupCallRing' | 'cancelPresenting' + | 'directCallAudioLevelsChange' | 'groupCallAudioLevelsChange' | 'groupCallEnded' | 'groupCallRaisedHandsChange' @@ -3038,7 +3039,8 @@ export class CallingClass { } #attachToCall(conversation: ConversationModel, call: Call): void { - this.#callsLookup[conversation.id] = call; + const conversationId = conversation.id; + this.#callsLookup[conversationId] = call; const reduxInterface = this.#reduxInterface; if (!reduxInterface) { @@ -3056,7 +3058,7 @@ export class CallingClass { if (call.state === CallState.Ended) { this.#stopDeviceReselectionTimer(); this.#lastMediaDeviceSettings = undefined; - delete this.#callsLookup[conversation.id]; + delete this.#callsLookup[conversationId]; } const localCallEvent = getLocalCallEventFromDirectCall(call); @@ -3072,7 +3074,7 @@ export class CallingClass { } reduxInterface.callStateChange({ - conversationId: conversation.id, + conversationId, callState: call.state, callEndedReason: call.endedReason, acceptedTime, @@ -3087,7 +3089,7 @@ export class CallingClass { // eslint-disable-next-line no-param-reassign call.handleRemoteVideoEnabled = () => { reduxInterface.remoteVideoChange({ - conversationId: conversation.id, + conversationId, hasVideo: call.remoteVideoEnabled, }); }; @@ -3095,11 +3097,20 @@ export class CallingClass { // eslint-disable-next-line no-param-reassign call.handleRemoteSharingScreen = () => { reduxInterface.remoteSharingScreenChange({ - conversationId: conversation.id, + conversationId, isSharingScreen: Boolean(call.remoteSharingScreen), }); }; + // eslint-disable-next-line no-param-reassign + call.handleAudioLevels = () => { + reduxInterface.directCallAudioLevelsChange({ + conversationId, + localAudioLevel: call.outgoingAudioLevel, + remoteAudioLevel: call.remoteAudioLevel, + }); + }; + // eslint-disable-next-line no-param-reassign call.handleLowBandwidthForVideo = _recovered => { // TODO: Implement handling of "low outgoing bandwidth for video" notification. @@ -3284,8 +3295,7 @@ export class CallingClass { iceServers, hideIp: shouldRelayCalls || isContactUntrusted, dataMode: DataMode.Normal, - // TODO: DESKTOP-3101 - // audioLevelsIntervalMillis: AUDIO_LEVEL_INTERVAL_MS, + audioLevelsIntervalMillis: AUDIO_LEVEL_INTERVAL_MS, }; log.info('CallingClass.handleStartCall(): Proceeding'); diff --git a/ts/state/createStore.ts b/ts/state/createStore.ts index 7480dbfc2..c49ce7612 100644 --- a/ts/state/createStore.ts +++ b/ts/state/createStore.ts @@ -47,6 +47,9 @@ const logger = createLogger({ if (action.type === 'calling/GROUP_CALL_AUDIO_LEVELS_CHANGE') { return false; } + if (action.type === 'calling/DIRECT_CALL_AUDIO_LEVELS_CHANGE') { + return false; + } return true; }, }); diff --git a/ts/state/ducks/calling.ts b/ts/state/ducks/calling.ts index 14d22f442..9ef6c9072 100644 --- a/ts/state/ducks/calling.ts +++ b/ts/state/ducks/calling.ts @@ -148,6 +148,7 @@ export type DirectCallStateType = { isSharingScreen?: boolean; isVideoCall: boolean; hasRemoteVideo?: boolean; + remoteAudioLevel: number; }; type GroupCallRingStateType = ReadonlyDeep< @@ -624,6 +625,8 @@ const CHANGE_IO_DEVICE_FULFILLED = 'calling/CHANGE_IO_DEVICE_FULFILLED'; const CLOSE_NEED_PERMISSION_SCREEN = 'calling/CLOSE_NEED_PERMISSION_SCREEN'; const DECLINE_DIRECT_CALL = 'calling/DECLINE_DIRECT_CALL'; const GROUP_CALL_AUDIO_LEVELS_CHANGE = 'calling/GROUP_CALL_AUDIO_LEVELS_CHANGE'; +const DIRECT_CALL_AUDIO_LEVELS_CHANGE = + 'calling/DIRECT_CALL_AUDIO_LEVELS_CHANGE'; const GROUP_CALL_ENDED = 'calling/GROUP_CALL_ENDED'; const GROUP_CALL_RAISED_HANDS_CHANGE = 'calling/GROUP_CALL_RAISED_HANDS_CHANGE'; const GROUP_CALL_STATE_CHANGE = 'calling/GROUP_CALL_STATE_CHANGE'; @@ -734,11 +737,19 @@ type GroupCallAudioLevelsChangeActionPayloadType = ReadonlyDeep<{ localAudioLevel: number; remoteDeviceStates: ReadonlyArray<{ audioLevel: number; demuxId: number }>; }>; - type GroupCallAudioLevelsChangeActionType = ReadonlyDeep<{ - type: 'calling/GROUP_CALL_AUDIO_LEVELS_CHANGE'; + type: typeof GROUP_CALL_AUDIO_LEVELS_CHANGE; payload: GroupCallAudioLevelsChangeActionPayloadType; }>; +type DirectCallAudioLevelsChangeActionPayloadType = ReadonlyDeep<{ + conversationId: string; + localAudioLevel: number; + remoteAudioLevel: number; +}>; +type DirectCallAudioLevelsChangeActionType = ReadonlyDeep<{ + type: typeof DIRECT_CALL_AUDIO_LEVELS_CHANGE; + payload: DirectCallAudioLevelsChangeActionPayloadType; +}>; type GroupCallEndedActionPayloadType = ReadonlyDeep<{ conversationId: string; @@ -959,6 +970,7 @@ export type CallingActionType = | CancelIncomingGroupCallRingActionType | ChangeCallViewActionType | DenyUserActionType + | DirectCallAudioLevelsChangeActionType | StartCallingLobbyActionType | StartCallLinkLobbyActionType | CallStateChangeFulfilledActionType @@ -1205,6 +1217,12 @@ function callStateChange( }; } +function directCallAudioLevelsChange( + payload: DirectCallAudioLevelsChangeActionPayloadType +): DirectCallAudioLevelsChangeActionType { + return { type: DIRECT_CALL_AUDIO_LEVELS_CHANGE, payload }; +} + function changeIODevice( payload: ChangeIODevicePayloadType ): ThunkAction< @@ -2693,6 +2711,7 @@ export const actions = { declineCall, deleteCallLink, denyUser, + directCallAudioLevelsChange, getPresentingSources, groupCallAudioLevelsChange, groupCallEnded, @@ -2941,6 +2960,7 @@ export function reducer( conversationId, isIncoming: false, isVideoCall: action.payload.hasLocalVideo, + remoteAudioLevel: 0, }; outgoingRing = true; newAdhocCalls = adhocCalls; @@ -3066,6 +3086,7 @@ export function reducer( callState: CallState.Prering, isIncoming: false, isVideoCall: action.payload.hasLocalVideo, + remoteAudioLevel: 0, }, }, activeCallState: { @@ -3225,6 +3246,7 @@ export function reducer( callState: CallState.Prering, isIncoming: true, isVideoCall: action.payload.isVideoCall, + remoteAudioLevel: 0, }, }, }; @@ -3307,6 +3329,7 @@ export function reducer( callState: CallState.Prering, isIncoming: false, isVideoCall: action.payload.hasLocalVideo, + remoteAudioLevel: 0, }, }, activeCallState: { @@ -3392,17 +3415,29 @@ export function reducer( if (action.type === GROUP_CALL_AUDIO_LEVELS_CHANGE) { const { callMode, conversationId, remoteDeviceStates } = action.payload; - const { activeCallState } = state; - const existingCall = getGroupCall(conversationId, state, callMode); + + if ( + activeCallState && + (activeCallState.state === 'Waiting' || + activeCallState.conversationId !== conversationId) + ) { + log.warn( + `${action.type}: Cannot update levels; activeCall doesn't match conversation`, + activeCallState + ); + return state; + } // The PiP check is an optimization. We don't need to update audio levels if the user // cannot see them. + const existingCall = getGroupCall(conversationId, state, callMode); if ( !activeCallState || - activeCallState.state === 'Waiting' || activeCallState.pip || - !existingCall + !existingCall || + (existingCall.callMode !== CallMode.Adhoc && + existingCall.callMode !== CallMode.Group) ) { return state; } @@ -3445,6 +3480,49 @@ export function reducer( }; } + if (action.type === DIRECT_CALL_AUDIO_LEVELS_CHANGE) { + const { conversationId } = action.payload; + const { activeCallState } = state; + const existingCall = getOwn(state.callsByConversation, conversationId); + + if ( + activeCallState && + (activeCallState.state === 'Waiting' || + activeCallState.conversationId !== conversationId) + ) { + log.warn( + `${action.type}: Cannot update levels; activeCall doesn't match conversation`, + activeCallState + ); + return state; + } + + // The PiP check is an optimization. We don't need to update audio levels if the user + // cannot see them. + if ( + !activeCallState || + activeCallState.pip || + !existingCall || + existingCall.callMode !== CallMode.Direct + ) { + return state; + } + + const localAudioLevel = truncateAudioLevel(action.payload.localAudioLevel); + const remoteAudioLevel = truncateAudioLevel( + action.payload.remoteAudioLevel + ); + + return { + ...state, + activeCallState: { ...activeCallState, localAudioLevel }, + callsByConversation: { + ...state.callsByConversation, + [conversationId]: { ...existingCall, remoteAudioLevel }, + }, + }; + } + if (action.type === GROUP_CALL_STATE_CHANGE) { const { callMode, diff --git a/ts/state/smart/CallManager.tsx b/ts/state/smart/CallManager.tsx index 794addce7..c7160c138 100644 --- a/ts/state/smart/CallManager.tsx +++ b/ts/state/smart/CallManager.tsx @@ -204,6 +204,7 @@ const mapStateToActiveCallProp = ( callMode: CallMode.Direct, callState: call.callState, peekedParticipants: [], + remoteAudioLevel: call.remoteAudioLevel, remoteParticipants: [ { hasRemoteVideo: Boolean(call.hasRemoteVideo), diff --git a/ts/test-electron/state/ducks/calling_test.ts b/ts/test-electron/state/ducks/calling_test.ts index 7caf96b00..4b37cbad0 100644 --- a/ts/test-electron/state/ducks/calling_test.ts +++ b/ts/test-electron/state/ducks/calling_test.ts @@ -66,6 +66,7 @@ describe('calling duck', () => { isIncoming: false, isVideoCall: false, hasRemoteVideo: false, + remoteAudioLevel: 0, }; const stateWithDirectCall: CallingStateType = { ...getEmptyState(), @@ -102,6 +103,7 @@ describe('calling duck', () => { isIncoming: true, isVideoCall: false, hasRemoteVideo: false, + remoteAudioLevel: 0, } satisfies DirectCallStateType, }, }; @@ -2296,6 +2298,7 @@ describe('calling duck', () => { conversationId: 'fake-conversation-id', isIncoming: false, isVideoCall: true, + remoteAudioLevel: 0, }); assert.deepEqual(result.activeCallState, { state: 'Active', @@ -2694,6 +2697,7 @@ describe('calling duck', () => { callState: CallState.Prering, isIncoming: false, isVideoCall: false, + remoteAudioLevel: 0, }); assert.deepEqual(result.activeCallState, { state: 'Active', @@ -2934,6 +2938,7 @@ describe('calling duck', () => { isIncoming: false, isVideoCall: false, hasRemoteVideo: false, + remoteAudioLevel: 0, }); }); }); diff --git a/ts/test-electron/state/selectors/calling_test.ts b/ts/test-electron/state/selectors/calling_test.ts index ba73c5ecc..a649445e7 100644 --- a/ts/test-electron/state/selectors/calling_test.ts +++ b/ts/test-electron/state/selectors/calling_test.ts @@ -60,6 +60,7 @@ describe('state/selectors/calling', () => { isIncoming: false, isVideoCall: false, hasRemoteVideo: false, + remoteAudioLevel: 0, }, }, }; @@ -89,6 +90,7 @@ describe('state/selectors/calling', () => { isIncoming: true, isVideoCall: false, hasRemoteVideo: false, + remoteAudioLevel: 0, }; const stateWithIncomingDirectCall: CallingStateType = { @@ -151,6 +153,7 @@ describe('state/selectors/calling', () => { isIncoming: false, isVideoCall: false, hasRemoteVideo: false, + remoteAudioLevel: 0, }, } ); @@ -176,6 +179,7 @@ describe('state/selectors/calling', () => { isIncoming: false, isVideoCall: false, hasRemoteVideo: false, + remoteAudioLevel: 0, } ); }); diff --git a/ts/types/Calling.ts b/ts/types/Calling.ts index acd247def..f979e84d1 100644 --- a/ts/types/Calling.ts +++ b/ts/types/Calling.ts @@ -78,6 +78,7 @@ export type ActiveDirectCallType = ActiveCallBaseType & { serviceId?: ServiceIdString; }, ]; + remoteAudioLevel: number; }; export type ActiveGroupCallType = ActiveCallBaseType & {