From 8e1391c70cca2baec3b21b7aee384d363f09c37d Mon Sep 17 00:00:00 2001 From: Evan Hahn <69474926+EvanHahn-Signal@users.noreply.github.com> Date: Fri, 8 Jan 2021 11:32:49 -0600 Subject: [PATCH] Share group calling frame buffers to reduce memory usage --- ts/calling/constants.ts | 5 ++- ts/calling/useGetCallingFrameBuffer.ts | 24 +++++++++++++ ts/components/CallingPipRemoteVideo.tsx | 6 +++- .../GroupCallRemoteParticipant.stories.tsx | 8 +++-- ts/components/GroupCallRemoteParticipant.tsx | 35 +++++++++---------- ts/components/GroupCallRemoteParticipants.tsx | 5 ++- ts/util/lint/exceptions.json | 22 ++++++------ 7 files changed, 71 insertions(+), 34 deletions(-) create mode 100644 ts/calling/useGetCallingFrameBuffer.ts diff --git a/ts/calling/constants.ts b/ts/calling/constants.ts index b3e0889cd..1c7b1a6ba 100644 --- a/ts/calling/constants.ts +++ b/ts/calling/constants.ts @@ -1,6 +1,9 @@ -// Copyright 2020 Signal Messenger, LLC +// Copyright 2020-2021 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only export const REQUESTED_VIDEO_WIDTH = 640; export const REQUESTED_VIDEO_HEIGHT = 480; export const REQUESTED_VIDEO_FRAMERATE = 30; + +export const MAX_FRAME_SIZE = 1920 * 1080; +export const FRAME_BUFFER_SIZE = MAX_FRAME_SIZE * 4; diff --git a/ts/calling/useGetCallingFrameBuffer.ts b/ts/calling/useGetCallingFrameBuffer.ts new file mode 100644 index 000000000..92c31e5f8 --- /dev/null +++ b/ts/calling/useGetCallingFrameBuffer.ts @@ -0,0 +1,24 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { useRef, useCallback } from 'react'; +import { FRAME_BUFFER_SIZE } from './constants'; + +/** + * A hook that returns a function. This function returns a "singleton" `ArrayBuffer` to be + * used in call video rendering. + * + * This is most useful for group calls, where we can reuse the same frame buffer instead + * of allocating one per participant. Be careful when using this buffer elsewhere, as it + * is not cleaned up and may hold stale data. + */ +export function useGetCallingFrameBuffer(): () => ArrayBuffer { + const ref = useRef(null); + + return useCallback(() => { + if (!ref.current) { + ref.current = new ArrayBuffer(FRAME_BUFFER_SIZE); + } + return ref.current; + }, []); +} diff --git a/ts/components/CallingPipRemoteVideo.tsx b/ts/components/CallingPipRemoteVideo.tsx index 589146846..7d1650d16 100644 --- a/ts/components/CallingPipRemoteVideo.tsx +++ b/ts/components/CallingPipRemoteVideo.tsx @@ -1,4 +1,4 @@ -// Copyright 2020 Signal Messenger, LLC +// Copyright 2020-2021 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only import React, { useMemo, useEffect } from 'react'; @@ -16,6 +16,7 @@ import { VideoFrameSource, } from '../types/Calling'; import { SetRendererCanvasType } from '../state/ducks/calling'; +import { useGetCallingFrameBuffer } from '../calling/useGetCallingFrameBuffer'; import { usePageVisibility } from '../util/hooks'; import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant'; @@ -77,6 +78,8 @@ export const CallingPipRemoteVideo = ({ }: PropsType): JSX.Element => { const { conversation } = activeCall; + const getGroupCallFrameBuffer = useGetCallingFrameBuffer(); + const isPageVisible = usePageVisibility(); const activeGroupCallSpeaker: @@ -155,6 +158,7 @@ export const CallingPipRemoteVideo = ({ return (
new ArrayBuffer(FRAME_BUFFER_SIZE)); + const createProps = ( overrideProps: OverridePropsType, isBlocked?: boolean ): PropsType => ({ + getFrameBuffer, // eslint-disable-next-line @typescript-eslint/no-explicit-any getGroupCallVideoFrameSource: noop as any, i18n, diff --git a/ts/components/GroupCallRemoteParticipant.tsx b/ts/components/GroupCallRemoteParticipant.tsx index 5040b1048..5c2d3e1f3 100644 --- a/ts/components/GroupCallRemoteParticipant.tsx +++ b/ts/components/GroupCallRemoteParticipant.tsx @@ -1,4 +1,4 @@ -// Copyright 2020 Signal Messenger, LLC +// Copyright 2020-2021 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only import React, { @@ -21,11 +21,10 @@ import { Avatar, AvatarSize } from './Avatar'; import { ConfirmationModal } from './ConfirmationModal'; import { Intl } from './Intl'; import { ContactName } from './conversation/ContactName'; - -// The max size video frame we'll support (in RGBA) -const FRAME_BUFFER_SIZE = 1920 * 1080 * 4; +import { MAX_FRAME_SIZE } from '../calling/constants'; interface BasePropsType { + getFrameBuffer: () => ArrayBuffer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; remoteParticipant: GroupCallRemoteParticipantType; @@ -47,7 +46,7 @@ export type PropsType = BasePropsType & (InPipPropsType | NotInPipPropsType); export const GroupCallRemoteParticipant: React.FC = React.memo( props => { - const { getGroupCallVideoFrameSource, i18n } = props; + const { getFrameBuffer, getGroupCallVideoFrameSource, i18n } = props; const { avatarPath, @@ -66,9 +65,6 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( const remoteVideoRef = useRef(null); const canvasContextRef = useRef(null); - const frameBufferRef = useRef( - new ArrayBuffer(FRAME_BUFFER_SIZE) - ); const videoFrameSource = useMemo( () => getGroupCallVideoFrameSource(demuxId), @@ -86,15 +82,22 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( return; } - const frameDimensions = videoFrameSource.receiveVideoFrame( - frameBufferRef.current - ); + // This frame buffer is shared by all participants, so it may contain pixel data + // for other participants, or pixel data from a previous frame. That's why we + // return early and use the `frameWidth` and `frameHeight`. + const frameBuffer = getFrameBuffer(); + const frameDimensions = videoFrameSource.receiveVideoFrame(frameBuffer); if (!frameDimensions) { return; } const [frameWidth, frameHeight] = frameDimensions; - if (frameWidth < 2 || frameHeight < 2) { + + if ( + frameWidth < 2 || + frameHeight < 2 || + frameWidth * frameHeight > MAX_FRAME_SIZE + ) { return; } @@ -103,11 +106,7 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( canvasContext.putImageData( new ImageData( - new Uint8ClampedArray( - frameBufferRef.current, - 0, - frameWidth * frameHeight * 4 - ), + new Uint8ClampedArray(frameBuffer, 0, frameWidth * frameHeight * 4), frameWidth, frameHeight ), @@ -116,7 +115,7 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( ); setIsWide(frameWidth > frameHeight); - }, [videoFrameSource]); + }, [getFrameBuffer, videoFrameSource]); useEffect(() => { if (!hasRemoteVideo) { diff --git a/ts/components/GroupCallRemoteParticipants.tsx b/ts/components/GroupCallRemoteParticipants.tsx index 41f0f2f24..b893a3746 100644 --- a/ts/components/GroupCallRemoteParticipants.tsx +++ b/ts/components/GroupCallRemoteParticipants.tsx @@ -1,4 +1,4 @@ -// Copyright 2020 Signal Messenger, LLC +// Copyright 2020-2021 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only import React, { useState, useMemo, useEffect } from 'react'; @@ -10,6 +10,7 @@ import { GroupCallVideoRequest, VideoFrameSource, } from '../types/Calling'; +import { useGetCallingFrameBuffer } from '../calling/useGetCallingFrameBuffer'; import { LocalizerType } from '../types/Util'; import { usePageVisibility } from '../util/hooks'; import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant'; @@ -72,6 +73,7 @@ export const GroupCallRemoteParticipants: React.FC = ({ height: 0, }); const isPageVisible = usePageVisibility(); + const getFrameBuffer = useGetCallingFrameBuffer(); // 1. Figure out the maximum number of possible rows that could fit on the screen. // @@ -216,6 +218,7 @@ export const GroupCallRemoteParticipants: React.FC = ({ return (