Improve handling of edited long message attachments
This commit is contained in:
@@ -1,11 +1,10 @@
|
|||||||
// Copyright 2024 Signal Messenger, LLC
|
// Copyright 2024 Signal Messenger, LLC
|
||||||
// SPDX-License-Identifier: AGPL-3.0-only
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
import { isNumber, partition } from 'lodash';
|
import { isNumber } from 'lodash';
|
||||||
|
|
||||||
import * as log from '../logging/log';
|
import * as log from '../logging/log';
|
||||||
import * as Errors from '../types/errors';
|
import * as Errors from '../types/errors';
|
||||||
import * as MIME from '../types/MIME';
|
|
||||||
import * as LinkPreview from '../types/LinkPreview';
|
import * as LinkPreview from '../types/LinkPreview';
|
||||||
|
|
||||||
import { getAuthor, isStory, messageHasPaymentEvent } from './helpers';
|
import { getAuthor, isStory, messageHasPaymentEvent } from './helpers';
|
||||||
@@ -531,22 +530,16 @@ export async function handleDataMessage(
|
|||||||
const ourPni = window.textsecure.storage.user.getCheckedPni();
|
const ourPni = window.textsecure.storage.user.getCheckedPni();
|
||||||
const ourServiceIds: Set<ServiceIdString> = new Set([ourAci, ourPni]);
|
const ourServiceIds: Set<ServiceIdString> = new Set([ourAci, ourPni]);
|
||||||
|
|
||||||
const [longMessageAttachments, normalAttachments] = partition(
|
|
||||||
dataMessage.attachments ?? [],
|
|
||||||
attachment => MIME.isLongMessage(attachment.contentType)
|
|
||||||
);
|
|
||||||
const bodyAttachment = longMessageAttachments[0];
|
|
||||||
|
|
||||||
// eslint-disable-next-line no-param-reassign
|
// eslint-disable-next-line no-param-reassign
|
||||||
message = window.MessageCache.register(message);
|
message = window.MessageCache.register(message);
|
||||||
|
|
||||||
message.set({
|
message.set({
|
||||||
id: messageId,
|
id: messageId,
|
||||||
attachments: normalAttachments,
|
attachments: dataMessage.attachments,
|
||||||
bodyAttachment,
|
bodyAttachment: dataMessage.bodyAttachment,
|
||||||
// We don't want to trim if we'll be downloading a body attachment; we might
|
// We don't want to trim if we'll be downloading a body attachment; we might
|
||||||
// drop bodyRanges which apply to the longer text we'll get in that download.
|
// drop bodyRanges which apply to the longer text we'll get in that download.
|
||||||
...(bodyAttachment
|
...(dataMessage.bodyAttachment
|
||||||
? {
|
? {
|
||||||
body: dataMessage.body,
|
body: dataMessage.body,
|
||||||
bodyRanges: dataMessage.bodyRanges,
|
bodyRanges: dataMessage.bodyRanges,
|
||||||
|
@@ -11,7 +11,7 @@ import {
|
|||||||
} from '../textsecure/processDataMessage';
|
} from '../textsecure/processDataMessage';
|
||||||
import type { ProcessedAttachment } from '../textsecure/Types.d';
|
import type { ProcessedAttachment } from '../textsecure/Types.d';
|
||||||
import { SignalService as Proto } from '../protobuf';
|
import { SignalService as Proto } from '../protobuf';
|
||||||
import { IMAGE_GIF, IMAGE_JPEG } from '../types/MIME';
|
import { IMAGE_GIF, IMAGE_JPEG, LONG_MESSAGE } from '../types/MIME';
|
||||||
import { generateAci } from '../types/ServiceId';
|
import { generateAci } from '../types/ServiceId';
|
||||||
import { uuidToBytes } from '../util/uuidToBytes';
|
import { uuidToBytes } from '../util/uuidToBytes';
|
||||||
|
|
||||||
@@ -86,6 +86,30 @@ describe('processDataMessage', () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should move long text attachments to bodyAttachment', () => {
|
||||||
|
const out = check({
|
||||||
|
attachments: [
|
||||||
|
UNPROCESSED_ATTACHMENT,
|
||||||
|
{
|
||||||
|
...UNPROCESSED_ATTACHMENT,
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepStrictEqual(out.attachments, [
|
||||||
|
{
|
||||||
|
...PROCESSED_ATTACHMENT,
|
||||||
|
downloadPath: 'random-path',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
assert.deepStrictEqual(out.bodyAttachment, {
|
||||||
|
...PROCESSED_ATTACHMENT,
|
||||||
|
downloadPath: 'random-path',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('should process attachments with incrementalMac/chunkSize', () => {
|
it('should process attachments with incrementalMac/chunkSize', () => {
|
||||||
const out = check({
|
const out = check({
|
||||||
attachments: [
|
attachments: [
|
||||||
|
162
ts/test-node/util/queueAttachmentDownloads_test.ts
Normal file
162
ts/test-node/util/queueAttachmentDownloads_test.ts
Normal file
@@ -0,0 +1,162 @@
|
|||||||
|
// Copyright 2025 Signal Messenger, LLC
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
/* eslint-disable @typescript-eslint/no-non-null-assertion */
|
||||||
|
import { assert } from 'chai';
|
||||||
|
import type { MessageAttributesType } from '../../model-types';
|
||||||
|
import type { AttachmentType } from '../../types/Attachment';
|
||||||
|
import { IMAGE_JPEG, LONG_MESSAGE } from '../../types/MIME';
|
||||||
|
import { generateMessageId } from '../../util/generateMessageId';
|
||||||
|
import { ensureBodyAttachmentsAreSeparated } from '../../util/queueAttachmentDownloads';
|
||||||
|
import * as logger from '../../logging/log';
|
||||||
|
|
||||||
|
export function composeMessage(
|
||||||
|
overrides?: Partial<MessageAttributesType>
|
||||||
|
): MessageAttributesType {
|
||||||
|
return {
|
||||||
|
...generateMessageId(Date.now()),
|
||||||
|
sent_at: Date.now(),
|
||||||
|
timestamp: Date.now(),
|
||||||
|
type: 'incoming',
|
||||||
|
conversationId: 'conversationId',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
export function composeAttachment(
|
||||||
|
overrides?: Partial<AttachmentType>
|
||||||
|
): AttachmentType {
|
||||||
|
return {
|
||||||
|
size: 100,
|
||||||
|
contentType: IMAGE_JPEG,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('ensureBodyAttachmentsAreSeparated', () => {
|
||||||
|
it('separates first body attachment out, and drops any additional ones', () => {
|
||||||
|
const msg = composeMessage({
|
||||||
|
attachments: [
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'normal attachment',
|
||||||
|
contentType: IMAGE_JPEG,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'long message 1',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'long message 2',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const result = ensureBodyAttachmentsAreSeparated(msg, {
|
||||||
|
logId: 'test',
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
assert.deepEqual(result.attachments, [msg.attachments?.[0]]);
|
||||||
|
assert.deepEqual(result.bodyAttachment, msg.attachments?.[1]);
|
||||||
|
});
|
||||||
|
it('retains existing bodyAttachment', () => {
|
||||||
|
const msg = composeMessage({
|
||||||
|
bodyAttachment: composeAttachment({
|
||||||
|
clientUuid: 'existing body attachment',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
attachments: [
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'normal attachment',
|
||||||
|
contentType: IMAGE_JPEG,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'long message 1',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const result = ensureBodyAttachmentsAreSeparated(msg, {
|
||||||
|
logId: 'test',
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
assert.deepEqual(result.attachments, [msg.attachments?.[0]]);
|
||||||
|
assert.deepEqual(result.bodyAttachment, msg.bodyAttachment);
|
||||||
|
});
|
||||||
|
it('separates first body attachment out for all editHistory', () => {
|
||||||
|
const msg = composeMessage({
|
||||||
|
attachments: [
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'normal attachment',
|
||||||
|
contentType: IMAGE_JPEG,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'long message attachment 1',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'long message attachment 2',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
editHistory: [
|
||||||
|
{
|
||||||
|
timestamp: Date.now(),
|
||||||
|
received_at: Date.now(),
|
||||||
|
attachments: [
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'edit attachment',
|
||||||
|
contentType: IMAGE_JPEG,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'long message attachment 1',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'long message attachment 2',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timestamp: Date.now(),
|
||||||
|
received_at: Date.now(),
|
||||||
|
bodyAttachment: composeAttachment({
|
||||||
|
clientUuid: 'long message attachment already as bodyattachment',
|
||||||
|
contentType: LONG_MESSAGE,
|
||||||
|
}),
|
||||||
|
attachments: [
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'edit attachment 1',
|
||||||
|
contentType: IMAGE_JPEG,
|
||||||
|
}),
|
||||||
|
composeAttachment({
|
||||||
|
clientUuid: 'edit attachment 2',
|
||||||
|
contentType: IMAGE_JPEG,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const result = ensureBodyAttachmentsAreSeparated(msg, {
|
||||||
|
logId: 'test',
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(result.attachments, [msg.attachments?.[0]]);
|
||||||
|
assert.deepEqual(result.bodyAttachment, msg.attachments?.[1]);
|
||||||
|
assert.deepEqual(result.editHistory, [
|
||||||
|
{
|
||||||
|
...msg.editHistory![0],
|
||||||
|
attachments: [msg.editHistory![0].attachments![0]],
|
||||||
|
bodyAttachment: msg.editHistory![0].attachments![1],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
...msg.editHistory![1],
|
||||||
|
attachments: [
|
||||||
|
msg.editHistory![1].attachments![0],
|
||||||
|
msg.editHistory![1].attachments![1],
|
||||||
|
],
|
||||||
|
bodyAttachment: msg.editHistory![1].bodyAttachment,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
1
ts/textsecure/Types.d.ts
vendored
1
ts/textsecure/Types.d.ts
vendored
@@ -202,6 +202,7 @@ export type ProcessedGiftBadge = {
|
|||||||
|
|
||||||
export type ProcessedDataMessage = {
|
export type ProcessedDataMessage = {
|
||||||
body?: string;
|
body?: string;
|
||||||
|
bodyAttachment?: ProcessedAttachment;
|
||||||
attachments: ReadonlyArray<ProcessedAttachment>;
|
attachments: ReadonlyArray<ProcessedAttachment>;
|
||||||
groupV2?: ProcessedGroupV2Context;
|
groupV2?: ProcessedGroupV2Context;
|
||||||
flags: number;
|
flags: number;
|
||||||
|
@@ -33,6 +33,8 @@ import { isAciString } from '../util/isAciString';
|
|||||||
import { normalizeAci } from '../util/normalizeAci';
|
import { normalizeAci } from '../util/normalizeAci';
|
||||||
import { bytesToUuid } from '../util/uuidToBytes';
|
import { bytesToUuid } from '../util/uuidToBytes';
|
||||||
import { createName } from '../util/attachmentPath';
|
import { createName } from '../util/attachmentPath';
|
||||||
|
import { partitionBodyAndNormalAttachments } from '../types/Attachment';
|
||||||
|
import { isNotNil } from '../util/isNotNil';
|
||||||
|
|
||||||
const FLAGS = Proto.DataMessage.Flags;
|
const FLAGS = Proto.DataMessage.Flags;
|
||||||
export const ATTACHMENT_MAX = 32;
|
export const ATTACHMENT_MAX = 32;
|
||||||
@@ -316,14 +318,22 @@ export function processDataMessage(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const processedAttachments = message.attachments
|
||||||
|
?.map((attachment: Proto.IAttachmentPointer) => ({
|
||||||
|
...processAttachment(attachment),
|
||||||
|
downloadPath: doCreateName(),
|
||||||
|
}))
|
||||||
|
.filter(isNotNil);
|
||||||
|
|
||||||
|
const { bodyAttachment, attachments } = partitionBodyAndNormalAttachments(
|
||||||
|
{ attachments: processedAttachments ?? [] },
|
||||||
|
{ logId: `processDataMessage(${timestamp})` }
|
||||||
|
);
|
||||||
|
|
||||||
const result: ProcessedDataMessage = {
|
const result: ProcessedDataMessage = {
|
||||||
body: dropNull(message.body),
|
body: dropNull(message.body),
|
||||||
attachments: (message.attachments ?? []).map(
|
bodyAttachment,
|
||||||
(attachment: Proto.IAttachmentPointer) => ({
|
attachments,
|
||||||
...processAttachment(attachment),
|
|
||||||
downloadPath: doCreateName(),
|
|
||||||
})
|
|
||||||
),
|
|
||||||
groupV2: processGroupV2Context(message.groupV2),
|
groupV2: processGroupV2Context(message.groupV2),
|
||||||
flags: message.flags ?? 0,
|
flags: message.flags ?? 0,
|
||||||
expireTimer: DurationInSeconds.fromSeconds(message.expireTimer ?? 0),
|
expireTimer: DurationInSeconds.fromSeconds(message.expireTimer ?? 0),
|
||||||
|
@@ -9,11 +9,13 @@ import {
|
|||||||
isUndefined,
|
isUndefined,
|
||||||
isString,
|
isString,
|
||||||
omit,
|
omit,
|
||||||
|
partition,
|
||||||
} from 'lodash';
|
} from 'lodash';
|
||||||
import { blobToArrayBuffer } from 'blob-util';
|
import { blobToArrayBuffer } from 'blob-util';
|
||||||
|
|
||||||
import type { LinkPreviewType } from './message/LinkPreviews';
|
import type { LinkPreviewType } from './message/LinkPreviews';
|
||||||
import type { LoggerType } from './Logging';
|
import type { LoggerType } from './Logging';
|
||||||
|
import * as logging from '../logging/log';
|
||||||
import * as MIME from './MIME';
|
import * as MIME from './MIME';
|
||||||
import { toLogFormat } from './errors';
|
import { toLogFormat } from './errors';
|
||||||
import { SignalService } from '../protobuf';
|
import { SignalService } from '../protobuf';
|
||||||
@@ -1301,3 +1303,48 @@ export function getAttachmentIdForLogging(attachment: AttachmentType): string {
|
|||||||
}
|
}
|
||||||
return '[MissingDigest]';
|
return '[MissingDigest]';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// We now partition out the bodyAttachment on receipt, but older
|
||||||
|
// messages may still have a bodyAttachment in the normal attachments field
|
||||||
|
export function partitionBodyAndNormalAttachments<
|
||||||
|
T extends Pick<AttachmentType, 'contentType'>,
|
||||||
|
>(
|
||||||
|
{
|
||||||
|
attachments,
|
||||||
|
existingBodyAttachment,
|
||||||
|
}: {
|
||||||
|
attachments: ReadonlyArray<T>;
|
||||||
|
existingBodyAttachment?: T;
|
||||||
|
},
|
||||||
|
{ logId, logger = logging }: { logId: string; logger?: LoggerType }
|
||||||
|
): {
|
||||||
|
bodyAttachment: T | undefined;
|
||||||
|
attachments: Array<T>;
|
||||||
|
} {
|
||||||
|
const [bodyAttachments, normalAttachments] = partition(
|
||||||
|
attachments,
|
||||||
|
attachment => MIME.isLongMessage(attachment.contentType)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (bodyAttachments.length > 1) {
|
||||||
|
logger.warn(
|
||||||
|
`${logId}: Received more than one long message attachment, ` +
|
||||||
|
`dropping ${bodyAttachments.length - 1}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bodyAttachments.length > 0) {
|
||||||
|
if (existingBodyAttachment) {
|
||||||
|
logger.warn(`${logId}: there is already an existing body attachment`);
|
||||||
|
} else {
|
||||||
|
logger.info(
|
||||||
|
`${logId}: Moving a long message attachment to message.bodyAttachment`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
bodyAttachment: existingBodyAttachment ?? bodyAttachments[0],
|
||||||
|
attachments: normalAttachments,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
@@ -46,8 +46,8 @@ export const isVideo = (value: string): value is MIMEType =>
|
|||||||
// recognize them as file attachments.
|
// recognize them as file attachments.
|
||||||
export const isAudio = (value: string): value is MIMEType =>
|
export const isAudio = (value: string): value is MIMEType =>
|
||||||
Boolean(value) && value.startsWith('audio/') && !value.endsWith('aiff');
|
Boolean(value) && value.startsWith('audio/') && !value.endsWith('aiff');
|
||||||
export const isLongMessage = (value: unknown): value is MIMEType =>
|
export const isLongMessage = (value: string): value is MIMEType =>
|
||||||
value === LONG_MESSAGE;
|
value === LONG_MESSAGE;
|
||||||
export const supportsIncrementalMac = (value: unknown): boolean => {
|
export const supportsIncrementalMac = (value: string): boolean => {
|
||||||
return value === VIDEO_MP4;
|
return value === VIDEO_MP4;
|
||||||
};
|
};
|
||||||
|
@@ -122,9 +122,11 @@ function hasRequiredAttachmentDownloads(
|
|||||||
): boolean {
|
): boolean {
|
||||||
const attachments: ReadonlyArray<AttachmentType> = message.attachments || [];
|
const attachments: ReadonlyArray<AttachmentType> = message.attachments || [];
|
||||||
|
|
||||||
const hasLongMessageAttachments = attachments.some(attachment => {
|
const hasLongMessageAttachments =
|
||||||
return MIME.isLongMessage(attachment.contentType);
|
Boolean(message.bodyAttachment) ||
|
||||||
});
|
attachments.some(attachment => {
|
||||||
|
return MIME.isLongMessage(attachment.contentType);
|
||||||
|
});
|
||||||
|
|
||||||
if (hasLongMessageAttachments) {
|
if (hasLongMessageAttachments) {
|
||||||
return true;
|
return true;
|
||||||
|
@@ -1,8 +1,7 @@
|
|||||||
// Copyright 2020 Signal Messenger, LLC
|
// Copyright 2020 Signal Messenger, LLC
|
||||||
// SPDX-License-Identifier: AGPL-3.0-only
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
import { partition } from 'lodash';
|
import * as defaultLogger from '../logging/log';
|
||||||
import * as logger from '../logging/log';
|
|
||||||
import { isLongMessage } from '../types/MIME';
|
import { isLongMessage } from '../types/MIME';
|
||||||
import { getMessageIdForLogging } from './idForLogging';
|
import { getMessageIdForLogging } from './idForLogging';
|
||||||
import {
|
import {
|
||||||
@@ -24,6 +23,7 @@ import {
|
|||||||
getAttachmentSignatureSafe,
|
getAttachmentSignatureSafe,
|
||||||
isDownloading,
|
isDownloading,
|
||||||
isDownloaded,
|
isDownloaded,
|
||||||
|
partitionBodyAndNormalAttachments,
|
||||||
} from '../types/Attachment';
|
} from '../types/Attachment';
|
||||||
import type { StickerType } from '../types/Stickers';
|
import type { StickerType } from '../types/Stickers';
|
||||||
import type { LinkPreviewType } from '../types/message/LinkPreviews';
|
import type { LinkPreviewType } from '../types/message/LinkPreviews';
|
||||||
@@ -43,6 +43,7 @@ import {
|
|||||||
shouldUseAttachmentDownloadQueue,
|
shouldUseAttachmentDownloadQueue,
|
||||||
} from './attachmentDownloadQueue';
|
} from './attachmentDownloadQueue';
|
||||||
import { queueUpdateMessage } from './messageBatcher';
|
import { queueUpdateMessage } from './messageBatcher';
|
||||||
|
import type { LoggerType } from '../types/Logging';
|
||||||
|
|
||||||
export type MessageAttachmentsDownloadedType = {
|
export type MessageAttachmentsDownloadedType = {
|
||||||
bodyAttachment?: AttachmentType;
|
bodyAttachment?: AttachmentType;
|
||||||
@@ -56,7 +57,7 @@ export type MessageAttachmentsDownloadedType = {
|
|||||||
|
|
||||||
function getLogger(source: AttachmentDownloadSource) {
|
function getLogger(source: AttachmentDownloadSource) {
|
||||||
const verbose = source !== AttachmentDownloadSource.BACKUP_IMPORT;
|
const verbose = source !== AttachmentDownloadSource.BACKUP_IMPORT;
|
||||||
const log = verbose ? logger : { ...logger, info: () => null };
|
const log = verbose ? defaultLogger : { ...defaultLogger, info: () => null };
|
||||||
return log;
|
return log;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,7 +65,9 @@ export async function handleAttachmentDownloadsForNewMessage(
|
|||||||
message: MessageModel,
|
message: MessageModel,
|
||||||
conversation: ConversationModel
|
conversation: ConversationModel
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const idLog = `handleAttachmentDownloadsForNewMessage/${conversation.idForLogging()} ${getMessageIdForLogging(message.attributes)}`;
|
const logId =
|
||||||
|
`handleAttachmentDownloadsForNewMessage/${conversation.idForLogging()} ` +
|
||||||
|
`${getMessageIdForLogging(message.attributes)}`;
|
||||||
|
|
||||||
// Only queue attachments for downloads if this is a story (with additional logic), or
|
// Only queue attachments for downloads if this is a story (with additional logic), or
|
||||||
// if it's either an outgoing message or we've accepted the conversation
|
// if it's either an outgoing message or we've accepted the conversation
|
||||||
@@ -79,7 +82,7 @@ export async function handleAttachmentDownloadsForNewMessage(
|
|||||||
|
|
||||||
if (shouldQueueForDownload) {
|
if (shouldQueueForDownload) {
|
||||||
if (shouldUseAttachmentDownloadQueue()) {
|
if (shouldUseAttachmentDownloadQueue()) {
|
||||||
addToAttachmentDownloadQueue(idLog, message);
|
addToAttachmentDownloadQueue(logId, message);
|
||||||
} else {
|
} else {
|
||||||
await queueAttachmentDownloadsForMessage(message);
|
await queueAttachmentDownloadsForMessage(message);
|
||||||
}
|
}
|
||||||
@@ -117,28 +120,21 @@ export async function queueAttachmentDownloads(
|
|||||||
attachmentDigestForImmediate?: string;
|
attachmentDigestForImmediate?: string;
|
||||||
} = {}
|
} = {}
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
const attachmentsToQueue = message.get('attachments') || [];
|
|
||||||
const messageId = message.id;
|
const messageId = message.id;
|
||||||
const idForLogging = getMessageIdForLogging(message.attributes);
|
const idForLogging = getMessageIdForLogging(message.attributes);
|
||||||
|
|
||||||
let count = 0;
|
let count = 0;
|
||||||
|
|
||||||
const idLog = `queueAttachmentDownloads(${idForLogging}})`;
|
const logId = `queueAttachmentDownloads(${idForLogging}})`;
|
||||||
const log = getLogger(source);
|
const log = getLogger(source);
|
||||||
|
|
||||||
const [longMessageAttachments, normalAttachments] = partition(
|
message.set(
|
||||||
attachmentsToQueue,
|
ensureBodyAttachmentsAreSeparated(message.attributes, {
|
||||||
attachment => isLongMessage(attachment.contentType)
|
logId,
|
||||||
|
logger: log,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
if (longMessageAttachments.length > 1) {
|
|
||||||
log.error(`${idLog}: Received more than one long message attachment`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (longMessageAttachments.length > 0) {
|
|
||||||
message.set({ bodyAttachment: longMessageAttachments[0] });
|
|
||||||
}
|
|
||||||
|
|
||||||
const bodyAttachmentsToDownload = [
|
const bodyAttachmentsToDownload = [
|
||||||
message.get('bodyAttachment'),
|
message.get('bodyAttachment'),
|
||||||
...(message
|
...(message
|
||||||
@@ -151,7 +147,7 @@ export async function queueAttachmentDownloads(
|
|||||||
|
|
||||||
if (bodyAttachmentsToDownload.length) {
|
if (bodyAttachmentsToDownload.length) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Queueing ${bodyAttachmentsToDownload.length} long message attachment download`
|
`${logId}: Queueing ${bodyAttachmentsToDownload.length} long message attachment download`
|
||||||
);
|
);
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
bodyAttachmentsToDownload.map(attachment =>
|
bodyAttachmentsToDownload.map(attachment =>
|
||||||
@@ -169,16 +165,11 @@ export async function queueAttachmentDownloads(
|
|||||||
count += bodyAttachmentsToDownload.length;
|
count += bodyAttachmentsToDownload.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (normalAttachments.length > 0) {
|
|
||||||
log.info(
|
|
||||||
`${idLog}: Queueing ${normalAttachments.length} normal attachment downloads`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const { attachments, count: attachmentsCount } = await queueNormalAttachments(
|
const { attachments, count: attachmentsCount } = await queueNormalAttachments(
|
||||||
{
|
{
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
attachments: normalAttachments,
|
attachments: message.get('attachments'),
|
||||||
otherAttachments: message
|
otherAttachments: message
|
||||||
.get('editHistory')
|
.get('editHistory')
|
||||||
?.flatMap(x => x.attachments ?? []),
|
?.flatMap(x => x.attachments ?? []),
|
||||||
@@ -189,19 +180,23 @@ export async function queueAttachmentDownloads(
|
|||||||
attachmentDigestForImmediate,
|
attachmentDigestForImmediate,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
if (attachmentsCount > 0) {
|
if (attachmentsCount > 0) {
|
||||||
message.set({ attachments });
|
message.set({ attachments });
|
||||||
|
log.info(
|
||||||
|
`${logId}: Queueing ${attachmentsCount} normal attachment downloads`
|
||||||
|
);
|
||||||
}
|
}
|
||||||
count += attachmentsCount;
|
count += attachmentsCount;
|
||||||
|
|
||||||
const previewsToQueue = message.get('preview') || [];
|
const previewsToQueue = message.get('preview') || [];
|
||||||
if (previewsToQueue.length > 0) {
|
if (previewsToQueue.length > 0) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Queueing ${previewsToQueue.length} preview attachment downloads`
|
`${logId}: Queueing ${previewsToQueue.length} preview attachment downloads`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const { preview, count: previewCount } = await queuePreviews({
|
const { preview, count: previewCount } = await queuePreviews({
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
previews: previewsToQueue,
|
previews: previewsToQueue,
|
||||||
otherPreviews: message.get('editHistory')?.flatMap(x => x.preview ?? []),
|
otherPreviews: message.get('editHistory')?.flatMap(x => x.preview ?? []),
|
||||||
@@ -218,12 +213,12 @@ export async function queueAttachmentDownloads(
|
|||||||
const numQuoteAttachments = message.get('quote')?.attachments?.length ?? 0;
|
const numQuoteAttachments = message.get('quote')?.attachments?.length ?? 0;
|
||||||
if (numQuoteAttachments > 0) {
|
if (numQuoteAttachments > 0) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Queueing ${numQuoteAttachments} ` +
|
`${logId}: Queueing ${numQuoteAttachments} ` +
|
||||||
'quote attachment downloads'
|
'quote attachment downloads'
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const { quote, count: thumbnailCount } = await queueQuoteAttachments({
|
const { quote, count: thumbnailCount } = await queueQuoteAttachments({
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
quote: message.get('quote'),
|
quote: message.get('quote'),
|
||||||
otherQuotes:
|
otherQuotes:
|
||||||
@@ -244,7 +239,7 @@ export async function queueAttachmentDownloads(
|
|||||||
const contactsToQueue = message.get('contact') || [];
|
const contactsToQueue = message.get('contact') || [];
|
||||||
if (contactsToQueue.length > 0) {
|
if (contactsToQueue.length > 0) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Queueing ${contactsToQueue.length} contact attachment downloads`
|
`${logId}: Queueing ${contactsToQueue.length} contact attachment downloads`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const contact = await Promise.all(
|
const contact = await Promise.all(
|
||||||
@@ -254,7 +249,7 @@ export async function queueAttachmentDownloads(
|
|||||||
}
|
}
|
||||||
// We've already downloaded this!
|
// We've already downloaded this!
|
||||||
if (item.avatar.avatar.path) {
|
if (item.avatar.avatar.path) {
|
||||||
log.info(`${idLog}: Contact attachment already downloaded`);
|
log.info(`${logId}: Contact attachment already downloaded`);
|
||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,9 +275,9 @@ export async function queueAttachmentDownloads(
|
|||||||
|
|
||||||
let sticker = message.get('sticker');
|
let sticker = message.get('sticker');
|
||||||
if (sticker && sticker.data && sticker.data.path) {
|
if (sticker && sticker.data && sticker.data.path) {
|
||||||
log.info(`${idLog}: Sticker attachment already downloaded`);
|
log.info(`${logId}: Sticker attachment already downloaded`);
|
||||||
} else if (sticker) {
|
} else if (sticker) {
|
||||||
log.info(`${idLog}: Queueing sticker download`);
|
log.info(`${logId}: Queueing sticker download`);
|
||||||
count += 1;
|
count += 1;
|
||||||
const { packId, stickerId, packKey } = sticker;
|
const { packId, stickerId, packKey } = sticker;
|
||||||
|
|
||||||
@@ -294,7 +289,7 @@ export async function queueAttachmentDownloads(
|
|||||||
data = await copyStickerToAttachments(packId, stickerId);
|
data = await copyStickerToAttachments(packId, stickerId);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.error(
|
log.error(
|
||||||
`${idLog}: Problem copying sticker (${packId}, ${stickerId}) to attachments:`,
|
`${logId}: Problem copying sticker (${packId}, ${stickerId}) to attachments:`,
|
||||||
Errors.toLogFormat(error)
|
Errors.toLogFormat(error)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -311,7 +306,7 @@ export async function queueAttachmentDownloads(
|
|||||||
source,
|
source,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
log.error(`${idLog}: Sticker data was missing`);
|
log.error(`${logId}: Sticker data was missing`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const stickerRef = {
|
const stickerRef = {
|
||||||
@@ -341,12 +336,12 @@ export async function queueAttachmentDownloads(
|
|||||||
|
|
||||||
let editHistory = message.get('editHistory');
|
let editHistory = message.get('editHistory');
|
||||||
if (editHistory) {
|
if (editHistory) {
|
||||||
log.info(`${idLog}: Looping through ${editHistory.length} edits`);
|
log.info(`${logId}: Looping through ${editHistory.length} edits`);
|
||||||
editHistory = await Promise.all(
|
editHistory = await Promise.all(
|
||||||
editHistory.map(async edit => {
|
editHistory.map(async edit => {
|
||||||
const { attachments: editAttachments, count: editAttachmentsCount } =
|
const { attachments: editAttachments, count: editAttachmentsCount } =
|
||||||
await queueNormalAttachments({
|
await queueNormalAttachments({
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
attachments: edit.attachments,
|
attachments: edit.attachments,
|
||||||
otherAttachments: attachments,
|
otherAttachments: attachments,
|
||||||
@@ -358,14 +353,14 @@ export async function queueAttachmentDownloads(
|
|||||||
count += editAttachmentsCount;
|
count += editAttachmentsCount;
|
||||||
if (editAttachmentsCount !== 0) {
|
if (editAttachmentsCount !== 0) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Queueing ${editAttachmentsCount} normal attachment ` +
|
`${logId}: Queueing ${editAttachmentsCount} normal attachment ` +
|
||||||
`downloads (edited:${edit.timestamp})`
|
`downloads (edited:${edit.timestamp})`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { preview: editPreview, count: editPreviewCount } =
|
const { preview: editPreview, count: editPreviewCount } =
|
||||||
await queuePreviews({
|
await queuePreviews({
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
previews: edit.preview,
|
previews: edit.preview,
|
||||||
otherPreviews: preview,
|
otherPreviews: preview,
|
||||||
@@ -377,7 +372,7 @@ export async function queueAttachmentDownloads(
|
|||||||
count += editPreviewCount;
|
count += editPreviewCount;
|
||||||
if (editPreviewCount !== 0) {
|
if (editPreviewCount !== 0) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Queueing ${editPreviewCount} preview attachment ` +
|
`${logId}: Queueing ${editPreviewCount} preview attachment ` +
|
||||||
`downloads (edited:${edit.timestamp})`
|
`downloads (edited:${edit.timestamp})`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -396,13 +391,13 @@ export async function queueAttachmentDownloads(
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
log.info(`${idLog}: Queued ${count} total attachment downloads`);
|
log.info(`${logId}: Queued ${count} total attachment downloads`);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function queueNormalAttachments({
|
export async function queueNormalAttachments({
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
attachments = [],
|
attachments = [],
|
||||||
otherAttachments,
|
otherAttachments,
|
||||||
@@ -412,7 +407,7 @@ export async function queueNormalAttachments({
|
|||||||
source,
|
source,
|
||||||
attachmentDigestForImmediate,
|
attachmentDigestForImmediate,
|
||||||
}: {
|
}: {
|
||||||
idLog: string;
|
logId: string;
|
||||||
messageId: string;
|
messageId: string;
|
||||||
attachments: MessageAttributesType['attachments'];
|
attachments: MessageAttributesType['attachments'];
|
||||||
otherAttachments: MessageAttributesType['attachments'];
|
otherAttachments: MessageAttributesType['attachments'];
|
||||||
@@ -446,9 +441,16 @@ export async function queueNormalAttachments({
|
|||||||
if (!attachment) {
|
if (!attachment) {
|
||||||
return attachment;
|
return attachment;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isLongMessage(attachment.contentType)) {
|
||||||
|
throw new Error(
|
||||||
|
`${logId}: queueNormalAttachments passed long-message attachment`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// We've already downloaded this!
|
// We've already downloaded this!
|
||||||
if (isDownloaded(attachment)) {
|
if (isDownloaded(attachment)) {
|
||||||
log.info(`${idLog}: Normal attachment already downloaded`);
|
log.info(`${logId}: Normal attachment already downloaded`);
|
||||||
return attachment;
|
return attachment;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -463,7 +465,7 @@ export async function queueNormalAttachments({
|
|||||||
(isDownloading(existingAttachment) || isDownloaded(existingAttachment))
|
(isDownloading(existingAttachment) || isDownloaded(existingAttachment))
|
||||||
) {
|
) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Normal attachment already downloaded in other attachments. Replacing`
|
`${logId}: Normal attachment already downloaded in other attachments. Replacing`
|
||||||
);
|
);
|
||||||
// Incrementing count so that we update the message's fields downstream
|
// Incrementing count so that we update the message's fields downstream
|
||||||
count += 1;
|
count += 1;
|
||||||
@@ -511,7 +513,7 @@ function getLinkPreviewSignature(preview: LinkPreviewType): string | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function queuePreviews({
|
async function queuePreviews({
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
previews = [],
|
previews = [],
|
||||||
otherPreviews,
|
otherPreviews,
|
||||||
@@ -520,7 +522,7 @@ async function queuePreviews({
|
|||||||
urgency,
|
urgency,
|
||||||
source,
|
source,
|
||||||
}: {
|
}: {
|
||||||
idLog: string;
|
logId: string;
|
||||||
messageId: string;
|
messageId: string;
|
||||||
previews: MessageAttributesType['preview'];
|
previews: MessageAttributesType['preview'];
|
||||||
otherPreviews: MessageAttributesType['preview'];
|
otherPreviews: MessageAttributesType['preview'];
|
||||||
@@ -550,7 +552,7 @@ async function queuePreviews({
|
|||||||
}
|
}
|
||||||
// We've already downloaded this!
|
// We've already downloaded this!
|
||||||
if (isDownloaded(item.image)) {
|
if (isDownloaded(item.image)) {
|
||||||
log.info(`${idLog}: Preview attachment already downloaded`);
|
log.info(`${logId}: Preview attachment already downloaded`);
|
||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
const signature = getLinkPreviewSignature(item);
|
const signature = getLinkPreviewSignature(item);
|
||||||
@@ -564,7 +566,7 @@ async function queuePreviews({
|
|||||||
(isDownloading(existingPreview.image) ||
|
(isDownloading(existingPreview.image) ||
|
||||||
isDownloaded(existingPreview.image))
|
isDownloaded(existingPreview.image))
|
||||||
) {
|
) {
|
||||||
log.info(`${idLog}: Preview already downloaded elsewhere. Replacing`);
|
log.info(`${logId}: Preview already downloaded elsewhere. Replacing`);
|
||||||
// Incrementing count so that we update the message's fields downstream
|
// Incrementing count so that we update the message's fields downstream
|
||||||
count += 1;
|
count += 1;
|
||||||
return existingPreview;
|
return existingPreview;
|
||||||
@@ -607,7 +609,7 @@ function getQuoteThumbnailSignature(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function queueQuoteAttachments({
|
async function queueQuoteAttachments({
|
||||||
idLog,
|
logId,
|
||||||
messageId,
|
messageId,
|
||||||
quote,
|
quote,
|
||||||
otherQuotes,
|
otherQuotes,
|
||||||
@@ -616,7 +618,7 @@ async function queueQuoteAttachments({
|
|||||||
urgency,
|
urgency,
|
||||||
source,
|
source,
|
||||||
}: {
|
}: {
|
||||||
idLog: string;
|
logId: string;
|
||||||
messageId: string;
|
messageId: string;
|
||||||
quote: QuotedMessageType | undefined;
|
quote: QuotedMessageType | undefined;
|
||||||
otherQuotes: ReadonlyArray<QuotedMessageType>;
|
otherQuotes: ReadonlyArray<QuotedMessageType>;
|
||||||
@@ -663,7 +665,7 @@ async function queueQuoteAttachments({
|
|||||||
}
|
}
|
||||||
// We've already downloaded this!
|
// We've already downloaded this!
|
||||||
if (isDownloaded(item.thumbnail)) {
|
if (isDownloaded(item.thumbnail)) {
|
||||||
log.info(`${idLog}: Quote attachment already downloaded`);
|
log.info(`${logId}: Quote attachment already downloaded`);
|
||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -679,7 +681,7 @@ async function queueQuoteAttachments({
|
|||||||
isDownloaded(existingThumbnail))
|
isDownloaded(existingThumbnail))
|
||||||
) {
|
) {
|
||||||
log.info(
|
log.info(
|
||||||
`${idLog}: Preview already downloaded elsewhere. Replacing`
|
`${logId}: Preview already downloaded elsewhere. Replacing`
|
||||||
);
|
);
|
||||||
// Incrementing count so that we update the message's fields downstream
|
// Incrementing count so that we update the message's fields downstream
|
||||||
count += 1;
|
count += 1;
|
||||||
@@ -708,3 +710,42 @@ async function queueQuoteAttachments({
|
|||||||
count,
|
count,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function ensureBodyAttachmentsAreSeparated(
|
||||||
|
messageAttributes: MessageAttributesType,
|
||||||
|
{ logId, logger = defaultLogger }: { logId: string; logger?: LoggerType }
|
||||||
|
): {
|
||||||
|
bodyAttachment: AttachmentType | undefined;
|
||||||
|
attachments: Array<AttachmentType>;
|
||||||
|
editHistory: Array<EditHistoryType> | undefined;
|
||||||
|
} {
|
||||||
|
const { bodyAttachment, attachments } = partitionBodyAndNormalAttachments(
|
||||||
|
{
|
||||||
|
attachments: messageAttributes.attachments ?? [],
|
||||||
|
existingBodyAttachment: messageAttributes.bodyAttachment,
|
||||||
|
},
|
||||||
|
{ logId, logger }
|
||||||
|
);
|
||||||
|
|
||||||
|
const updatedEditHistory = messageAttributes.editHistory?.map(edit => {
|
||||||
|
return {
|
||||||
|
...edit,
|
||||||
|
...partitionBodyAndNormalAttachments(
|
||||||
|
{
|
||||||
|
attachments: edit.attachments ?? [],
|
||||||
|
existingBodyAttachment: edit.bodyAttachment,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
logId: `${logId}/editHistory(${edit.timestamp})`,
|
||||||
|
logger,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
bodyAttachment: bodyAttachment ?? messageAttributes.bodyAttachment,
|
||||||
|
attachments,
|
||||||
|
editHistory: updatedEditHistory,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Reference in New Issue
Block a user