diff --git a/ts/model-types.d.ts b/ts/model-types.d.ts index 8d96a17b0..f9af9140e 100644 --- a/ts/model-types.d.ts +++ b/ts/model-types.d.ts @@ -168,6 +168,9 @@ type MessageType = | 'verified-change' | 'message-request-response-event'; +// Note: when adding a property that is likely to be set across many messages, +// consider adding a database column as well and updating `MESSAGE_COLUMNS` +// in `ts/sql/Server.ts` export type MessageAttributesType = { bodyAttachment?: AttachmentType; bodyRanges?: ReadonlyArray; diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index 2b1faa049..37480f626 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -62,6 +62,7 @@ import type { ClientOnlyReadableInterface, ClientOnlyWritableInterface, } from './Interface'; +import { hydrateMessage } from './hydration'; import type { MessageAttributesType } from '../model-types'; import type { AttachmentDownloadJobType } from '../types/AttachmentDownload'; @@ -545,7 +546,7 @@ function handleSearchMessageJSON( messages: Array ): Array { return messages.map(message => { - const parsedMessage = JSON.parse(message.json); + const parsedMessage = hydrateMessage(message); assertDev( message.ftsSnippet ?? typeof message.mentionStart === 'number', 'Neither ftsSnippet nor matching mention returned from message search' @@ -553,14 +554,12 @@ function handleSearchMessageJSON( const snippet = message.ftsSnippet ?? generateSnippetAroundMention({ - body: parsedMessage.body, + body: parsedMessage.body || '', mentionStart: message.mentionStart ?? 0, mentionLength: message.mentionLength ?? 1, }); return { - json: message.json, - // Empty array is a default value. `message.json` has the real field bodyRanges: [], ...parsedMessage, @@ -734,7 +733,7 @@ async function removeMessages( function handleMessageJSON( messages: Array ): Array { - return messages.map(message => JSON.parse(message.json)); + return messages.map(message => hydrateMessage(message)); } async function getNewerMessagesByConversation( diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 3894d3191..5dfc2b649 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -115,8 +115,76 @@ export type StoredItemType = { value: BytesToStrings; }; export type MessageType = MessageAttributesType; + +// See: ts/sql/Interface.ts +// +// When adding a new column: +// +// - Make sure the name matches the one in `MessageAttributeTypes` +// - Update `hydrateMessage` +// +export const MESSAGE_COLUMNS = [ + 'json', + + 'id', + 'body', + 'conversationId', + 'expirationStartTimestamp', + 'expireTimer', + 'hasAttachments', + 'hasFileAttachments', + 'hasVisualMediaAttachments', + 'isChangeCreatedByUs', + 'isErased', + 'isViewOnce', + 'mentionsMe', + 'received_at', + 'received_at_ms', + 'schemaVersion', + 'serverGuid', + 'sent_at', + 'source', + 'sourceServiceId', + 'sourceDevice', + 'storyId', + 'type', + 'readStatus', + 'seenStatus', + 'serverTimestamp', + 'timestamp', + 'unidentifiedDeliveryReceived', +] as const; + export type MessageTypeUnhydrated = { json: string; + + id: string; + body: string | null; + conversationId: string | null; + expirationStartTimestamp: number | null; + expireTimer: number | null; + hasAttachments: 0 | 1 | null; + hasFileAttachments: 0 | 1 | null; + hasVisualMediaAttachments: 0 | 1 | null; + isChangeCreatedByUs: 0 | 1 | null; + isErased: 0 | 1 | null; + isViewOnce: 0 | 1 | null; + mentionsMe: 0 | 1 | null; + received_at: number | null; + received_at_ms: number | null; + schemaVersion: number | null; + serverGuid: string | null; + sent_at: number | null; + source: string | null; + sourceServiceId: string | null; + sourceDevice: number | null; + serverTimestamp: number | null; + storyId: string | null; + type: string; + timestamp: number | null; + readStatus: number | null; + seenStatus: number | null; + unidentifiedDeliveryReceived: 0 | 1 | null; }; export type PreKeyIdType = `${ServiceIdString}:${number}`; @@ -147,9 +215,7 @@ export type StoredPreKeyType = PreKeyType & { privateKey: string; publicKey: string; }; -export type ServerSearchResultMessageType = { - json: string; - +export type ServerSearchResultMessageType = MessageTypeUnhydrated & { // If the FTS matches text in message.body, snippet will be populated ftsSnippet: string | null; @@ -159,7 +225,6 @@ export type ServerSearchResultMessageType = { mentionLength: number | null; }; export type ClientSearchResultMessageType = MessageType & { - json: string; bodyRanges: ReadonlyArray; snippet: string; }; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 98ca9c446..41b32e33e 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -55,13 +55,7 @@ import { isNormalNumber } from '../util/isNormalNumber'; import { isNotNil } from '../util/isNotNil'; import { parseIntOrThrow } from '../util/parseIntOrThrow'; import { updateSchema } from './migrations'; -import type { - ArrayQuery, - EmptyQuery, - JSONRows, - Query, - QueryFragment, -} from './util'; +import type { ArrayQuery, EmptyQuery, JSONRows, Query } from './util'; import { batchMultiVarQuery, bulkAdd, @@ -80,7 +74,9 @@ import { sqlConstant, sqlFragment, sqlJoin, + QueryFragment, } from './util'; +import { hydrateMessage } from './hydration'; import { getAttachmentCiphertextLength } from '../AttachmentCrypto'; import { SeenStatus } from '../MessageSeenStatus'; @@ -181,7 +177,7 @@ import type { UnprocessedUpdateType, WritableDB, } from './Interface'; -import { AttachmentDownloadSource } from './Interface'; +import { AttachmentDownloadSource, MESSAGE_COLUMNS } from './Interface'; import { _removeAllCallLinks, beginDeleteAllCallLinks, @@ -582,6 +578,10 @@ export function prepare | Record>( return result; } +const MESSAGE_COLUMNS_FRAGMENTS = MESSAGE_COLUMNS.map( + column => new QueryFragment(column, []) +); + function rowToConversation(row: ConversationRow): ConversationType { const { expireTimerVersion } = row; const parsedJson = JSON.parse(row.json); @@ -603,6 +603,7 @@ function rowToConversation(row: ConversationRow): ConversationType { profileLastFetchedAt, }; } + function rowToSticker(row: StickerRow): StickerType { return { ...row, @@ -1938,6 +1939,10 @@ function searchMessages( .run({ conversationId, limit }); } + const prefixedColumns = sqlJoin( + MESSAGE_COLUMNS_FRAGMENTS.map(name => sqlFragment`messages.${name}`) + ); + // The `MATCH` is necessary in order to for `snippet()` helper function to // give us the right results. We can't call `snippet()` in the query above // because it would bloat the temporary table with text data and we want @@ -1945,9 +1950,7 @@ function searchMessages( const ftsFragment = sqlFragment` SELECT messages.rowid, - messages.json, - messages.sent_at, - messages.received_at, + ${prefixedColumns}, snippet(messages_fts, -1, ${SNIPPET_LEFT_PLACEHOLDER}, ${SNIPPET_RIGHT_PLACEHOLDER}, ${SNIPPET_TRUNCATION_PLACEHOLDER}, 10) AS ftsSnippet FROM tmp_filtered_results INNER JOIN messages_fts @@ -1966,6 +1969,12 @@ function searchMessages( const [sqlQuery, params] = sql`${ftsFragment};`; result = writable.prepare(sqlQuery).all(params); } else { + const coalescedColumns = MESSAGE_COLUMNS_FRAGMENTS.map( + name => sqlFragment` + COALESCE(messages.${name}, ftsResults.${name}) AS ${name} + ` + ); + // If contactServiceIdsMatchingQuery is not empty, we due an OUTER JOIN // between: // 1) the messages that mention at least one of @@ -1978,9 +1987,7 @@ function searchMessages( const [sqlQuery, params] = sql` SELECT messages.rowid as rowid, - COALESCE(messages.json, ftsResults.json) as json, - COALESCE(messages.sent_at, ftsResults.sent_at) as sent_at, - COALESCE(messages.received_at, ftsResults.received_at) as received_at, + ${sqlJoin(coalescedColumns)}, ftsResults.ftsSnippet, mentionAci, start as mentionStart, @@ -2082,7 +2089,9 @@ export function getMostRecentAddressableMessages( limit = 5 ): Array { const [query, parameters] = sql` - SELECT json FROM messages + SELECT + ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} + FROM messages INDEXED BY messages_by_date_addressable WHERE conversationId IS ${conversationId} AND @@ -2093,7 +2102,7 @@ export function getMostRecentAddressableMessages( const rows = db.prepare(query).all(parameters); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } export function getMostRecentAddressableNondisappearingMessages( @@ -2102,7 +2111,9 @@ export function getMostRecentAddressableNondisappearingMessages( limit = 5 ): Array { const [query, parameters] = sql` - SELECT json FROM messages + SELECT + ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} + FROM messages INDEXED BY messages_by_date_addressable_nondisappearing WHERE expireTimer IS NULL AND @@ -2114,7 +2125,7 @@ export function getMostRecentAddressableNondisappearingMessages( const rows = db.prepare(query).all(parameters); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } export function removeSyncTaskById(db: WritableDB, id: string): void { @@ -2248,7 +2259,6 @@ export function saveMessage( const { body, conversationId, - groupV2Change, hasAttachments, hasFileAttachments, hasVisualMediaAttachments, @@ -2257,6 +2267,7 @@ export function saveMessage( isViewOnce, mentionsMe, received_at, + received_at_ms, schemaVersion, sent_at, serverGuid, @@ -2264,13 +2275,22 @@ export function saveMessage( sourceServiceId, sourceDevice, storyId, + timestamp, type, readStatus, expireTimer, expirationStartTimestamp, - attachments, + seenStatus: originalSeenStatus, + serverTimestamp, + unidentifiedDeliveryReceived, + + ...json } = data; - let { seenStatus } = data; + + // Extracted separately since we store this field in JSON + const { attachments, groupV2Change } = data; + + let seenStatus = originalSeenStatus; if (attachments) { strictAssert( @@ -2313,6 +2333,7 @@ export function saveMessage( isViewOnce: isViewOnce ? 1 : 0, mentionsMe: mentionsMe ? 1 : 0, received_at: received_at || null, + received_at_ms: received_at_ms || null, schemaVersion: schemaVersion || 0, serverGuid: serverGuid || null, sent_at: sent_at || null, @@ -2321,43 +2342,22 @@ export function saveMessage( sourceDevice: sourceDevice || null, storyId: storyId || null, type: type || null, + timestamp: timestamp ?? 0, readStatus: readStatus ?? null, seenStatus: seenStatus ?? SeenStatus.NotApplicable, - }; + serverTimestamp: serverTimestamp ?? null, + unidentifiedDeliveryReceived: unidentifiedDeliveryReceived ? 1 : 0, + } satisfies Omit; if (id && !forceSave) { prepare( db, ` UPDATE messages SET - id = $id, - json = $json, - - body = $body, - conversationId = $conversationId, - expirationStartTimestamp = $expirationStartTimestamp, - expireTimer = $expireTimer, - hasAttachments = $hasAttachments, - hasFileAttachments = $hasFileAttachments, - hasVisualMediaAttachments = $hasVisualMediaAttachments, - isChangeCreatedByUs = $isChangeCreatedByUs, - isErased = $isErased, - isViewOnce = $isViewOnce, - mentionsMe = $mentionsMe, - received_at = $received_at, - schemaVersion = $schemaVersion, - serverGuid = $serverGuid, - sent_at = $sent_at, - source = $source, - sourceServiceId = $sourceServiceId, - sourceDevice = $sourceDevice, - storyId = $storyId, - type = $type, - readStatus = $readStatus, - seenStatus = $seenStatus + ${MESSAGE_COLUMNS.map(name => `${name} = $${name}`).join(', ')} WHERE id = $id; ` - ).run({ ...payloadWithoutJson, json: objectToJSON(data) }); + ).run({ ...payloadWithoutJson, json: objectToJSON(json) }); if (jobToInsert) { insertJob(db, jobToInsert); @@ -2366,79 +2366,28 @@ export function saveMessage( return id; } - const toCreate = { - ...data, - id: id || generateMessageId(data.received_at).id, - }; + const createdId = id || generateMessageId(data.received_at).id; prepare( db, ` INSERT INTO messages ( - id, - json, - - body, - conversationId, - expirationStartTimestamp, - expireTimer, - hasAttachments, - hasFileAttachments, - hasVisualMediaAttachments, - isChangeCreatedByUs, - isErased, - isViewOnce, - mentionsMe, - received_at, - schemaVersion, - serverGuid, - sent_at, - source, - sourceServiceId, - sourceDevice, - storyId, - type, - readStatus, - seenStatus - ) values ( - $id, - $json, - - $body, - $conversationId, - $expirationStartTimestamp, - $expireTimer, - $hasAttachments, - $hasFileAttachments, - $hasVisualMediaAttachments, - $isChangeCreatedByUs, - $isErased, - $isViewOnce, - $mentionsMe, - $received_at, - $schemaVersion, - $serverGuid, - $sent_at, - $source, - $sourceServiceId, - $sourceDevice, - $storyId, - $type, - $readStatus, - $seenStatus + ${MESSAGE_COLUMNS.join(', ')} + ) VALUES ( + ${MESSAGE_COLUMNS.map(name => `$${name}`).join(', ')} ); ` ).run({ ...payloadWithoutJson, - id: toCreate.id, - json: objectToJSON(toCreate), + id: createdId, + json: objectToJSON(json), }); if (jobToInsert) { insertJob(db, jobToInsert); } - return toCreate.id; + return createdId; } function saveMessages( @@ -2507,7 +2456,13 @@ export function getMessageById( id: string ): MessageType | undefined { const row = db - .prepare('SELECT json FROM messages WHERE id = $id;') + .prepare( + ` + SELECT ${MESSAGE_COLUMNS.join(', ')} + FROM messages + WHERE id = $id; + ` + ) .get({ id, }); @@ -2516,7 +2471,7 @@ export function getMessageById( return undefined; } - return jsonToObject(row.json); + return hydrateMessage(row); } function getMessagesById( @@ -2528,22 +2483,30 @@ function getMessagesById( messageIds, (batch: ReadonlyArray): Array => { const query = db.prepare( - `SELECT json FROM messages WHERE id IN (${Array(batch.length) - .fill('?') - .join(',')});` + ` + SELECT ${MESSAGE_COLUMNS.join(', ')} + FROM messages + WHERE id IN ( + ${Array(batch.length).fill('?').join(',')} + );` ); - const rows: JSONRows = query.all(batch); - return rows.map(row => jsonToObject(row.json)); + const rows: Array = query.all(batch); + return rows.map(row => hydrateMessage(row)); } ); } function _getAllMessages(db: ReadableDB): Array { - const rows: JSONRows = db - .prepare('SELECT json FROM messages ORDER BY id ASC;') + const rows: Array = db + .prepare( + ` + SELECT ${MESSAGE_COLUMNS.join(', ')} + FROM messages ORDER BY id ASC + ` + ) .all(); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } function _removeAllMessages(db: WritableDB): void { db.exec(` @@ -2574,10 +2537,10 @@ function getMessageBySender( sent_at: number; } ): MessageType | undefined { - const rows: JSONRows = prepare( + const rows: Array = prepare( db, ` - SELECT json FROM messages WHERE + SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE (source = $source OR sourceServiceId = $sourceServiceId) AND sourceDevice = $sourceDevice AND sent_at = $sent_at @@ -2603,7 +2566,7 @@ function getMessageBySender( return undefined; } - return jsonToObject(rows[0].json); + return hydrateMessage(rows[0]); } export function _storyIdPredicate( @@ -2667,7 +2630,9 @@ function getUnreadByConversationAndMarkRead( db.prepare(updateExpirationQuery).run(updateExpirationParams); const [selectQuery, selectParams] = sql` - SELECT id, json FROM messages + SELECT + ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} + FROM messages WHERE conversationId = ${conversationId} AND seenStatus = ${SeenStatus.Unseen} AND @@ -2701,7 +2666,7 @@ function getUnreadByConversationAndMarkRead( db.prepare(updateStatusQuery).run(updateStatusParams); return rows.map(row => { - const json = jsonToObject(row.json); + const json = hydrateMessage(row); return { originalReadStatus: json.readStatus, readStatus: ReadStatus.Read, @@ -2929,7 +2894,10 @@ function getRecentStoryReplies( }; const createQuery = (timeFilter: QueryFragment): QueryFragment => sqlFragment` - SELECT json FROM messages WHERE + SELECT + ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} + FROM messages + WHERE (${messageId} IS NULL OR id IS NOT ${messageId}) AND isStory IS 0 AND storyId IS ${storyId} AND @@ -2940,9 +2908,9 @@ function getRecentStoryReplies( `; const template = sqlFragment` - SELECT first.json FROM (${createQuery(timeFilters.first)}) as first + SELECT first.* FROM (${createQuery(timeFilters.first)}) as first UNION ALL - SELECT second.json FROM (${createQuery(timeFilters.second)}) as second + SELECT second.* FROM (${createQuery(timeFilters.second)}) as second `; const [query, params] = sql`${template} LIMIT ${limit}`; @@ -2988,7 +2956,9 @@ function getAdjacentMessagesByConversation( requireFileAttachments; const createQuery = (timeFilter: QueryFragment): QueryFragment => sqlFragment` - SELECT json FROM messages WHERE + SELECT + ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} + FROM messages WHERE conversationId = ${conversationId} AND ${ requireDifferentMessage @@ -3014,15 +2984,15 @@ function getAdjacentMessagesByConversation( `; let template = sqlFragment` - SELECT first.json FROM (${createQuery(timeFilters.first)}) as first + SELECT first.* FROM (${createQuery(timeFilters.first)}) as first UNION ALL - SELECT second.json FROM (${createQuery(timeFilters.second)}) as second + SELECT second.* FROM (${createQuery(timeFilters.second)}) as second `; // See `filterValidAttachments` in ts/state/ducks/lightbox.ts if (requireVisualMediaAttachments) { template = sqlFragment` - SELECT json + SELECT messages.* FROM (${template}) as messages WHERE ( @@ -3037,7 +3007,7 @@ function getAdjacentMessagesByConversation( `; } else if (requireFileAttachments) { template = sqlFragment` - SELECT json + SELECT messages.* FROM (${template}) as messages WHERE ( @@ -3086,7 +3056,7 @@ function getAllStories( } ): GetAllStoriesResultType { const [storiesQuery, storiesParams] = sql` - SELECT json, id + SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} FROM messages WHERE isStory = 1 AND @@ -3094,10 +3064,7 @@ function getAllStories( (${sourceServiceId} IS NULL OR sourceServiceId IS ${sourceServiceId}) ORDER BY received_at ASC, sent_at ASC; `; - const rows: ReadonlyArray<{ - id: string; - json: string; - }> = db.prepare(storiesQuery).all(storiesParams); + const rows = db.prepare(storiesQuery).all(storiesParams); const [repliesQuery, repliesParams] = sql` SELECT DISTINCT storyId @@ -3126,7 +3093,7 @@ function getAllStories( ); return rows.map(row => ({ - ...jsonToObject(row.json), + ...hydrateMessage(row), hasReplies: Boolean(repliesLookup.has(row.id)), hasRepliesFromSelf: Boolean(repliesFromSelfLookup.has(row.id)), })); @@ -3301,7 +3268,7 @@ function getLastConversationActivity( const row = prepare( db, ` - SELECT json FROM messages + SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages INDEXED BY messages_activity WHERE conversationId IS $conversationId AND @@ -3320,7 +3287,7 @@ function getLastConversationActivity( return undefined; } - return jsonToObject(row.json); + return hydrateMessage(row); } function getLastConversationPreview( db: ReadableDB, @@ -3332,19 +3299,15 @@ function getLastConversationPreview( includeStoryReplies: boolean; } ): MessageType | undefined { - type Row = Readonly<{ - json: string; - }>; - const index = includeStoryReplies ? 'messages_preview' : 'messages_preview_without_story'; - const row: Row | undefined = prepare( + const row: MessageTypeUnhydrated | undefined = prepare( db, ` - SELECT json FROM ( - SELECT json, expiresAt FROM messages + SELECT ${MESSAGE_COLUMNS.join(', ')}, expiresAt FROM ( + SELECT ${MESSAGE_COLUMNS.join(', ')}, expiresAt FROM messages INDEXED BY ${index} WHERE conversationId IS $conversationId AND @@ -3361,7 +3324,7 @@ function getLastConversationPreview( now: Date.now(), }); - return row ? jsonToObject(row.json) : undefined; + return row ? hydrateMessage(row) : undefined; } function getConversationMessageStats( @@ -3400,7 +3363,7 @@ function getLastConversationMessage( const row = db .prepare( ` - SELECT json FROM messages WHERE + SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE conversationId = $conversationId ORDER BY received_at DESC, sent_at DESC LIMIT 1; @@ -3414,7 +3377,7 @@ function getLastConversationMessage( return undefined; } - return jsonToObject(row.json); + return hydrateMessage(row); } function getOldestUnseenMessageForConversation( @@ -3730,7 +3693,7 @@ function getCallHistoryMessageByCallId( } ): MessageType | undefined { const [query, params] = sql` - SELECT json + SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} FROM messages WHERE conversationId = ${options.conversationId} AND type = 'call-history' @@ -3740,7 +3703,7 @@ function getCallHistoryMessageByCallId( if (row == null) { return; } - return jsonToObject(row.json); + return hydrateMessage(row); } function getCallHistory( @@ -4524,45 +4487,57 @@ function getMessagesBySentAt( db: ReadableDB, sentAt: number ): Array { + // Make sure to preserve order of columns + const editedColumns = MESSAGE_COLUMNS_FRAGMENTS.map(name => { + if (name.fragment === 'received_at' || name.fragment === 'sent_at') { + return name; + } + return sqlFragment`messages.${name}`; + }); + const [query, params] = sql` - SELECT messages.json, received_at, sent_at FROM edited_messages + SELECT ${sqlJoin(editedColumns)} + FROM edited_messages INNER JOIN messages ON messages.id = edited_messages.messageId WHERE edited_messages.sentAt = ${sentAt} UNION - SELECT json, received_at, sent_at FROM messages + SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)} + FROM messages WHERE sent_at = ${sentAt} ORDER BY messages.received_at DESC, messages.sent_at DESC; `; const rows = db.prepare(query).all(params); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } function getExpiredMessages(db: ReadableDB): Array { const now = Date.now(); - const rows: JSONRows = db + const rows: Array = db .prepare( ` - SELECT json FROM messages WHERE + SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}, expiresAt + FROM messages + WHERE expiresAt <= $now ORDER BY expiresAt ASC; ` ) .all({ now }); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } function getMessagesUnexpectedlyMissingExpirationStartTimestamp( db: ReadableDB ): Array { - const rows: JSONRows = db + const rows: Array = db .prepare( ` - SELECT json FROM messages + SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages INDEXED BY messages_unexpectedly_missing_expiration_start_timestamp WHERE expireTimer > 0 AND @@ -4579,7 +4554,7 @@ function getMessagesUnexpectedlyMissingExpirationStartTimestamp( ) .all(); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } function getSoonestMessageExpiry(db: ReadableDB): undefined | number { @@ -4607,7 +4582,7 @@ function getNextTapToViewMessageTimestampToAgeOut( const row = db .prepare( ` - SELECT json FROM messages + SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE -- we want this query to use the messages_view_once index rather than received_at likelihood(isViewOnce = 1, 0.01) @@ -4621,7 +4596,7 @@ function getNextTapToViewMessageTimestampToAgeOut( if (!row) { return undefined; } - const data = jsonToObject(row.json); + const data = hydrateMessage(row); const result = data.received_at_ms; return isNormalNumber(result) ? result : undefined; } @@ -4630,16 +4605,16 @@ function getTapToViewMessagesNeedingErase( db: ReadableDB, maxTimestamp: number ): Array { - const rows: JSONRows = db + const rows: Array = db .prepare( ` - SELECT json + SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE isViewOnce = 1 AND (isErased IS NULL OR isErased != 1) AND ( - IFNULL(json ->> '$.received_at_ms', 0) <= $maxTimestamp + IFNULL(received_at_ms, 0) <= $maxTimestamp ) ` ) @@ -4647,7 +4622,7 @@ function getTapToViewMessagesNeedingErase( maxTimestamp, }); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } const MAX_UNPROCESSED_ATTEMPTS = 10; @@ -6716,10 +6691,10 @@ function getMessagesNeedingUpgrade( limit: number, { maxVersion }: { maxVersion: number } ): Array { - const rows: JSONRows = db + const rows: Array = db .prepare( ` - SELECT json + SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE (schemaVersion IS NULL OR schemaVersion < $maxVersion) AND @@ -6736,7 +6711,7 @@ function getMessagesNeedingUpgrade( limit, }); - return rows.map(row => jsonToObject(row.json)); + return rows.map(row => hydrateMessage(row)); } // Exported for tests @@ -7023,12 +6998,14 @@ function pageMessages( rowids, (batch: ReadonlyArray): Array => { const query = writable.prepare( - `SELECT json FROM messages WHERE rowid IN (${Array(batch.length) - .fill('?') - .join(',')});` + ` + SELECT ${MESSAGE_COLUMNS.join(', ')} + FROM messages + WHERE rowid IN (${Array(batch.length).fill('?').join(',')}); + ` ); - const rows: JSONRows = query.all(batch); - return rows.map(row => jsonToObject(row.json)); + const rows: Array = query.all(batch); + return rows.map(row => hydrateMessage(row)); } ); @@ -7446,11 +7423,14 @@ function getUnreadEditedMessagesAndMarkRead( } ): GetUnreadByConversationAndMarkReadResultType { return db.transaction(() => { + const editedColumns = MESSAGE_COLUMNS_FRAGMENTS.filter( + name => name.fragment !== 'sent_at' && name.fragment !== 'readStatus' + ).map(name => sqlFragment`messages.${name}`); + const [selectQuery, selectParams] = sql` SELECT - messages.id, - messages.json, - edited_messages.sentAt, + ${sqlJoin(editedColumns)}, + edited_messages.sentAt as sent_at, edited_messages.readStatus FROM edited_messages JOIN messages @@ -7481,7 +7461,7 @@ function getUnreadEditedMessagesAndMarkRead( } return rows.map(row => { - const json = jsonToObject(row.json); + const json = hydrateMessage(row); return { originalReadStatus: row.readStatus, readStatus: ReadStatus.Read, @@ -7494,8 +7474,6 @@ function getUnreadEditedMessagesAndMarkRead( 'sourceServiceId', 'type', ]), - // Use the edited message timestamp - sent_at: row.sentAt, }; }); })(); diff --git a/ts/sql/hydration.ts b/ts/sql/hydration.ts new file mode 100644 index 000000000..162109351 --- /dev/null +++ b/ts/sql/hydration.ts @@ -0,0 +1,88 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import type { ReadStatus } from '../messages/MessageReadStatus'; +import type { SeenStatus } from '../MessageSeenStatus'; +import type { ServiceIdString } from '../types/ServiceId'; +import { dropNull } from '../util/dropNull'; + +/* eslint-disable camelcase */ + +import type { + MessageTypeUnhydrated, + MessageType, + MESSAGE_COLUMNS, +} from './Interface'; + +function toBoolean(value: number | null): boolean | undefined { + if (value == null) { + return undefined; + } + return value === 1; +} + +export function hydrateMessage(row: MessageTypeUnhydrated): MessageType { + const { + json, + id, + body, + conversationId, + expirationStartTimestamp, + expireTimer, + hasAttachments, + hasFileAttachments, + hasVisualMediaAttachments, + isErased, + isViewOnce, + mentionsMe, + received_at, + received_at_ms, + schemaVersion, + serverGuid, + sent_at, + source, + sourceServiceId, + sourceDevice, + storyId, + type, + readStatus, + seenStatus, + timestamp, + serverTimestamp, + unidentifiedDeliveryReceived, + } = row; + + return { + ...(JSON.parse(json) as Omit< + MessageType, + (typeof MESSAGE_COLUMNS)[number] + >), + + id, + body: dropNull(body), + conversationId: conversationId || '', + expirationStartTimestamp: dropNull(expirationStartTimestamp), + expireTimer: dropNull(expireTimer) as MessageType['expireTimer'], + hasAttachments: toBoolean(hasAttachments), + hasFileAttachments: toBoolean(hasFileAttachments), + hasVisualMediaAttachments: toBoolean(hasVisualMediaAttachments), + isErased: toBoolean(isErased), + isViewOnce: toBoolean(isViewOnce), + mentionsMe: toBoolean(mentionsMe), + received_at: received_at || 0, + received_at_ms: dropNull(received_at_ms), + schemaVersion: dropNull(schemaVersion), + serverGuid: dropNull(serverGuid), + sent_at: sent_at || 0, + source: dropNull(source), + sourceServiceId: dropNull(sourceServiceId) as ServiceIdString | undefined, + sourceDevice: dropNull(sourceDevice), + storyId: dropNull(storyId), + type: type as MessageType['type'], + readStatus: readStatus == null ? undefined : (readStatus as ReadStatus), + seenStatus: seenStatus == null ? undefined : (seenStatus as SeenStatus), + timestamp: timestamp || 0, + serverTimestamp: dropNull(serverTimestamp), + unidentifiedDeliveryReceived: toBoolean(unidentifiedDeliveryReceived), + }; +} diff --git a/ts/sql/migrations/1270-normalize-messages.ts b/ts/sql/migrations/1270-normalize-messages.ts new file mode 100644 index 000000000..495215c42 --- /dev/null +++ b/ts/sql/migrations/1270-normalize-messages.ts @@ -0,0 +1,53 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +import type { Database } from '@signalapp/better-sqlite3'; +import type { LoggerType } from '../../types/Logging'; +import { sql } from '../util'; + +export const version = 1270; + +export function updateToSchemaVersion1270( + currentVersion: number, + db: Database, + logger: LoggerType +): void { + if (currentVersion >= 1270) { + return; + } + + db.transaction(() => { + const [query] = sql` + ALTER TABLE messages + ADD COLUMN timestamp INTEGER; + ALTER TABLE messages + ADD COLUMN received_at_ms INTEGER; + ALTER TABLE messages + ADD COLUMN unidentifiedDeliveryReceived INTEGER; + ALTER TABLE messages + ADD COLUMN serverTimestamp INTEGER; + + ALTER TABLE messages + RENAME COLUMN source TO legacySource; + ALTER TABLE messages + ADD COLUMN source TEXT; + + UPDATE messages SET + timestamp = json_extract(json, '$.timestamp'), + received_at_ms = json_extract(json, '$.received_at_ms'), + unidentifiedDeliveryReceived = + json_extract(json, '$.unidentifiedDeliveryReceived'), + serverTimestamp = + json_extract(json, '$.serverTimestamp'), + source = IFNULL(json_extract(json, '$.source'), '+' || legacySource); + + ALTER TABLE messages + DROP COLUMN legacySource; + `; + + db.exec(query); + + db.pragma('user_version = 1270'); + })(); + + logger.info('updateToSchemaVersion1270: success!'); +} diff --git a/ts/sql/migrations/51-centralize-conversation-jobs.ts b/ts/sql/migrations/51-centralize-conversation-jobs.ts index 902cf4682..f9d730d33 100644 --- a/ts/sql/migrations/51-centralize-conversation-jobs.ts +++ b/ts/sql/migrations/51-centralize-conversation-jobs.ts @@ -4,7 +4,7 @@ import type { LoggerType } from '../../types/Logging'; import { isRecord } from '../../util/isRecord'; import type { WritableDB } from '../Interface'; -import { getJobsInQueue, getMessageById, insertJob } from '../Server'; +import { getJobsInQueue, insertJob } from '../Server'; export default function updateToSchemaVersion51( currentVersion: number, @@ -24,6 +24,10 @@ export default function updateToSchemaVersion51( const reactionsJobs = getJobsInQueue(db, 'reactions'); deleteJobsInQueue.run({ queueType: 'reactions' }); + const getMessageById = db.prepare( + 'SELECT conversationId FROM messages WHERE id IS ?' + ); + reactionsJobs.forEach(job => { const { data, id } = job; @@ -42,7 +46,7 @@ export default function updateToSchemaVersion51( return; } - const message = getMessageById(db, messageId); + const message = getMessageById.get(messageId); if (!message) { logger.warn( `updateToSchemaVersion51: Unable to find message for reaction job ${id}` diff --git a/ts/sql/migrations/78-merge-receipt-jobs.ts b/ts/sql/migrations/78-merge-receipt-jobs.ts index 04c2b4d40..d59fc3d22 100644 --- a/ts/sql/migrations/78-merge-receipt-jobs.ts +++ b/ts/sql/migrations/78-merge-receipt-jobs.ts @@ -4,7 +4,7 @@ import type { LoggerType } from '../../types/Logging'; import { isRecord } from '../../util/isRecord'; import type { WritableDB } from '../Interface'; -import { getJobsInQueue, getMessageById, insertJob } from '../Server'; +import { getJobsInQueue, insertJob } from '../Server'; export default function updateToSchemaVersion78( currentVersion: number, @@ -41,6 +41,10 @@ export default function updateToSchemaVersion78( }, ]; + const getMessageById = db.prepare( + 'SELECT conversationId FROM messages WHERE id IS ?' + ); + for (const queue of queues) { const prevJobs = getJobsInQueue(db, queue.queueType); deleteJobsInQueue.run({ queueType: queue.queueType }); @@ -62,7 +66,7 @@ export default function updateToSchemaVersion78( return; } - const message = getMessageById(db, messageId); + const message = getMessageById.get(messageId); if (!message) { logger.warn( `updateToSchemaVersion78: Unable to find message for ${queue.queueType} job ${id}` diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index 6e559882a..e2f10b885 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -102,10 +102,11 @@ import { updateToSchemaVersion1220 } from './1220-blob-sessions'; import { updateToSchemaVersion1230 } from './1230-call-links-admin-key-index'; import { updateToSchemaVersion1240 } from './1240-defunct-call-links-table'; import { updateToSchemaVersion1250 } from './1250-defunct-call-links-storage'; +import { updateToSchemaVersion1260 } from './1260-sync-tasks-rowid'; import { - updateToSchemaVersion1260, + updateToSchemaVersion1270, version as MAX_VERSION, -} from './1260-sync-tasks-rowid'; +} from './1270-normalize-messages'; import { DataWriter } from '../Server'; function updateToSchemaVersion1( @@ -2078,6 +2079,7 @@ export const SCHEMA_VERSIONS = [ updateToSchemaVersion1240, updateToSchemaVersion1250, updateToSchemaVersion1260, + updateToSchemaVersion1270, ]; export class DBVersionFromFutureError extends Error { diff --git a/ts/sql/util.ts b/ts/sql/util.ts index 8b98bbea4..ac0462f2e 100644 --- a/ts/sql/util.ts +++ b/ts/sql/util.ts @@ -11,7 +11,8 @@ export type ArrayQuery = Array>; export type Query = { [key: string]: null | number | bigint | string | Uint8Array; }; -export type JSONRows = Array<{ readonly json: string }>; +export type JSONRow = Readonly<{ json: string }>; +export type JSONRows = Array; export type TableType = | 'attachment_downloads' diff --git a/ts/test-electron/backup/attachments_test.ts b/ts/test-electron/backup/attachments_test.ts index 5be7f70ea..afb1b2826 100644 --- a/ts/test-electron/backup/attachments_test.ts +++ b/ts/test-electron/backup/attachments_test.ts @@ -166,8 +166,8 @@ describe('backup/attachments', () => { // path & iv will not be roundtripped [ composeMessage(1, { - hasAttachments: 1, - hasVisualMediaAttachments: 1, + hasAttachments: true, + hasVisualMediaAttachments: true, attachments: [ omit(longMessageAttachment, NON_ROUNDTRIPPED_FIELDS), omit(normalAttachment, NON_ROUNDTRIPPED_FIELDS), @@ -284,8 +284,8 @@ describe('backup/attachments', () => { // path & iv will not be roundtripped [ composeMessage(1, { - hasAttachments: 1, - hasVisualMediaAttachments: 1, + hasAttachments: true, + hasVisualMediaAttachments: true, attachments: [ omit(attachment1, NON_ROUNDTRIPPED_FIELDS), omit(attachment2, NON_ROUNDTRIPPED_FIELDS), @@ -307,8 +307,8 @@ describe('backup/attachments', () => { ], [ composeMessage(1, { - hasAttachments: 1, - hasVisualMediaAttachments: 1, + hasAttachments: true, + hasVisualMediaAttachments: true, // path, iv, and uploadTimestamp will not be roundtripped, // but there will be a backupLocator @@ -341,7 +341,7 @@ describe('backup/attachments', () => { ], [ composeMessage(1, { - hasAttachments: 1, + hasAttachments: true, attachments: [ { ...omit(attachment, NON_ROUNDTRIPPED_BACKUP_LOCATOR_FIELDS), @@ -604,8 +604,8 @@ describe('backup/attachments', () => { [ { ...existingMessage, - hasAttachments: 1, - hasVisualMediaAttachments: 1, + hasAttachments: true, + hasVisualMediaAttachments: true, attachments: [ { ...omit( diff --git a/ts/test-electron/backup/helpers.ts b/ts/test-electron/backup/helpers.ts index b34aab532..212b67bca 100644 --- a/ts/test-electron/backup/helpers.ts +++ b/ts/test-electron/backup/helpers.ts @@ -110,10 +110,20 @@ function sortAndNormalize( // Get rid of unserializable `undefined` values. return JSON.parse( JSON.stringify({ - // Migration defaults - hasAttachments: 0, + // Defaults + hasAttachments: false, + hasFileAttachments: false, + hasVisualMediaAttachments: false, + isErased: false, + isViewOnce: false, + mentionsMe: false, + seenStatus: 0, + readStatus: 0, + unidentifiedDeliveryReceived: false, + + // Drop more `undefined` values + ...JSON.parse(JSON.stringify(rest)), - ...rest, conversationId: mapConvoId(conversationId), reactions: reactions?.map(({ fromId, ...restOfReaction }) => { return { diff --git a/ts/test-electron/backup/non_bubble_test.ts b/ts/test-electron/backup/non_bubble_test.ts index 25315568d..9b9fc5b69 100644 --- a/ts/test-electron/backup/non_bubble_test.ts +++ b/ts/test-electron/backup/non_bubble_test.ts @@ -76,7 +76,6 @@ describe('backup/non-bubble messages', () => { flags: Proto.DataMessage.Flags.END_SESSION, attachments: [], contact: [], - hasAttachments: 0, }, ]); }); diff --git a/ts/test-node/sql/migration_1060_test.ts b/ts/test-node/sql/migration_1060_test.ts index 4b48f9c5d..6b19d7934 100644 --- a/ts/test-node/sql/migration_1060_test.ts +++ b/ts/test-node/sql/migration_1060_test.ts @@ -6,11 +6,11 @@ import { v4 as generateGuid } from 'uuid'; import { dequeueOldestSyncTasks, - getMostRecentAddressableMessages, removeSyncTaskById, saveSyncTasks, } from '../../sql/Server'; -import type { WritableDB } from '../../sql/Interface'; +import type { WritableDB, ReadableDB, MessageType } from '../../sql/Interface'; +import { sql, jsonToObject } from '../../sql/util'; import { insertData, updateToVersion, createDB } from './helpers'; import { MAX_SYNC_TASK_ATTEMPTS } from '../../util/syncTasks.types'; import { WEEK } from '../../util/durations'; @@ -20,6 +20,27 @@ import type { SyncTaskType } from '../../util/syncTasks'; /* eslint-disable camelcase */ +// Snapshot before: 1270 +export function getMostRecentAddressableMessages( + db: ReadableDB, + conversationId: string, + limit = 5 +): Array { + const [query, parameters] = sql` + SELECT json FROM messages + INDEXED BY messages_by_date_addressable + WHERE + conversationId IS ${conversationId} AND + isAddressableMessage = 1 + ORDER BY received_at DESC, sent_at DESC + LIMIT ${limit}; + `; + + const rows = db.prepare(query).all(parameters); + + return rows.map(row => jsonToObject(row.json)); +} + function generateMessage(json: MessageAttributesType) { const { conversationId, received_at, sent_at, type } = json; diff --git a/ts/test-node/sql/migration_1080_test.ts b/ts/test-node/sql/migration_1080_test.ts index 2fcb60335..8632a0042 100644 --- a/ts/test-node/sql/migration_1080_test.ts +++ b/ts/test-node/sql/migration_1080_test.ts @@ -4,8 +4,8 @@ import { assert } from 'chai'; import { v4 as generateGuid } from 'uuid'; -import type { WritableDB } from '../../sql/Interface'; -import { getMostRecentAddressableNondisappearingMessages } from '../../sql/Server'; +import type { WritableDB, ReadableDB, MessageType } from '../../sql/Interface'; +import { sql, jsonToObject } from '../../sql/util'; import { createDB, insertData, updateToVersion } from './helpers'; import type { MessageAttributesType } from '../../model-types'; @@ -26,6 +26,28 @@ function generateMessage(json: MessageAttributesType) { }; } +// Snapshot before: 1270 +export function getMostRecentAddressableNondisappearingMessages( + db: ReadableDB, + conversationId: string, + limit = 5 +): Array { + const [query, parameters] = sql` + SELECT json FROM messages + INDEXED BY messages_by_date_addressable_nondisappearing + WHERE + expireTimer IS NULL AND + conversationId IS ${conversationId} AND + isAddressableMessage = 1 + ORDER BY received_at DESC, sent_at DESC + LIMIT ${limit}; + `; + + const rows = db.prepare(query).all(parameters); + + return rows.map(row => jsonToObject(row.json)); +} + describe('SQL/updateToSchemaVersion1080', () => { let db: WritableDB; beforeEach(() => { diff --git a/ts/test-node/sql/migrations_test.ts b/ts/test-node/sql/migrations_test.ts index f054b0d17..4e0e1b7ce 100644 --- a/ts/test-node/sql/migrations_test.ts +++ b/ts/test-node/sql/migrations_test.ts @@ -1351,10 +1351,8 @@ describe('SQL migrations test', () => { db.exec( ` INSERT INTO messages - (id, json) - VALUES ('${MESSAGE_ID_1}', '${JSON.stringify({ - conversationId: CONVERSATION_ID_1, - })}') + (id, conversationId) + VALUES ('${MESSAGE_ID_1}', '${CONVERSATION_ID_1}'); ` ); @@ -2482,10 +2480,8 @@ describe('SQL migrations test', () => { db.exec( ` INSERT INTO messages - (id, json) - VALUES ('${MESSAGE_ID_1}', '${JSON.stringify({ - conversationId: CONVERSATION_ID_1, - })}') + (id, conversationId) + VALUES ('${MESSAGE_ID_1}', '${CONVERSATION_ID_1}'); ` ); diff --git a/ts/test-node/types/Message2_test.ts b/ts/test-node/types/Message2_test.ts index 3f717a75c..a4e11b428 100644 --- a/ts/test-node/types/Message2_test.ts +++ b/ts/test-node/types/Message2_test.ts @@ -126,11 +126,9 @@ describe('Message', () => { it('should initialize schema version to zero', () => { const input = getDefaultMessage({ body: 'Imagine there is no heaven…', - attachments: [], }); const expected = getDefaultMessage({ body: 'Imagine there is no heaven…', - attachments: [], schemaVersion: 0, }); @@ -203,7 +201,6 @@ describe('Message', () => { hasVisualMediaAttachments: undefined, hasFileAttachments: undefined, schemaVersion: Message.CURRENT_SCHEMA_VERSION, - contact: [], }); const expectedAttachmentData = 'It’s easy if you try'; @@ -655,7 +652,6 @@ describe('Message', () => { }); const expected = getDefaultMessage({ body: 'hey there!', - contact: [], }); const result = await upgradeVersion(message, getDefaultContext()); assert.deepEqual(result, expected); @@ -848,7 +844,6 @@ describe('Message', () => { key: 'key', }, ], - contact: [], }); const result = await Message.upgradeSchema(message, { ...getDefaultContext(), diff --git a/ts/types/Message2.ts b/ts/types/Message2.ts index c7a38e689..92bb7a1c9 100644 --- a/ts/types/Message2.ts +++ b/ts/types/Message2.ts @@ -285,6 +285,10 @@ export const _mapAttachments = message: MessageAttributesType, context: ContextType ): Promise => { + if (!message.attachments?.length) { + return message; + } + const upgradeWithContext = esbuildAnonymize((attachment: AttachmentType) => upgradeAttachment(attachment, context, message) ); @@ -356,6 +360,10 @@ export const _mapContact = message: MessageAttributesType, context: ContextType ): Promise => { + if (!message.contact?.length) { + return message; + } + const upgradeWithContext = esbuildAnonymize( (contact: EmbeddedContactType) => upgradeContact(contact, context, message) @@ -501,23 +509,25 @@ const toVersion10 = _withSchemaVersion({ schemaVersion: 10, upgrade: async (message, context) => { const processPreviews = _mapPreviewAttachments(migrateDataToFileSystem); - const processSticker = async ( - stickerMessage: MessageAttributesType, - stickerContext: ContextType - ): Promise => { - const { sticker } = stickerMessage; - if (!sticker || !sticker.data || !sticker.data.data) { - return stickerMessage; - } + const processSticker = esbuildAnonymize( + async ( + stickerMessage: MessageAttributesType, + stickerContext: ContextType + ): Promise => { + const { sticker } = stickerMessage; + if (!sticker || !sticker.data || !sticker.data.data) { + return stickerMessage; + } - return { - ...stickerMessage, - sticker: { - ...sticker, - data: await migrateDataToFileSystem(sticker.data, stickerContext), - }, - }; - }; + return { + ...stickerMessage, + sticker: { + ...sticker, + data: await migrateDataToFileSystem(sticker.data, stickerContext), + }, + }; + } + ); const previewProcessed = await processPreviews(message, context); const stickerProcessed = await processSticker(previewProcessed, context);