diff --git a/about_preload.js b/about_preload.js index 1d7d969b3..2f9fa80f7 100644 --- a/about_preload.js +++ b/about_preload.js @@ -25,4 +25,4 @@ window.closeAbout = () => ipcRenderer.send('close-about'); window.i18n = i18n.setup(locale, localeMessages); -require('./ts/logging/set_up_renderer_logging'); +require('./ts/logging/set_up_renderer_logging').initialize(); diff --git a/debug_log_preload.js b/debug_log_preload.js index 6f8c18fa9..c63f7a552 100644 --- a/debug_log_preload.js +++ b/debug_log_preload.js @@ -32,7 +32,7 @@ window.getEnvironment = getEnvironment; window.Backbone = require('backbone'); require('./ts/backbone/views/whisper_view'); require('./ts/backbone/views/toast_view'); -require('./ts/logging/set_up_renderer_logging'); +require('./ts/logging/set_up_renderer_logging').initialize(); window.closeDebugLog = () => ipcRenderer.send('close-debug-log'); window.Backbone = require('backbone'); diff --git a/js/expiring_messages.js b/js/expiring_messages.js index 5618ca5f2..fb181937a 100644 --- a/js/expiring_messages.js +++ b/js/expiring_messages.js @@ -18,32 +18,38 @@ MessageCollection: Whisper.MessageCollection, }); - await Promise.all( - messages.map(async fromDB => { - const message = MessageController.register(fromDB.id, fromDB); + const messageIds = []; + const inMemoryMessages = []; + const messageCleanup = []; - window.log.info('Message expired', { - sentAt: message.get('sent_at'), - }); + messages.forEach(dbMessage => { + const message = MessageController.register(dbMessage.id, dbMessage); + messageIds.push(message.id); + inMemoryMessages.push(message); + messageCleanup.push(message.cleanup()); + }); - // We delete after the trigger to allow the conversation time to process - // the expiration before the message is removed from the database. - await window.Signal.Data.removeMessage(message.id, { - Message: Whisper.Message, - }); + // We delete after the trigger to allow the conversation time to process + // the expiration before the message is removed from the database. + await window.Signal.Data.removeMessages(messageIds); + await Promise.all(messageCleanup); - Whisper.events.trigger( - 'messageExpired', - message.id, - message.conversationId - ); + inMemoryMessages.forEach(message => { + window.log.info('Message expired', { + sentAt: message.get('sent_at'), + }); - const conversation = message.getConversation(); - if (conversation) { - conversation.trigger('expired', message); - } - }) - ); + Whisper.events.trigger( + 'messageExpired', + message.id, + message.conversationId + ); + + const conversation = message.getConversation(); + if (conversation) { + conversation.trigger('expired', message); + } + }); } catch (error) { window.log.error( 'destroyExpiredMessages: Error deleting expired messages', diff --git a/js/expiring_tap_to_view_messages.js b/js/expiring_tap_to_view_messages.js index 52ff70ea8..32cc027db 100644 --- a/js/expiring_tap_to_view_messages.js +++ b/js/expiring_tap_to_view_messages.js @@ -58,7 +58,9 @@ return; } - const nextCheck = toAgeOut.get('received_at') + THIRTY_DAYS; + const receivedAt = + toAgeOut.get('received_at_ms') || toAgeOut.get('received_at'); + const nextCheck = receivedAt + THIRTY_DAYS; Whisper.TapToViewMessagesListener.nextCheck = nextCheck; window.log.info( diff --git a/js/message_controller.js b/js/message_controller.js index c42933b50..43f2bd85f 100644 --- a/js/message_controller.js +++ b/js/message_controller.js @@ -6,6 +6,8 @@ window.Whisper = window.Whisper || {}; const messageLookup = Object.create(null); + const msgIDsBySender = new Map(); + const msgIDsBySentAt = new Map(); const SECOND = 1000; const MINUTE = SECOND * 60; @@ -31,10 +33,18 @@ timestamp: Date.now(), }; + msgIDsBySentAt.set(message.get('sent_at'), id); + msgIDsBySender.set(message.getSenderIdentifier(), id); + return message; } function unregister(id) { + const { message } = messageLookup[id] || {}; + if (message) { + msgIDsBySender.delete(message.getSenderIdentifier()); + msgIDsBySentAt.delete(message.get('sent_at')); + } delete messageLookup[id]; } @@ -50,7 +60,7 @@ now - timestamp > FIVE_MINUTES && (!conversation || !conversation.messageCollection.length) ) { - delete messageLookup[message.id]; + unregister(message.id); } } } @@ -60,6 +70,22 @@ return existing && existing.message ? existing.message : null; } + function findBySentAt(sentAt) { + const id = msgIDsBySentAt.get(sentAt); + if (!id) { + return null; + } + return getById(id); + } + + function findBySender(sender) { + const id = msgIDsBySender.get(sender); + if (!id) { + return null; + } + return getById(id); + } + function _get() { return messageLookup; } @@ -70,6 +96,8 @@ register, unregister, cleanup, + findBySender, + findBySentAt, getById, _get, }; diff --git a/libtextsecure/test/_test.js b/libtextsecure/test/_test.js index 6d5a73d42..54b1b7faf 100644 --- a/libtextsecure/test/_test.js +++ b/libtextsecure/test/_test.js @@ -67,3 +67,16 @@ window.Whisper.events = { on() {}, trigger() {}, }; + +before(async () => { + try { + window.log.info('Initializing SQL in renderer'); + await window.sqlInitializer.initialize(); + window.log.info('SQL initialized in renderer'); + } catch (err) { + window.log.error( + 'SQL failed to initialize', + err && err.stack ? err.stack : err + ); + } +}); diff --git a/main.js b/main.js index 7126fec46..d2d1e9d1a 100644 --- a/main.js +++ b/main.js @@ -19,6 +19,8 @@ const electron = require('electron'); const packageJson = require('./package.json'); const GlobalErrors = require('./app/global_errors'); const { setup: setupSpellChecker } = require('./app/spell_check'); +const { redactAll } = require('./js/modules/privacy'); +const removeUserConfig = require('./app/user_config').remove; GlobalErrors.addHandler(); @@ -30,6 +32,7 @@ const getRealPath = pify(fs.realpath); const { app, BrowserWindow, + clipboard, dialog, ipcMain: ipc, Menu, @@ -1058,12 +1061,37 @@ app.on('ready', async () => { loadingWindow.loadURL(prepareURL([__dirname, 'loading.html'])); }); - const success = await sqlInitPromise; - - if (!success) { + try { + await sqlInitPromise; + } catch (error) { console.log('sql.initialize was unsuccessful; returning early'); + const buttonIndex = dialog.showMessageBoxSync({ + buttons: [ + locale.messages.copyErrorAndQuit.message, + locale.messages.deleteAndRestart.message, + ], + defaultId: 0, + detail: redactAll(error.stack), + message: locale.messages.databaseError.message, + noLink: true, + type: 'error', + }); + + if (buttonIndex === 0) { + clipboard.writeText( + `Database startup error:\n\n${redactAll(error.stack)}` + ); + } else { + await sql.removeDB(); + removeUserConfig(); + app.relaunch(); + } + + app.exit(1); + return; } + // eslint-disable-next-line more/no-then appStartInitialSpellcheckSetting = await getSpellCheckSetting(); await sqlChannels.initialize(); @@ -1075,10 +1103,10 @@ app.on('ready', async () => { await sql.removeIndexedDBFiles(); await sql.removeItemById(IDB_KEY); } - } catch (error) { + } catch (err) { console.log( '(ready event handler) error deleting IndexedDB:', - error && error.stack ? error.stack : error + err && err.stack ? err.stack : err ); } @@ -1113,10 +1141,10 @@ app.on('ready', async () => { try { await attachments.clearTempPath(userDataPath); - } catch (error) { + } catch (err) { logger.error( 'main/ready: Error deleting temp dir:', - error && error.stack ? error.stack : error + err && err.stack ? err.stack : err ); } await attachmentChannel.initialize({ @@ -1458,6 +1486,17 @@ ipc.on('locale-data', event => { event.returnValue = locale.messages; }); +// Used once to initialize SQL in the renderer process +ipc.once('user-config-key', event => { + // eslint-disable-next-line no-param-reassign + event.returnValue = userConfig.get('key'); +}); + +ipc.on('get-user-data-path', event => { + // eslint-disable-next-line no-param-reassign + event.returnValue = app.getPath('userData'); +}); + function getDataFromMainWindow(name, callback) { ipc.once(`get-success-${name}`, (_event, error, value) => callback(error, value) diff --git a/permissions_popup_preload.js b/permissions_popup_preload.js index 63730f95d..ee3ccd02b 100644 --- a/permissions_popup_preload.js +++ b/permissions_popup_preload.js @@ -49,7 +49,7 @@ window.subscribeToSystemThemeChange = fn => { }); }; -require('./ts/logging/set_up_renderer_logging'); +require('./ts/logging/set_up_renderer_logging').initialize(); window.closePermissionsPopup = () => ipcRenderer.send('close-permissions-popup'); diff --git a/preload.js b/preload.js index 66e8dc143..fc3888b03 100644 --- a/preload.js +++ b/preload.js @@ -22,6 +22,8 @@ try { const { app } = remote; const { nativeTheme } = remote.require('electron'); + window.sqlInitializer = require('./ts/sql/initialize'); + window.PROTO_ROOT = 'protos'; const config = require('url').parse(window.location.toString(), true).query; @@ -400,7 +402,7 @@ try { // We pull these dependencies in now, from here, because they have Node.js dependencies - require('./ts/logging/set_up_renderer_logging'); + require('./ts/logging/set_up_renderer_logging').initialize(); if (config.proxyUrl) { window.log.info('Using provided proxy url'); diff --git a/settings_preload.js b/settings_preload.js index 9a4d21aa6..5a138750e 100644 --- a/settings_preload.js +++ b/settings_preload.js @@ -129,4 +129,4 @@ function makeSetter(name) { window.Backbone = require('backbone'); require('./ts/backbone/views/whisper_view'); require('./ts/backbone/views/toast_view'); -require('./ts/logging/set_up_renderer_logging'); +require('./ts/logging/set_up_renderer_logging').initialize(); diff --git a/test/_test.js b/test/_test.js index 12a61bf8f..ffe8e5578 100644 --- a/test/_test.js +++ b/test/_test.js @@ -76,6 +76,16 @@ function deleteIndexedDB() { /* Delete the database before running any tests */ before(async () => { await deleteIndexedDB(); + try { + window.log.info('Initializing SQL in renderer'); + await window.sqlInitializer.initialize(); + window.log.info('SQL initialized in renderer'); + } catch (err) { + window.log.error( + 'SQL failed to initialize', + err && err.stack ? err.stack : err + ); + } await window.Signal.Data.removeAll(); await window.storage.fetch(); }); diff --git a/ts/background.ts b/ts/background.ts index a3c2f8a3e..30e5580b0 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -3,8 +3,19 @@ import { DataMessageClass } from './textsecure.d'; import { WhatIsThis } from './window.d'; +import { assert } from './util/assert'; export async function startApp(): Promise { + try { + window.log.info('Initializing SQL in renderer'); + await window.sqlInitializer.initialize(); + window.log.info('SQL initialized in renderer'); + } catch (err) { + window.log.error( + 'SQL failed to initialize', + err && err.stack ? err.stack : err + ); + } const eventHandlerQueue = new window.PQueue({ concurrency: 1, timeout: 1000 * 60 * 2, @@ -1615,6 +1626,9 @@ export async function startApp(): Promise { let connectCount = 0; let connecting = false; async function connect(firstRun?: boolean) { + window.receivedAtCounter = + window.storage.get('lastReceivedAtCounter') || Date.now(); + if (connecting) { window.log.warn('connect already running', { connectCount }); return; @@ -2038,7 +2052,7 @@ export async function startApp(): Promise { logger: window.log, }); - let interval: NodeJS.Timer | null = setInterval(() => { + let interval: NodeJS.Timer | null = setInterval(async () => { const view = window.owsDesktopApp.appView; if (view) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -2046,6 +2060,24 @@ export async function startApp(): Promise { interval = null; view.onEmpty(); window.logAppLoadedEvent(); + const attachmentDownloadQueue = window.attachmentDownloadQueue || []; + const THREE_DAYS_AGO = Date.now() - 3600 * 72 * 1000; + const MAX_ATTACHMENT_MSGS_TO_DOWNLOAD = 250; + const attachmentsToDownload = attachmentDownloadQueue.filter( + (message, index) => + index <= MAX_ATTACHMENT_MSGS_TO_DOWNLOAD || + message.getReceivedAt() < THREE_DAYS_AGO + ); + window.log.info( + 'Downloading recent attachments of total attachments', + attachmentsToDownload.length, + attachmentDownloadQueue.length + ); + await Promise.all( + attachmentsToDownload.map(message => + message.queueAttachmentDownloads() + ) + ); } }, 500); @@ -2646,14 +2678,15 @@ export async function startApp(): Promise { sent_at: data.timestamp, serverTimestamp: data.serverTimestamp, sent_to: sentTo, - received_at: now, + received_at: data.receivedAtCounter, + received_at_ms: data.receivedAtDate, conversationId: descriptor.id, type: 'outgoing', sent: true, unidentifiedDeliveries: data.unidentifiedDeliveries || [], expirationStartTimestamp: Math.min( - data.expirationStartTimestamp || data.timestamp || Date.now(), - Date.now() + data.expirationStartTimestamp || data.timestamp || now, + now ), } as WhatIsThis); } @@ -2856,13 +2889,18 @@ export async function startApp(): Promise { data: WhatIsThis, descriptor: MessageDescriptor ) { + assert( + Boolean(data.receivedAtCounter), + `Did not receive receivedAtCounter for message: ${data.timestamp}` + ); return new window.Whisper.Message({ source: data.source, sourceUuid: data.sourceUuid, sourceDevice: data.sourceDevice, sent_at: data.timestamp, serverTimestamp: data.serverTimestamp, - received_at: Date.now(), + received_at: data.receivedAtCounter, + received_at_ms: data.receivedAtDate, conversationId: descriptor.id, unidentifiedDeliveryReceived: data.unidentifiedDeliveryReceived, type: 'incoming', diff --git a/ts/components/LightboxGallery.stories.tsx b/ts/components/LightboxGallery.stories.tsx index 6fad0185a..568a84461 100644 --- a/ts/components/LightboxGallery.stories.tsx +++ b/ts/components/LightboxGallery.stories.tsx @@ -40,7 +40,8 @@ story.add('Image and Video', () => { message: { attachments: [], id: 'image-msg', - received_at: Date.now(), + received_at: 1, + received_at_ms: Date.now(), }, objectURL: '/fixtures/tina-rolf-269345-unsplash.jpg', }, @@ -55,7 +56,8 @@ story.add('Image and Video', () => { message: { attachments: [], id: 'video-msg', - received_at: Date.now(), + received_at: 2, + received_at_ms: Date.now(), }, objectURL: '/fixtures/pixabay-Soap-Bubble-7141.mp4', }, @@ -79,7 +81,8 @@ story.add('Missing Media', () => { message: { attachments: [], id: 'image-msg', - received_at: Date.now(), + received_at: 3, + received_at_ms: Date.now(), }, objectURL: undefined, }, diff --git a/ts/components/conversation/media-gallery/AttachmentSection.stories.tsx b/ts/components/conversation/media-gallery/AttachmentSection.stories.tsx index c924d1e9c..4bde261e7 100644 --- a/ts/components/conversation/media-gallery/AttachmentSection.stories.tsx +++ b/ts/components/conversation/media-gallery/AttachmentSection.stories.tsx @@ -52,7 +52,8 @@ const createRandomFile = ( contentType, message: { id: random(now).toString(), - received_at: random(startTime, startTime + timeWindow), + received_at: Math.floor(Math.random() * 10), + received_at_ms: random(startTime, startTime + timeWindow), attachments: [], }, attachment: { diff --git a/ts/components/conversation/media-gallery/AttachmentSection.tsx b/ts/components/conversation/media-gallery/AttachmentSection.tsx index 662cc01e3..978d26dc7 100644 --- a/ts/components/conversation/media-gallery/AttachmentSection.tsx +++ b/ts/components/conversation/media-gallery/AttachmentSection.tsx @@ -9,6 +9,7 @@ import { MediaGridItem } from './MediaGridItem'; import { MediaItemType } from '../../LightboxGallery'; import { missingCaseError } from '../../../util/missingCaseError'; import { LocalizerType } from '../../../types/Util'; +import { getMessageTimestamp } from '../../../util/getMessageTimestamp'; export type Props = { i18n: LocalizerType; @@ -58,7 +59,7 @@ export class AttachmentSection extends React.Component { fileSize={attachment.size} shouldShowSeparator={shouldShowSeparator} onClick={onClick} - timestamp={message.received_at} + timestamp={getMessageTimestamp(message)} /> ); default: diff --git a/ts/components/conversation/media-gallery/MediaGallery.tsx b/ts/components/conversation/media-gallery/MediaGallery.tsx index 38a6b805d..3be5abf2f 100644 --- a/ts/components/conversation/media-gallery/MediaGallery.tsx +++ b/ts/components/conversation/media-gallery/MediaGallery.tsx @@ -12,6 +12,7 @@ import { groupMediaItemsByDate } from './groupMediaItemsByDate'; import { ItemClickEvent } from './types/ItemClickEvent'; import { missingCaseError } from '../../../util/missingCaseError'; import { LocalizerType } from '../../../types/Util'; +import { getMessageTimestamp } from '../../../util/getMessageTimestamp'; import { MediaItemType } from '../../LightboxGallery'; @@ -145,7 +146,7 @@ export class MediaGallery extends React.Component { const sections = groupMediaItemsByDate(now, mediaItems).map(section => { const first = section.mediaItems[0]; const { message } = first; - const date = moment(message.received_at); + const date = moment(getMessageTimestamp(message)); const header = section.type === 'yearMonth' ? date.format(MONTH_FORMAT) diff --git a/ts/components/conversation/media-gallery/groupMediaItemsByDate.ts b/ts/components/conversation/media-gallery/groupMediaItemsByDate.ts index 89dd44111..e217b8bd8 100644 --- a/ts/components/conversation/media-gallery/groupMediaItemsByDate.ts +++ b/ts/components/conversation/media-gallery/groupMediaItemsByDate.ts @@ -5,6 +5,7 @@ import moment from 'moment'; import { compact, groupBy, sortBy } from 'lodash'; import { MediaItemType } from '../../LightboxGallery'; +import { getMessageTimestamp } from '../../../util/getMessageTimestamp'; // import { missingCaseError } from '../../../util/missingCaseError'; @@ -120,7 +121,7 @@ const withSection = (referenceDateTime: moment.Moment) => ( const thisMonth = moment(referenceDateTime).startOf('month'); const { message } = mediaItem; - const mediaItemReceivedDate = moment.utc(message.received_at); + const mediaItemReceivedDate = moment.utc(getMessageTimestamp(message)); if (mediaItemReceivedDate.isAfter(today)) { return { order: 0, diff --git a/ts/components/conversation/media-gallery/types/Message.ts b/ts/components/conversation/media-gallery/types/Message.ts index 76d029e38..4d295670d 100644 --- a/ts/components/conversation/media-gallery/types/Message.ts +++ b/ts/components/conversation/media-gallery/types/Message.ts @@ -9,4 +9,6 @@ export type Message = { // Assuming this is for the API // eslint-disable-next-line camelcase received_at: number; + // eslint-disable-next-line camelcase + received_at_ms: number; }; diff --git a/ts/groups.ts b/ts/groups.ts index 568cc56b5..bebee4ec3 100644 --- a/ts/groups.ts +++ b/ts/groups.ts @@ -2452,7 +2452,8 @@ async function updateGroup({ return { ...changeMessage, conversationId: conversation.id, - received_at: finalReceivedAt, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: finalReceivedAt, sent_at: syntheticSentAt, }; }); diff --git a/ts/logging/main_process_logging.ts b/ts/logging/main_process_logging.ts index 0448820b2..4981ed44c 100644 --- a/ts/logging/main_process_logging.ts +++ b/ts/logging/main_process_logging.ts @@ -61,9 +61,9 @@ export async function initialize(): Promise { }, 500); } - const logFile = path.join(logPath, 'log.log'); + const logFile = path.join(logPath, 'main.log'); const loggerOptions: bunyan.LoggerOptions = { - name: 'log', + name: 'main', streams: [ { type: 'rotating-file', @@ -83,31 +83,6 @@ export async function initialize(): Promise { const logger = bunyan.createLogger(loggerOptions); - ipc.on('batch-log', (_first, batch: unknown) => { - if (!Array.isArray(batch)) { - logger.error( - 'batch-log IPC event was called with a non-array; dropping logs' - ); - return; - } - - batch.forEach(item => { - if (isLogEntry(item)) { - const levelString = getLogLevelString(item.level); - logger[levelString]( - { - time: item.time, - }, - item.msg - ); - } else { - logger.error( - 'batch-log IPC event was called with an invalid log entry; dropping entry' - ); - } - }); - }); - ipc.on('fetch-log', event => { fetch(logPath).then( data => { diff --git a/ts/logging/set_up_renderer_logging.ts b/ts/logging/set_up_renderer_logging.ts index d4c6d0951..6a090481d 100644 --- a/ts/logging/set_up_renderer_logging.ts +++ b/ts/logging/set_up_renderer_logging.ts @@ -7,11 +7,11 @@ import { ipcRenderer as ipc } from 'electron'; import _ from 'lodash'; -import { levelFromName } from 'bunyan'; +import * as path from 'path'; +import * as bunyan from 'bunyan'; import { uploadDebugLogs } from './debuglogs'; import { redactAll } from '../../js/modules/privacy'; -import { createBatcher } from '../util/batcher'; import { LogEntryType, LogLevel, @@ -23,7 +23,7 @@ import * as log from './log'; import { reallyJsonStringify } from '../util/reallyJsonStringify'; // To make it easier to visually scan logs, we make all levels the same length -const levelMaxLength: number = Object.keys(levelFromName).reduce( +const levelMaxLength: number = Object.keys(bunyan.levelFromName).reduce( (maxLength, level) => Math.max(maxLength, level.length), 0 ); @@ -96,6 +96,30 @@ function fetch(): Promise { }); } +let globalLogger: undefined | bunyan; + +export function initialize(): void { + if (globalLogger) { + throw new Error('Already called initialize!'); + } + + const basePath = ipc.sendSync('get-user-data-path'); + const logFile = path.join(basePath, 'logs', 'app.log'); + const loggerOptions: bunyan.LoggerOptions = { + name: 'app', + streams: [ + { + type: 'rotating-file', + path: logFile, + period: '1d', + count: 3, + }, + ], + }; + + globalLogger = bunyan.createLogger(loggerOptions); +} + const publish = uploadDebugLogs; // A modern logging interface for the browser @@ -103,14 +127,6 @@ const publish = uploadDebugLogs; const env = window.getEnvironment(); const IS_PRODUCTION = env === 'production'; -const ipcBatcher = createBatcher({ - wait: 500, - maxSize: 500, - processBatch: (items: Array) => { - ipc.send('batch-log', items); - }, -}); - // The Bunyan API: https://github.com/trentm/node-bunyan#log-method-api function logAtLevel(level: LogLevel, ...args: ReadonlyArray): void { if (!IS_PRODUCTION) { @@ -120,11 +136,16 @@ function logAtLevel(level: LogLevel, ...args: ReadonlyArray): void { console._log(prefix, now(), ...args); } - ipcBatcher.add({ - level, - msg: cleanArgs(args), - time: new Date().toISOString(), - }); + const levelString = getLogLevelString(level); + const msg = cleanArgs(args); + const time = new Date().toISOString(); + + if (!globalLogger) { + throw new Error('Logger has not been initialized yet'); + return; + } + + globalLogger[levelString]({ time }, msg); } log.setLogAtLevel(logAtLevel); diff --git a/ts/model-types.d.ts b/ts/model-types.d.ts index 7daba82ad..541b9b164 100644 --- a/ts/model-types.d.ts +++ b/ts/model-types.d.ts @@ -134,6 +134,7 @@ export type MessageAttributesType = { groupV2Change?: GroupV2ChangeType; // Required. Used to sort messages in the database for the conversation timeline. received_at?: number; + received_at_ms?: number; // More of a legacy feature, needed as we were updating the schema of messages in the // background, when we were still in IndexedDB, before attachments had gone to disk // We set this so that the idle message upgrade process doesn't pick this message up diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index 549a74980..53764e9eb 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -2282,7 +2282,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, type: 'chat-session-refreshed', sent_at: receivedAt, - received_at: receivedAt, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: receivedAt, unread: 1, // TODO: DESKTOP-722 // this type does not fully implement the interface it is expected to @@ -2315,7 +2316,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, type: 'keychange', sent_at: this.get('timestamp'), - received_at: timestamp, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: timestamp, key_changed: keyChangedId, unread: 1, // TODO: DESKTOP-722 @@ -2373,7 +2375,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, type: 'verified-change', sent_at: lastMessage, - received_at: timestamp, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: timestamp, verifiedChanged: verifiedChangeId, verified, local: options.local, @@ -2435,7 +2438,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, type: 'call-history', sent_at: timestamp, - received_at: timestamp, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: timestamp, unread, callHistoryDetails: detailsToSave, // TODO: DESKTOP-722 @@ -2481,11 +2485,13 @@ export class ConversationModel extends window.Backbone.Model< profileChange: unknown, conversationId?: string ): Promise { + const now = Date.now(); const message = ({ conversationId: this.id, type: 'profile-change', - sent_at: Date.now(), - received_at: Date.now(), + sent_at: now, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: now, unread: true, changedId: conversationId || this.id, profileChange, @@ -2984,7 +2990,8 @@ export class ConversationModel extends window.Backbone.Model< type: 'outgoing', conversationId: this.get('id'), sent_at: timestamp, - received_at: timestamp, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: timestamp, recipients, deletedForEveryoneTimestamp: targetTimestamp, // TODO: DESKTOP-722 @@ -3093,7 +3100,8 @@ export class ConversationModel extends window.Backbone.Model< type: 'outgoing', conversationId: this.get('id'), sent_at: timestamp, - received_at: timestamp, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: timestamp, recipients, reaction: outgoingReaction, // TODO: DESKTOP-722 @@ -3244,7 +3252,8 @@ export class ConversationModel extends window.Backbone.Model< preview, attachments, sent_at: now, - received_at: now, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: now, expireTimer, recipients, sticker, @@ -3866,7 +3875,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, // No type; 'incoming' messages are specially treated by conversation.markRead() sent_at: timestamp, - received_at: timestamp, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: timestamp, flags: window.textsecure.protobuf.DataMessage.Flags.EXPIRATION_TIMER_UPDATE, expirationTimerUpdate: { @@ -3970,7 +3980,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, // No type; 'incoming' messages are specially treated by conversation.markRead() sent_at: timestamp, - received_at: timestamp, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: timestamp, // TODO: DESKTOP-722 } as unknown) as MessageAttributesType); @@ -4003,7 +4014,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, type: 'outgoing', sent_at: now, - received_at: now, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: now, destination: this.get('e164'), destinationUuid: this.get('uuid'), recipients: this.getRecipients(), @@ -4059,7 +4071,8 @@ export class ConversationModel extends window.Backbone.Model< conversationId: this.id, type: 'outgoing', sent_at: now, - received_at: now, + received_at: window.Signal.Util.incrementMessageCounter(), + received_at_ms: now, // TODO: DESKTOP-722 } as unknown) as MessageAttributesType); diff --git a/ts/models/messages.ts b/ts/models/messages.ts index f31bdd31b..4f90a329b 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -198,6 +198,29 @@ export class MessageModel extends window.Backbone.Model { }; } + getSenderIdentifier(): string { + const sentAt = this.get('sent_at'); + const source = this.get('source'); + const sourceUuid = this.get('sourceUuid'); + const sourceDevice = this.get('sourceDevice'); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const sourceId = window.ConversationController.ensureContactIds({ + e164: source, + uuid: sourceUuid, + })!; + + return `${sourceId}.${sourceDevice}-${sentAt}`; + } + + getReceivedAt(): number { + // We would like to get the received_at_ms ideally since received_at is + // now an incrementing counter for messages and not the actual time that + // the message was received. If this field doesn't exist on the message + // then we can trust received_at. + return Number(this.get('received_at_ms') || this.get('received_at')); + } + isNormalBubble(): boolean { return ( !this.isCallHistory() && @@ -381,7 +404,7 @@ export class MessageModel extends window.Backbone.Model { return { sentAt: this.get('sent_at'), - receivedAt: this.get('received_at'), + receivedAt: this.getReceivedAt(), message: { ...this.getPropsForMessage(), disableMenu: true, @@ -1904,9 +1927,7 @@ export class MessageModel extends window.Backbone.Model { window.Whisper.Notifications.removeBy({ messageId: this.id }); if (!skipSave) { - await window.Signal.Data.saveMessage(this.attributes, { - Message: window.Whisper.Message, - }); + window.Signal.Util.updateMessageBatcher.add(this.attributes); } } @@ -1955,9 +1976,7 @@ export class MessageModel extends window.Backbone.Model { const id = this.get('id'); if (id && !skipSave) { - await window.Signal.Data.saveMessage(this.attributes, { - Message: window.Whisper.Message, - }); + window.Signal.Util.updateMessageBatcher.add(this.attributes); } } } @@ -2822,20 +2841,32 @@ export class MessageModel extends window.Backbone.Model { } ); - const collection = await window.Signal.Data.getMessagesBySentAt(id, { - MessageCollection: window.Whisper.MessageCollection, - }); - const found = collection.find(item => { - const messageAuthorId = item.getContactId(); + const inMemoryMessage = window.MessageController.findBySentAt(id); - return authorConversationId === messageAuthorId; - }); + let queryMessage; - if (!found) { - quote.referencedMessageNotFound = true; - return message; + if (inMemoryMessage) { + queryMessage = inMemoryMessage; + } else { + window.log.info('copyFromQuotedMessage: db lookup needed', id); + const collection = await window.Signal.Data.getMessagesBySentAt(id, { + MessageCollection: window.Whisper.MessageCollection, + }); + const found = collection.find(item => { + const messageAuthorId = item.getContactId(); + + return authorConversationId === messageAuthorId; + }); + + if (!found) { + quote.referencedMessageNotFound = true; + return message; + } + + queryMessage = window.MessageController.register(found.id, found); } - if (found.isTapToView()) { + + if (queryMessage.isTapToView()) { quote.text = null; quote.attachments = [ { @@ -2846,7 +2877,6 @@ export class MessageModel extends window.Backbone.Model { return message; } - const queryMessage = window.MessageController.register(found.id, found); quote.text = queryMessage.get('body'); if (firstAttachment) { firstAttachment.thumbnail = null; @@ -2946,9 +2976,20 @@ export class MessageModel extends window.Backbone.Model { ); // First, check for duplicates. If we find one, stop processing here. - const existingMessage = await getMessageBySender(this.attributes, { - Message: window.Whisper.Message, - }); + const inMemoryMessage = window.MessageController.findBySender( + this.getSenderIdentifier() + ); + if (!inMemoryMessage) { + window.log.info( + 'handleDataMessage: duplicate check db lookup needed', + this.getSenderIdentifier() + ); + } + const existingMessage = + inMemoryMessage || + (await getMessageBySender(this.attributes, { + Message: window.Whisper.Message, + })); const isUpdate = Boolean(data && data.isRecipientUpdate); if (existingMessage && type === 'incoming') { @@ -3422,7 +3463,7 @@ export class MessageModel extends window.Backbone.Model { dataMessage.expireTimer, source, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - message.get('received_at')!, + message.getReceivedAt()!, { fromGroupUpdate: message.isGroupUpdate(), } @@ -3437,7 +3478,7 @@ export class MessageModel extends window.Backbone.Model { undefined, source, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - message.get('received_at')! + message.getReceivedAt()! ); } } @@ -3573,7 +3614,8 @@ export class MessageModel extends window.Backbone.Model { (this.getConversation()!.getAccepted() || message.isOutgoing()) && !shouldHoldOffDownload ) { - await message.queueAttachmentDownloads(); + window.attachmentDownloadQueue = window.attachmentDownloadQueue || []; + window.attachmentDownloadQueue.unshift(message); } // Does this message have any pending, previously-received associated reactions? @@ -3591,24 +3633,11 @@ export class MessageModel extends window.Backbone.Model { ) ); - await window.Signal.Data.saveMessage(message.attributes, { - Message: window.Whisper.Message, - forceSave: true, - }); - - conversation.trigger('newmessage', message); - - if (message.get('unread')) { - await conversation.notify(message); - } - - // Increment the sent message count if this is an outgoing message - if (type === 'outgoing') { - conversation.incrementSentMessageCount(); - } - - window.Whisper.events.trigger('incrementProgress'); - confirm(); + window.log.info( + 'handleDataMessage: Batching save for', + message.get('sent_at') + ); + this.saveAndNotify(conversation, confirm); } catch (error) { const errorForLog = error && error.stack ? error.stack : error; window.log.error( @@ -3622,6 +3651,29 @@ export class MessageModel extends window.Backbone.Model { }); } + async saveAndNotify( + conversation: ConversationModel, + confirm: () => void + ): Promise { + await window.Signal.Util.saveNewMessageBatcher.add(this.attributes); + + window.log.info('Message saved', this.get('sent_at')); + + conversation.trigger('newmessage', this); + + if (this.get('unread')) { + await conversation.notify(this); + } + + // Increment the sent message count if this is an outgoing message + if (this.get('type') === 'outgoing') { + conversation.incrementSentMessageCount(); + } + + window.Whisper.events.trigger('incrementProgress'); + confirm(); + } + async handleReaction( reaction: typeof window.WhatIsThis, shouldPersist = true diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index 728dd33ad..14be25df3 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -9,18 +9,7 @@ /* eslint-disable @typescript-eslint/ban-types */ import { ipcRenderer } from 'electron'; -import { - cloneDeep, - compact, - fromPairs, - get, - groupBy, - isFunction, - last, - map, - omit, - set, -} from 'lodash'; +import { cloneDeep, get, groupBy, last, map, omit, set } from 'lodash'; import { arrayBufferToBase64, base64ToArrayBuffer } from '../Crypto'; import { CURRENT_SCHEMA_VERSION } from '../../js/modules/types/message'; @@ -36,7 +25,6 @@ import { import { AttachmentDownloadJobType, ClientInterface, - ClientJobType, ConversationType, IdentityKeyType, ItemType, @@ -44,7 +32,6 @@ import { MessageTypeUnhydrated, PreKeyType, SearchResultMessageType, - ServerInterface, SessionType, SignedPreKeyType, StickerPackStatusType, @@ -52,8 +39,10 @@ import { StickerType, UnprocessedType, } from './Interface'; +import Server from './Server'; import { MessageModel } from '../models/messages'; import { ConversationModel } from '../models/conversations'; +import { waitForPendingQueries } from './Queueing'; // We listen to a lot of events on ipcRenderer, often on the same channel. This prevents // any warnings that might be sent to the console in that case. @@ -65,7 +54,6 @@ if (ipcRenderer && ipcRenderer.setMaxListeners) { const DATABASE_UPDATE_TIMEOUT = 2 * 60 * 1000; // two minutes -const SQL_CHANNEL_KEY = 'sql-channel'; const ERASE_SQL_KEY = 'erase-sql-key'; const ERASE_ATTACHMENTS_KEY = 'erase-attachments'; const ERASE_STICKERS_KEY = 'erase-stickers'; @@ -74,19 +62,6 @@ const ERASE_DRAFTS_KEY = 'erase-drafts'; const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments'; const ENSURE_FILE_PERMISSIONS = 'ensure-file-permissions'; -type ClientJobUpdateType = { - resolve: Function; - reject: Function; - args?: Array; -}; - -const _jobs: { [id: string]: ClientJobType } = Object.create(null); -const _DEBUG = false; -let _jobCounter = 0; -let _shuttingDown = false; -let _shutdownCallback: Function | null = null; -let _shutdownPromise: Promise | null = null; - // Because we can't force this module to conform to an interface, we narrow our exports // to this one default export, which does conform to the interface. // Note: In Javascript, you need to access the .default property when requiring it @@ -233,25 +208,10 @@ const dataInterface: ClientInterface = { // Client-side only, and test-only _removeConversations, - _jobs, }; export default dataInterface; -const channelsAsUnknown = fromPairs( - compact( - map(dataInterface, (value: any) => { - if (isFunction(value)) { - return [value.name, makeChannel(value.name)]; - } - - return null; - }) - ) -) as any; - -const channels: ServerInterface = channelsAsUnknown; - function _cleanData( data: unknown ): ReturnType['cleaned'] { @@ -267,185 +227,14 @@ function _cleanData( } function _cleanMessageData(data: MessageType): MessageType { + // Ensure that all messages have the received_at set properly + if (!data.received_at) { + assert(false, 'received_at was not set on the message'); + data.received_at = window.Signal.Util.incrementMessageCounter(); + } return _cleanData(omit(data, ['dataMessage'])); } -async function _shutdown() { - const jobKeys = Object.keys(_jobs); - window.log.info( - `data.shutdown: shutdown requested. ${jobKeys.length} jobs outstanding` - ); - - if (_shutdownPromise) { - await _shutdownPromise; - - return; - } - - _shuttingDown = true; - - // No outstanding jobs, return immediately - if (jobKeys.length === 0 || _DEBUG) { - return; - } - - // Outstanding jobs; we need to wait until the last one is done - _shutdownPromise = new Promise((resolve, reject) => { - _shutdownCallback = (error: Error) => { - window.log.info('data.shutdown: process complete'); - if (error) { - reject(error); - - return; - } - - resolve(); - }; - }); - - await _shutdownPromise; -} - -function _makeJob(fnName: string) { - if (_shuttingDown && fnName !== 'close') { - throw new Error( - `Rejecting SQL channel job (${fnName}); application is shutting down` - ); - } - - _jobCounter += 1; - const id = _jobCounter; - - if (_DEBUG) { - window.log.info(`SQL channel job ${id} (${fnName}) started`); - } - _jobs[id] = { - fnName, - start: Date.now(), - }; - - return id; -} - -function _updateJob(id: number, data: ClientJobUpdateType) { - const { resolve, reject } = data; - const { fnName, start } = _jobs[id]; - - _jobs[id] = { - ..._jobs[id], - ...data, - resolve: (value: any) => { - _removeJob(id); - const end = Date.now(); - const delta = end - start; - if (delta > 10 || _DEBUG) { - window.log.info( - `SQL channel job ${id} (${fnName}) succeeded in ${end - start}ms` - ); - } - - return resolve(value); - }, - reject: (error: Error) => { - _removeJob(id); - const end = Date.now(); - window.log.info( - `SQL channel job ${id} (${fnName}) failed in ${end - start}ms` - ); - - if (error && error.message && error.message.includes('SQLITE_CORRUPT')) { - window.log.error( - 'Detected SQLITE_CORRUPT error; restarting the application immediately' - ); - window.restart(); - } - - return reject(error); - }, - }; -} - -function _removeJob(id: number) { - if (_DEBUG) { - _jobs[id].complete = true; - - return; - } - - delete _jobs[id]; - - if (_shutdownCallback) { - const keys = Object.keys(_jobs); - if (keys.length === 0) { - _shutdownCallback(); - } - } -} - -function _getJob(id: number) { - return _jobs[id]; -} - -if (ipcRenderer && ipcRenderer.on) { - ipcRenderer.on( - `${SQL_CHANNEL_KEY}-done`, - (_, jobId, errorForDisplay, result) => { - const job = _getJob(jobId); - if (!job) { - throw new Error( - `Received SQL channel reply to job ${jobId}, but did not have it in our registry!` - ); - } - - const { resolve, reject, fnName } = job; - - if (!resolve || !reject) { - throw new Error( - `SQL channel job ${jobId} (${fnName}): didn't have a resolve or reject` - ); - } - - if (errorForDisplay) { - return reject( - new Error( - `Error received from SQL channel job ${jobId} (${fnName}): ${errorForDisplay}` - ) - ); - } - - return resolve(result); - } - ); -} else { - window.log.warn('sql/Client: ipcRenderer.on is not available!'); -} - -function makeChannel(fnName: string) { - return async (...args: Array) => { - const jobId = _makeJob(fnName); - - return new Promise((resolve, reject) => { - try { - ipcRenderer.send(SQL_CHANNEL_KEY, jobId, fnName, ...args); - - _updateJob(jobId, { - resolve, - reject, - args: _DEBUG ? args : undefined, - }); - - setTimeout(() => { - reject(new Error(`SQL channel job ${jobId} (${fnName}) timed out`)); - }, DATABASE_UPDATE_TIMEOUT); - } catch (error) { - _removeJob(jobId); - - reject(error); - } - }); - }; -} - function keysToArrayBuffer(keys: Array, data: any) { const updated = cloneDeep(data); @@ -481,25 +270,23 @@ function keysFromArrayBuffer(keys: Array, data: any) { // Top-level calls async function shutdown() { - // Stop accepting new SQL jobs, flush outstanding queue - await _shutdown(); - + await waitForPendingQueries(); // Close database await close(); } // Note: will need to restart the app after calling this, to set up afresh async function close() { - await channels.close(); + await Server.close(); } // Note: will need to restart the app after calling this, to set up afresh async function removeDB() { - await channels.removeDB(); + await Server.removeDB(); } async function removeIndexedDBFiles() { - await channels.removeIndexedDBFiles(); + await Server.removeIndexedDBFiles(); } // Identity Keys @@ -510,14 +297,14 @@ async function createOrUpdateIdentityKey(data: IdentityKeyType) { ...data, id: window.ConversationController.getConversationId(data.id), }); - await channels.createOrUpdateIdentityKey(updated); + await Server.createOrUpdateIdentityKey(updated); } async function getIdentityKeyById(identifier: string) { const id = window.ConversationController.getConversationId(identifier); if (!id) { throw new Error('getIdentityKeyById: unable to find conversationId'); } - const data = await channels.getIdentityKeyById(id); + const data = await Server.getIdentityKeyById(id); return keysToArrayBuffer(IDENTITY_KEY_KEYS, data); } @@ -525,20 +312,20 @@ async function bulkAddIdentityKeys(array: Array) { const updated = map(array, data => keysFromArrayBuffer(IDENTITY_KEY_KEYS, data) ); - await channels.bulkAddIdentityKeys(updated); + await Server.bulkAddIdentityKeys(updated); } async function removeIdentityKeyById(identifier: string) { const id = window.ConversationController.getConversationId(identifier); if (!id) { throw new Error('removeIdentityKeyById: unable to find conversationId'); } - await channels.removeIdentityKeyById(id); + await Server.removeIdentityKeyById(id); } async function removeAllIdentityKeys() { - await channels.removeAllIdentityKeys(); + await Server.removeAllIdentityKeys(); } async function getAllIdentityKeys() { - const keys = await channels.getAllIdentityKeys(); + const keys = await Server.getAllIdentityKeys(); return keys.map(key => keysToArrayBuffer(IDENTITY_KEY_KEYS, key)); } @@ -547,25 +334,25 @@ async function getAllIdentityKeys() { async function createOrUpdatePreKey(data: PreKeyType) { const updated = keysFromArrayBuffer(PRE_KEY_KEYS, data); - await channels.createOrUpdatePreKey(updated); + await Server.createOrUpdatePreKey(updated); } async function getPreKeyById(id: number) { - const data = await channels.getPreKeyById(id); + const data = await Server.getPreKeyById(id); return keysToArrayBuffer(PRE_KEY_KEYS, data); } async function bulkAddPreKeys(array: Array) { const updated = map(array, data => keysFromArrayBuffer(PRE_KEY_KEYS, data)); - await channels.bulkAddPreKeys(updated); + await Server.bulkAddPreKeys(updated); } async function removePreKeyById(id: number) { - await channels.removePreKeyById(id); + await Server.removePreKeyById(id); } async function removeAllPreKeys() { - await channels.removeAllPreKeys(); + await Server.removeAllPreKeys(); } async function getAllPreKeys() { - const keys = await channels.getAllPreKeys(); + const keys = await Server.getAllPreKeys(); return keys.map(key => keysToArrayBuffer(PRE_KEY_KEYS, key)); } @@ -575,15 +362,15 @@ async function getAllPreKeys() { const PRE_KEY_KEYS = ['privateKey', 'publicKey']; async function createOrUpdateSignedPreKey(data: SignedPreKeyType) { const updated = keysFromArrayBuffer(PRE_KEY_KEYS, data); - await channels.createOrUpdateSignedPreKey(updated); + await Server.createOrUpdateSignedPreKey(updated); } async function getSignedPreKeyById(id: number) { - const data = await channels.getSignedPreKeyById(id); + const data = await Server.getSignedPreKeyById(id); return keysToArrayBuffer(PRE_KEY_KEYS, data); } async function getAllSignedPreKeys() { - const keys = await channels.getAllSignedPreKeys(); + const keys = await Server.getAllSignedPreKeys(); return keys.map((key: SignedPreKeyType) => keysToArrayBuffer(PRE_KEY_KEYS, key) @@ -591,13 +378,13 @@ async function getAllSignedPreKeys() { } async function bulkAddSignedPreKeys(array: Array) { const updated = map(array, data => keysFromArrayBuffer(PRE_KEY_KEYS, data)); - await channels.bulkAddSignedPreKeys(updated); + await Server.bulkAddSignedPreKeys(updated); } async function removeSignedPreKeyById(id: number) { - await channels.removeSignedPreKeyById(id); + await Server.removeSignedPreKeyById(id); } async function removeAllSignedPreKeys() { - await channels.removeAllSignedPreKeys(); + await Server.removeAllSignedPreKeys(); } // Items @@ -620,16 +407,16 @@ async function createOrUpdateItem(data: ItemType) { const keys = ITEM_KEYS[id]; const updated = Array.isArray(keys) ? keysFromArrayBuffer(keys, data) : data; - await channels.createOrUpdateItem(updated); + await Server.createOrUpdateItem(updated); } async function getItemById(id: string) { const keys = ITEM_KEYS[id]; - const data = await channels.getItemById(id); + const data = await Server.getItemById(id); return Array.isArray(keys) ? keysToArrayBuffer(keys, data) : data; } async function getAllItems() { - const items = await channels.getAllItems(); + const items = await Server.getAllItems(); return map(items, item => { const { id } = item; @@ -645,48 +432,48 @@ async function bulkAddItems(array: Array) { return keys && Array.isArray(keys) ? keysFromArrayBuffer(keys, data) : data; }); - await channels.bulkAddItems(updated); + await Server.bulkAddItems(updated); } async function removeItemById(id: string) { - await channels.removeItemById(id); + await Server.removeItemById(id); } async function removeAllItems() { - await channels.removeAllItems(); + await Server.removeAllItems(); } // Sessions async function createOrUpdateSession(data: SessionType) { - await channels.createOrUpdateSession(data); + await Server.createOrUpdateSession(data); } async function createOrUpdateSessions(array: Array) { - await channels.createOrUpdateSessions(array); + await Server.createOrUpdateSessions(array); } async function getSessionById(id: string) { - const session = await channels.getSessionById(id); + const session = await Server.getSessionById(id); return session; } async function getSessionsById(id: string) { - const sessions = await channels.getSessionsById(id); + const sessions = await Server.getSessionsById(id); return sessions; } async function bulkAddSessions(array: Array) { - await channels.bulkAddSessions(array); + await Server.bulkAddSessions(array); } async function removeSessionById(id: string) { - await channels.removeSessionById(id); + await Server.removeSessionById(id); } async function removeSessionsByConversation(conversationId: string) { - await channels.removeSessionsByConversation(conversationId); + await Server.removeSessionsByConversation(conversationId); } async function removeAllSessions() { - await channels.removeAllSessions(); + await Server.removeAllSessions(); } async function getAllSessions() { - const sessions = await channels.getAllSessions(); + const sessions = await Server.getAllSessions(); return sessions; } @@ -694,22 +481,22 @@ async function getAllSessions() { // Conversation async function getConversationCount() { - return channels.getConversationCount(); + return Server.getConversationCount(); } async function saveConversation(data: ConversationType) { - await channels.saveConversation(data); + await Server.saveConversation(data); } async function saveConversations(array: Array) { - await channels.saveConversations(array); + await Server.saveConversations(array); } async function getConversationById( id: string, { Conversation }: { Conversation: typeof ConversationModel } ) { - const data = await channels.getConversationById(id); + const data = await Server.getConversationById(id); return new Conversation(data); } @@ -737,7 +524,7 @@ async function updateConversations(array: Array) { !pathsChanged.length, `Paths were cleaned: ${JSON.stringify(pathsChanged)}` ); - await channels.updateConversations(cleaned); + await Server.updateConversations(cleaned); } async function removeConversation( @@ -749,18 +536,18 @@ async function removeConversation( // Note: It's important to have a fully database-hydrated model to delete here because // it needs to delete all associated on-disk files along with the database delete. if (existing) { - await channels.removeConversation(id); + await Server.removeConversation(id); await existing.cleanup(); } } // Note: this method will not clean up external files, just delete from SQL async function _removeConversations(ids: Array) { - await channels.removeConversation(ids); + await Server.removeConversation(ids); } async function eraseStorageServiceStateFromConversations() { - await channels.eraseStorageServiceStateFromConversations(); + await Server.eraseStorageServiceStateFromConversations(); } async function getAllConversations({ @@ -768,7 +555,7 @@ async function getAllConversations({ }: { ConversationCollection: typeof ConversationModelCollectionType; }): Promise { - const conversations = await channels.getAllConversations(); + const conversations = await Server.getAllConversations(); const collection = new ConversationCollection(); collection.add(conversations); @@ -777,7 +564,7 @@ async function getAllConversations({ } async function getAllConversationIds() { - const ids = await channels.getAllConversationIds(); + const ids = await Server.getAllConversationIds(); return ids; } @@ -787,7 +574,7 @@ async function getAllPrivateConversations({ }: { ConversationCollection: typeof ConversationModelCollectionType; }) { - const conversations = await channels.getAllPrivateConversations(); + const conversations = await Server.getAllPrivateConversations(); const collection = new ConversationCollection(); collection.add(conversations); @@ -803,7 +590,7 @@ async function getAllGroupsInvolvingId( ConversationCollection: typeof ConversationModelCollectionType; } ) { - const conversations = await channels.getAllGroupsInvolvingId(id); + const conversations = await Server.getAllGroupsInvolvingId(id); const collection = new ConversationCollection(); collection.add(conversations); @@ -812,7 +599,7 @@ async function getAllGroupsInvolvingId( } async function searchConversations(query: string) { - const conversations = await channels.searchConversations(query); + const conversations = await Server.searchConversations(query); return conversations; } @@ -828,7 +615,7 @@ async function searchMessages( query: string, { limit }: { limit?: number } = {} ) { - const messages = await channels.searchMessages(query, { limit }); + const messages = await Server.searchMessages(query, { limit }); return handleSearchMessageJSON(messages); } @@ -838,7 +625,7 @@ async function searchMessagesInConversation( conversationId: string, { limit }: { limit?: number } = {} ) { - const messages = await channels.searchMessagesInConversation( + const messages = await Server.searchMessagesInConversation( query, conversationId, { limit } @@ -850,14 +637,14 @@ async function searchMessagesInConversation( // Message async function getMessageCount(conversationId?: string) { - return channels.getMessageCount(conversationId); + return Server.getMessageCount(conversationId); } async function saveMessage( data: MessageType, { forceSave, Message }: { forceSave?: boolean; Message: typeof MessageModel } ) { - const id = await channels.saveMessage(_cleanMessageData(data), { + const id = await Server.saveMessage(_cleanMessageData(data), { forceSave, }); Message.updateTimers(); @@ -869,7 +656,7 @@ async function saveMessages( arrayOfMessages: Array, { forceSave }: { forceSave?: boolean } = {} ) { - await channels.saveMessages( + await Server.saveMessages( arrayOfMessages.map(message => _cleanMessageData(message)), { forceSave } ); @@ -884,21 +671,21 @@ async function removeMessage( // Note: It's important to have a fully database-hydrated model to delete here because // it needs to delete all associated on-disk files along with the database delete. if (message) { - await channels.removeMessage(id); + await Server.removeMessage(id); await message.cleanup(); } } // Note: this method will not clean up external files, just delete from SQL async function removeMessages(ids: Array) { - await channels.removeMessages(ids); + await Server.removeMessages(ids); } async function getMessageById( id: string, { Message }: { Message: typeof MessageModel } ) { - const message = await channels.getMessageById(id); + const message = await Server.getMessageById(id); if (!message) { return null; } @@ -912,13 +699,13 @@ async function _getAllMessages({ }: { MessageCollection: typeof MessageModelCollectionType; }) { - const messages = await channels._getAllMessages(); + const messages = await Server._getAllMessages(); return new MessageCollection(messages); } async function getAllMessageIds() { - const ids = await channels.getAllMessageIds(); + const ids = await Server.getAllMessageIds(); return ids; } @@ -937,7 +724,7 @@ async function getMessageBySender( }, { Message }: { Message: typeof MessageModel } ) { - const messages = await channels.getMessageBySender({ + const messages = await Server.getMessageBySender({ source, sourceUuid, sourceDevice, @@ -956,7 +743,7 @@ async function getUnreadByConversation( MessageCollection, }: { MessageCollection: typeof MessageModelCollectionType } ) { - const messages = await channels.getUnreadByConversation(conversationId); + const messages = await Server.getUnreadByConversation(conversationId); return new MessageCollection(messages); } @@ -981,15 +768,12 @@ async function getOlderMessagesByConversation( MessageCollection: typeof MessageModelCollectionType; } ) { - const messages = await channels.getOlderMessagesByConversation( - conversationId, - { - limit, - receivedAt, - sentAt, - messageId, - } - ); + const messages = await Server.getOlderMessagesByConversation(conversationId, { + limit, + receivedAt, + sentAt, + messageId, + }); return new MessageCollection(handleMessageJSON(messages)); } @@ -1007,14 +791,11 @@ async function getNewerMessagesByConversation( MessageCollection: typeof MessageModelCollectionType; } ) { - const messages = await channels.getNewerMessagesByConversation( - conversationId, - { - limit, - receivedAt, - sentAt, - } - ); + const messages = await Server.getNewerMessagesByConversation(conversationId, { + limit, + receivedAt, + sentAt, + }); return new MessageCollection(handleMessageJSON(messages)); } @@ -1027,7 +808,7 @@ async function getLastConversationActivity({ ourConversationId: string; Message: typeof MessageModel; }): Promise { - const result = await channels.getLastConversationActivity({ + const result = await Server.getLastConversationActivity({ conversationId, ourConversationId, }); @@ -1045,7 +826,7 @@ async function getLastConversationPreview({ ourConversationId: string; Message: typeof MessageModel; }): Promise { - const result = await channels.getLastConversationPreview({ + const result = await Server.getLastConversationPreview({ conversationId, ourConversationId, }); @@ -1055,9 +836,7 @@ async function getLastConversationPreview({ return undefined; } async function getMessageMetricsForConversation(conversationId: string) { - const result = await channels.getMessageMetricsForConversation( - conversationId - ); + const result = await Server.getMessageMetricsForConversation(conversationId); return result; } @@ -1065,13 +844,13 @@ function hasGroupCallHistoryMessage( conversationId: string, eraId: string ): Promise { - return channels.hasGroupCallHistoryMessage(conversationId, eraId); + return Server.hasGroupCallHistoryMessage(conversationId, eraId); } async function migrateConversationMessages( obsoleteId: string, currentId: string ) { - await channels.migrateConversationMessages(obsoleteId, currentId); + await Server.migrateConversationMessages(obsoleteId, currentId); } async function removeAllMessagesInConversation( @@ -1113,7 +892,7 @@ async function removeAllMessagesInConversation( await queue.onIdle(); window.log.info(`removeAllMessagesInConversation/${logId}: Deleting...`); - await channels.removeMessages(ids); + await Server.removeMessages(ids); } while (messages.length > 0); } @@ -1123,7 +902,7 @@ async function getMessagesBySentAt( MessageCollection, }: { MessageCollection: typeof MessageModelCollectionType } ) { - const messages = await channels.getMessagesBySentAt(sentAt); + const messages = await Server.getMessagesBySentAt(sentAt); return new MessageCollection(messages); } @@ -1133,7 +912,7 @@ async function getExpiredMessages({ }: { MessageCollection: typeof MessageModelCollectionType; }) { - const messages = await channels.getExpiredMessages(); + const messages = await Server.getExpiredMessages(); return new MessageCollection(messages); } @@ -1143,7 +922,7 @@ async function getOutgoingWithoutExpiresAt({ }: { MessageCollection: typeof MessageModelCollectionType; }) { - const messages = await channels.getOutgoingWithoutExpiresAt(); + const messages = await Server.getOutgoingWithoutExpiresAt(); return new MessageCollection(messages); } @@ -1153,7 +932,7 @@ async function getNextExpiringMessage({ }: { Message: typeof MessageModel; }) { - const message = await channels.getNextExpiringMessage(); + const message = await Server.getNextExpiringMessage(); if (message) { return new Message(message); @@ -1167,7 +946,7 @@ async function getNextTapToViewMessageToAgeOut({ }: { Message: typeof MessageModel; }) { - const message = await channels.getNextTapToViewMessageToAgeOut(); + const message = await Server.getNextTapToViewMessageToAgeOut(); if (!message) { return null; } @@ -1179,7 +958,7 @@ async function getTapToViewMessagesNeedingErase({ }: { MessageCollection: typeof MessageModelCollectionType; }) { - const messages = await channels.getTapToViewMessagesNeedingErase(); + const messages = await Server.getTapToViewMessagesNeedingErase(); return new MessageCollection(messages); } @@ -1187,22 +966,22 @@ async function getTapToViewMessagesNeedingErase({ // Unprocessed async function getUnprocessedCount() { - return channels.getUnprocessedCount(); + return Server.getUnprocessedCount(); } async function getAllUnprocessed() { - return channels.getAllUnprocessed(); + return Server.getAllUnprocessed(); } async function getUnprocessedById(id: string) { - return channels.getUnprocessedById(id); + return Server.getUnprocessedById(id); } async function saveUnprocessed( data: UnprocessedType, { forceSave }: { forceSave?: boolean } = {} ) { - const id = await channels.saveUnprocessed(_cleanData(data), { forceSave }); + const id = await Server.saveUnprocessed(_cleanData(data), { forceSave }); return id; } @@ -1211,27 +990,27 @@ async function saveUnprocesseds( arrayOfUnprocessed: Array, { forceSave }: { forceSave?: boolean } = {} ) { - await channels.saveUnprocesseds(_cleanData(arrayOfUnprocessed), { + await Server.saveUnprocesseds(_cleanData(arrayOfUnprocessed), { forceSave, }); } async function updateUnprocessedAttempts(id: string, attempts: number) { - await channels.updateUnprocessedAttempts(id, attempts); + await Server.updateUnprocessedAttempts(id, attempts); } async function updateUnprocessedWithData(id: string, data: UnprocessedType) { - await channels.updateUnprocessedWithData(id, data); + await Server.updateUnprocessedWithData(id, data); } async function updateUnprocessedsWithData(array: Array) { - await channels.updateUnprocessedsWithData(array); + await Server.updateUnprocessedsWithData(array); } async function removeUnprocessed(id: string | Array) { - await channels.removeUnprocessed(id); + await Server.removeUnprocessed(id); } async function removeAllUnprocessed() { - await channels.removeAllUnprocessed(); + await Server.removeAllUnprocessed(); } // Attachment downloads @@ -1240,98 +1019,98 @@ async function getNextAttachmentDownloadJobs( limit?: number, options?: { timestamp?: number } ) { - return channels.getNextAttachmentDownloadJobs(limit, options); + return Server.getNextAttachmentDownloadJobs(limit, options); } async function saveAttachmentDownloadJob(job: AttachmentDownloadJobType) { - await channels.saveAttachmentDownloadJob(_cleanData(job)); + await Server.saveAttachmentDownloadJob(_cleanData(job)); } async function setAttachmentDownloadJobPending(id: string, pending: boolean) { - await channels.setAttachmentDownloadJobPending(id, pending); + await Server.setAttachmentDownloadJobPending(id, pending); } async function resetAttachmentDownloadPending() { - await channels.resetAttachmentDownloadPending(); + await Server.resetAttachmentDownloadPending(); } async function removeAttachmentDownloadJob(id: string) { - await channels.removeAttachmentDownloadJob(id); + await Server.removeAttachmentDownloadJob(id); } async function removeAllAttachmentDownloadJobs() { - await channels.removeAllAttachmentDownloadJobs(); + await Server.removeAllAttachmentDownloadJobs(); } // Stickers async function getStickerCount() { - return channels.getStickerCount(); + return Server.getStickerCount(); } async function createOrUpdateStickerPack(pack: StickerPackType) { - await channels.createOrUpdateStickerPack(pack); + await Server.createOrUpdateStickerPack(pack); } async function updateStickerPackStatus( packId: string, status: StickerPackStatusType, options?: { timestamp: number } ) { - await channels.updateStickerPackStatus(packId, status, options); + await Server.updateStickerPackStatus(packId, status, options); } async function createOrUpdateSticker(sticker: StickerType) { - await channels.createOrUpdateSticker(sticker); + await Server.createOrUpdateSticker(sticker); } async function updateStickerLastUsed( packId: string, stickerId: number, timestamp: number ) { - await channels.updateStickerLastUsed(packId, stickerId, timestamp); + await Server.updateStickerLastUsed(packId, stickerId, timestamp); } async function addStickerPackReference(messageId: string, packId: string) { - await channels.addStickerPackReference(messageId, packId); + await Server.addStickerPackReference(messageId, packId); } async function deleteStickerPackReference(messageId: string, packId: string) { - const paths = await channels.deleteStickerPackReference(messageId, packId); + const paths = await Server.deleteStickerPackReference(messageId, packId); return paths; } async function deleteStickerPack(packId: string) { - const paths = await channels.deleteStickerPack(packId); + const paths = await Server.deleteStickerPack(packId); return paths; } async function getAllStickerPacks() { - const packs = await channels.getAllStickerPacks(); + const packs = await Server.getAllStickerPacks(); return packs; } async function getAllStickers() { - const stickers = await channels.getAllStickers(); + const stickers = await Server.getAllStickers(); return stickers; } async function getRecentStickers() { - const recentStickers = await channels.getRecentStickers(); + const recentStickers = await Server.getRecentStickers(); return recentStickers; } async function clearAllErrorStickerPackAttempts() { - await channels.clearAllErrorStickerPackAttempts(); + await Server.clearAllErrorStickerPackAttempts(); } // Emojis async function updateEmojiUsage(shortName: string) { - await channels.updateEmojiUsage(shortName); + await Server.updateEmojiUsage(shortName); } async function getRecentEmojis(limit = 32) { - return channels.getRecentEmojis(limit); + return Server.getRecentEmojis(limit); } // Other async function removeAll() { - await channels.removeAll(); + await Server.removeAll(); } async function removeAllConfiguration() { - await channels.removeAllConfiguration(); + await Server.removeAllConfiguration(); } async function cleanupOrphanedAttachments() { @@ -1376,7 +1155,7 @@ async function getMessagesNeedingUpgrade( limit: number, { maxVersion = CURRENT_SCHEMA_VERSION }: { maxVersion: number } ) { - const messages = await channels.getMessagesNeedingUpgrade(limit, { + const messages = await Server.getMessagesNeedingUpgrade(limit, { maxVersion, }); @@ -1387,7 +1166,7 @@ async function getMessagesWithVisualMediaAttachments( conversationId: string, { limit }: { limit: number } ) { - return channels.getMessagesWithVisualMediaAttachments(conversationId, { + return Server.getMessagesWithVisualMediaAttachments(conversationId, { limit, }); } @@ -1396,7 +1175,7 @@ async function getMessagesWithFileAttachments( conversationId: string, { limit }: { limit: number } ) { - return channels.getMessagesWithFileAttachments(conversationId, { + return Server.getMessagesWithFileAttachments(conversationId, { limit, }); } diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 848f37ea3..4454deddb 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -259,7 +259,12 @@ export type ServerInterface = DataInterface & { configDir: string; key: string; messages: LocaleMessagesType; - }) => Promise; + }) => Promise; + + initializeRenderer: (options: { + configDir: string; + key: string; + }) => Promise; removeKnownAttachments: ( allAttachments: Array @@ -393,7 +398,6 @@ export type ClientInterface = DataInterface & { // Client-side only, and test-only _removeConversations: (ids: Array) => Promise; - _jobs: { [id: string]: ClientJobType }; }; export type ClientJobType = { diff --git a/ts/sql/Queueing.ts b/ts/sql/Queueing.ts new file mode 100644 index 000000000..445a1a7d9 --- /dev/null +++ b/ts/sql/Queueing.ts @@ -0,0 +1,141 @@ +// Copyright 2018-2020 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import Queue from 'p-queue'; +import { ServerInterface } from './Interface'; + +let allQueriesDone: () => void | undefined; +let sqlQueries = 0; +let singleQueue: Queue | null = null; +let multipleQueue: Queue | null = null; + +// Note: we don't want queue timeouts, because delays here are due to in-progress sql +// operations. For example we might try to start a transaction when the prevous isn't +// done, causing that database operation to fail. +function makeNewSingleQueue(): Queue { + singleQueue = new Queue({ concurrency: 1 }); + return singleQueue; +} +function makeNewMultipleQueue(): Queue { + multipleQueue = new Queue({ concurrency: 10 }); + return multipleQueue; +} + +const DEBUG = false; + +function makeSQLJob( + fn: ServerInterface[keyof ServerInterface], + args: Array, + callName: keyof ServerInterface +) { + if (DEBUG) { + // eslint-disable-next-line no-console + console.log(`SQL(${callName}) queued`); + } + return async () => { + sqlQueries += 1; + const start = Date.now(); + if (DEBUG) { + // eslint-disable-next-line no-console + console.log(`SQL(${callName}) started`); + } + let result; + try { + // Ignoring this error TS2556: Expected 3 arguments, but got 0 or more. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + result = await fn(...args); + } finally { + sqlQueries -= 1; + if (allQueriesDone && sqlQueries <= 0) { + allQueriesDone(); + } + } + const end = Date.now(); + const delta = end - start; + if (DEBUG || delta > 10) { + // eslint-disable-next-line no-console + console.log(`SQL(${callName}) succeeded in ${end - start}ms`); + } + return result; + }; +} + +async function handleCall( + fn: ServerInterface[keyof ServerInterface], + args: Array, + callName: keyof ServerInterface +) { + if (!fn) { + throw new Error(`sql channel: ${callName} is not an available function`); + } + + let result; + + // We queue here to keep multi-query operations atomic. Without it, any multistage + // data operation (even within a BEGIN/COMMIT) can become interleaved, since all + // requests share one database connection. + + // A needsSerial method must be run in our single concurrency queue. + if (fn.needsSerial) { + if (singleQueue) { + result = await singleQueue.add(makeSQLJob(fn, args, callName)); + } else if (multipleQueue) { + const queue = makeNewSingleQueue(); + + const multipleQueueLocal = multipleQueue; + queue.add(() => multipleQueueLocal.onIdle()); + multipleQueue = null; + + result = await queue.add(makeSQLJob(fn, args, callName)); + } else { + const queue = makeNewSingleQueue(); + result = await queue.add(makeSQLJob(fn, args, callName)); + } + } else { + // The request can be parallelized. To keep the same structure as the above block + // we force this section into the 'lonely if' pattern. + // eslint-disable-next-line no-lonely-if + if (multipleQueue) { + result = await multipleQueue.add(makeSQLJob(fn, args, callName)); + } else if (singleQueue) { + const queue = makeNewMultipleQueue(); + queue.pause(); + + const singleQueueRef = singleQueue; + + singleQueue = null; + const promise = queue.add(makeSQLJob(fn, args, callName)); + if (singleQueueRef) { + await singleQueueRef.onIdle(); + } + + queue.start(); + result = await promise; + } else { + const queue = makeNewMultipleQueue(); + result = await queue.add(makeSQLJob(fn, args, callName)); + } + } + + return result; +} + +export async function waitForPendingQueries(): Promise { + return new Promise(resolve => { + if (sqlQueries === 0) { + resolve(); + } else { + allQueriesDone = () => resolve(); + } + }); +} + +export function applyQueueing(dataInterface: ServerInterface): ServerInterface { + return Object.keys(dataInterface).reduce((acc, callName) => { + const serverInterfaceKey = callName as keyof ServerInterface; + acc[serverInterfaceKey] = async (...args: Array) => + handleCall(dataInterface[serverInterfaceKey], args, serverInterfaceKey); + return acc; + }, {} as ServerInterface); +} diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 97b8bacd6..6b5442a6a 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -14,7 +14,6 @@ import mkdirp from 'mkdirp'; import rimraf from 'rimraf'; import PQueue from 'p-queue'; import sql from '@journeyapps/sqlcipher'; -import { app, clipboard, dialog } from 'electron'; import pify from 'pify'; import { v4 as generateUUID } from 'uuid'; @@ -31,8 +30,6 @@ import { pick, } from 'lodash'; -import { redactAll } from '../../js/modules/privacy'; -import { remove as removeUserConfig } from '../../app/user_config'; import { combineNames } from '../util/combineNames'; import { GroupV2MemberType } from '../model-types.d'; @@ -54,6 +51,7 @@ import { StickerType, UnprocessedType, } from './Interface'; +import { applyQueueing } from './Queueing'; declare global { // We want to extend `Function`'s properties, so we need to use an interface. @@ -195,13 +193,14 @@ const dataInterface: ServerInterface = { // Server-only initialize, + initializeRenderer, removeKnownAttachments, removeKnownStickers, removeKnownDraftAttachments, }; -export default dataInterface; +export default applyQueueing(dataInterface); function objectToJSON(data: any) { return JSON.stringify(data); @@ -210,6 +209,14 @@ function jsonToObject(json: string): any { return JSON.parse(json); } +function isRenderer() { + if (typeof process === 'undefined' || !process) { + return true; + } + + return process.type === 'renderer'; +} + async function openDatabase(filePath: string): Promise { return new Promise((resolve, reject) => { let instance: sql.Database | undefined; @@ -1702,6 +1709,7 @@ async function updateSchema(instance: PromisifiedSQLDatabase) { } let globalInstance: PromisifiedSQLDatabase | undefined; +let globalInstanceRenderer: PromisifiedSQLDatabase | undefined; let databaseFilePath: string | undefined; let indexedDBPath: string | undefined; @@ -1788,37 +1796,57 @@ async function initialize({ await getMessageCount(); } catch (error) { console.log('Database startup error:', error.stack); - const buttonIndex = dialog.showMessageBoxSync({ - buttons: [ - messages.copyErrorAndQuit.message, - messages.deleteAndRestart.message, - ], - defaultId: 0, - detail: redactAll(error.stack), - message: messages.databaseError.message, - noLink: true, - type: 'error', - }); - - if (buttonIndex === 0) { - clipboard.writeText( - `Database startup error:\n\n${redactAll(error.stack)}` - ); - } else { - if (promisified) { - await promisified.close(); - } - await removeDB(); - removeUserConfig(); - app.relaunch(); + if (promisified) { + await promisified.close(); } + throw error; + } +} - app.exit(1); - - return false; +async function initializeRenderer({ + configDir, + key, +}: { + configDir: string; + key: string; +}) { + if (!isRenderer()) { + throw new Error('Cannot call from main process.'); + } + if (globalInstanceRenderer) { + throw new Error('Cannot initialize more than once!'); + } + if (!isString(configDir)) { + throw new Error('initialize: configDir is required!'); + } + if (!isString(key)) { + throw new Error('initialize: key is required!'); } - return true; + if (!indexedDBPath) { + indexedDBPath = join(configDir, 'IndexedDB'); + } + + const dbDir = join(configDir, 'sql'); + + if (!databaseFilePath) { + databaseFilePath = join(dbDir, 'db.sqlite'); + } + + let promisified: PromisifiedSQLDatabase | undefined; + + try { + promisified = await openAndSetUpSQLCipher(databaseFilePath, { key }); + + // At this point we can allow general access to the database + globalInstanceRenderer = promisified; + + // test database + await getMessageCount(); + } catch (error) { + window.log.error('Database startup error:', error.stack); + throw error; + } } async function close() { @@ -1857,6 +1885,13 @@ async function removeIndexedDBFiles() { } function getInstance(): PromisifiedSQLDatabase { + if (isRenderer()) { + if (!globalInstanceRenderer) { + throw new Error('getInstance: globalInstanceRenderer not set!'); + } + return globalInstanceRenderer; + } + if (!globalInstance) { throw new Error('getInstance: globalInstance not set!'); } @@ -2285,6 +2320,7 @@ async function updateConversation(data: ConversationType) { } ); } + async function updateConversations(array: Array) { const db = getInstance(); await db.run('BEGIN TRANSACTION;'); @@ -2544,7 +2580,7 @@ async function saveMessage( `UPDATE messages SET id = $id, json = $json, - + body = $body, conversationId = $conversationId, expirationStartTimestamp = $expirationStartTimestamp, @@ -2616,7 +2652,7 @@ async function saveMessage( `INSERT INTO messages ( id, json, - + body, conversationId, expirationStartTimestamp, @@ -2638,7 +2674,7 @@ async function saveMessage( ) values ( $id, $json, - + $body, $conversationId, $expirationStartTimestamp, @@ -2967,7 +3003,7 @@ async function getLastConversationActivity({ const row = await db.get( `SELECT * FROM messages WHERE conversationId = $conversationId AND - (type IS NULL + (type IS NULL OR type NOT IN ( 'profile-change', diff --git a/ts/sql/initialize.ts b/ts/sql/initialize.ts new file mode 100644 index 000000000..e7b749370 --- /dev/null +++ b/ts/sql/initialize.ts @@ -0,0 +1,16 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { ipcRenderer as ipc } from 'electron'; +import fs from 'fs-extra'; +import pify from 'pify'; +import sql from './Server'; + +const getRealPath = pify(fs.realpath); + +export async function initialize(): Promise { + const configDir = await getRealPath(ipc.sendSync('get-user-data-path')); + const key = ipc.sendSync('user-config-key'); + + await sql.initializeRenderer({ configDir, key }); +} diff --git a/ts/test-electron/models/messages_test.ts b/ts/test-electron/models/messages_test.ts index 9dafe9896..b2d232cba 100644 --- a/ts/test-electron/models/messages_test.ts +++ b/ts/test-electron/models/messages_test.ts @@ -23,7 +23,10 @@ describe('Message', () => { function createMessage(attrs: { [key: string]: unknown }) { const messages = new window.Whisper.MessageCollection(); - return messages.add(attrs); + return messages.add({ + received_at: Date.now(), + ...attrs, + }); } before(async () => { diff --git a/ts/test-node/components/media-gallery/groupMessagesByDate_test.ts b/ts/test-node/components/media-gallery/groupMessagesByDate_test.ts index 8974923d4..9b6af88f5 100644 --- a/ts/test-node/components/media-gallery/groupMessagesByDate_test.ts +++ b/ts/test-node/components/media-gallery/groupMessagesByDate_test.ts @@ -17,6 +17,7 @@ const toMediaItem = (date: Date): MediaItemType => ({ message: { id: 'id', received_at: date.getTime(), + received_at_ms: date.getTime(), attachments: [], }, attachment: { @@ -57,6 +58,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1523534400000, + received_at_ms: 1523534400000, attachments: [], }, attachment: { @@ -71,6 +73,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1523491260000, + received_at_ms: 1523491260000, attachments: [], }, attachment: { @@ -90,6 +93,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1523491140000, + received_at_ms: 1523491140000, attachments: [], }, attachment: { @@ -109,6 +113,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1523232060000, + received_at_ms: 1523232060000, attachments: [], }, attachment: { @@ -128,6 +133,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1523231940000, + received_at_ms: 1523231940000, attachments: [], }, attachment: { @@ -142,6 +148,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1522540860000, + received_at_ms: 1522540860000, attachments: [], }, attachment: { @@ -163,6 +170,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1522540740000, + received_at_ms: 1522540740000, attachments: [], }, attachment: { @@ -177,6 +185,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1519912800000, + received_at_ms: 1519912800000, attachments: [], }, attachment: { @@ -198,6 +207,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1298937540000, + received_at_ms: 1298937540000, attachments: [], }, attachment: { @@ -212,6 +222,7 @@ describe('groupMediaItemsByDate', () => { message: { id: 'id', received_at: 1296554400000, + received_at_ms: 1296554400000, attachments: [], }, attachment: { diff --git a/ts/textsecure.d.ts b/ts/textsecure.d.ts index 33a6c607d..ec9c3556b 100644 --- a/ts/textsecure.d.ts +++ b/ts/textsecure.d.ts @@ -23,6 +23,7 @@ export type UnprocessedType = { decrypted?: string; envelope?: string; id: string; + timestamp: number; serverTimestamp?: number; source?: string; sourceDevice?: number; @@ -795,6 +796,8 @@ export declare class EnvelopeClass { // Note: these additional properties are added in the course of processing id: string; + receivedAtCounter: number; + receivedAtDate: number; unidentifiedDeliveryReceived?: boolean; messageAgeSec?: number; } diff --git a/ts/textsecure/MessageReceiver.ts b/ts/textsecure/MessageReceiver.ts index bf38c4942..899ea1be6 100644 --- a/ts/textsecure/MessageReceiver.ts +++ b/ts/textsecure/MessageReceiver.ts @@ -203,7 +203,7 @@ class MessageReceiverInner extends EventTarget { this.cacheAddBatcher = createBatcher({ wait: 200, maxSize: 30, - processBatch: this.cacheAndQueueBatch.bind(this), + processBatch: this.cacheAndHandleBatch.bind(this), }); this.cacheUpdateBatcher = createBatcher({ wait: 500, @@ -237,7 +237,7 @@ class MessageReceiverInner extends EventTarget { } // We always process our cache before processing a new websocket message - this.pendingQueue.add(async () => this.queueAllCached()); + this.pendingQueue.add(async () => this.handleAllCached()); this.count = 0; if (this.hasConnected) { @@ -428,13 +428,16 @@ class MessageReceiverInner extends EventTarget { ? envelope.serverTimestamp.toNumber() : null; + envelope.receivedAtCounter = window.Signal.Util.incrementMessageCounter(); + envelope.receivedAtDate = Date.now(); + // Calculate the message age (time on server). envelope.messageAgeSec = this.calculateMessageAge( headers, envelope.serverTimestamp ); - this.cacheAndQueue(envelope, plaintext, request); + this.cacheAndHandle(envelope, plaintext, request); } catch (e) { request.respond(500, 'Bad encrypted websocket message'); window.log.error( @@ -553,16 +556,17 @@ class MessageReceiverInner extends EventTarget { this.dispatchEvent(ev); } - async queueAllCached() { + async handleAllCached() { const items = await this.getAllFromCache(); const max = items.length; for (let i = 0; i < max; i += 1) { // eslint-disable-next-line no-await-in-loop - await this.queueCached(items[i]); + await this.handleCachedEnvelope(items[i]); } } - async queueCached(item: UnprocessedType) { + async handleCachedEnvelope(item: UnprocessedType) { + window.log.info('MessageReceiver.handleCachedEnvelope', item.id); try { let envelopePlaintext: ArrayBuffer; @@ -576,7 +580,7 @@ class MessageReceiverInner extends EventTarget { ); } else { throw new Error( - 'MessageReceiver.queueCached: item.envelope was malformed' + 'MessageReceiver.handleCachedEnvelope: item.envelope was malformed' ); } @@ -584,6 +588,8 @@ class MessageReceiverInner extends EventTarget { envelopePlaintext ); envelope.id = item.id; + envelope.receivedAtCounter = item.timestamp; + envelope.receivedAtDate = Date.now(); envelope.source = envelope.source || item.source; envelope.sourceUuid = envelope.sourceUuid || item.sourceUuid; envelope.sourceDevice = envelope.sourceDevice || item.sourceDevice; @@ -605,13 +611,13 @@ class MessageReceiverInner extends EventTarget { } else { throw new Error('Cached decrypted value was not a string!'); } - this.queueDecryptedEnvelope(envelope, payloadPlaintext); + this.handleDecryptedEnvelope(envelope, payloadPlaintext); } else { - this.queueEnvelope(envelope); + this.handleEnvelope(envelope); } } catch (error) { window.log.error( - 'queueCached error handling item', + 'handleCachedEnvelope error handling item', item.id, 'removing it. Error:', error && error.stack ? error.stack : error @@ -622,7 +628,7 @@ class MessageReceiverInner extends EventTarget { await window.textsecure.storage.unprocessed.remove(id); } catch (deleteError) { window.log.error( - 'queueCached error deleting item', + 'handleCachedEnvelope error deleting item', item.id, 'Error:', deleteError && deleteError.stack ? deleteError.stack : deleteError @@ -656,7 +662,7 @@ class MessageReceiverInner extends EventTarget { if (this.isEmptied) { this.clearRetryTimeout(); this.retryCachedTimeout = setTimeout(() => { - this.pendingQueue.add(async () => this.queueAllCached()); + this.pendingQueue.add(async () => this.handleAllCached()); }, RETRY_TIMEOUT); } } @@ -705,7 +711,8 @@ class MessageReceiverInner extends EventTarget { ); } - async cacheAndQueueBatch(items: Array) { + async cacheAndHandleBatch(items: Array) { + window.log.info('MessageReceiver.cacheAndHandleBatch', items.length); const dataArray = items.map(item => item.data); try { await window.textsecure.storage.unprocessed.batchAdd(dataArray); @@ -714,16 +721,16 @@ class MessageReceiverInner extends EventTarget { item.request.respond(200, 'OK'); } catch (error) { window.log.error( - 'cacheAndQueueBatch: Failed to send 200 to server; still queuing envelope' + 'cacheAndHandleBatch: Failed to send 200 to server; still queuing envelope' ); } - this.queueEnvelope(item.envelope); + this.handleEnvelope(item.envelope); }); this.maybeScheduleRetryTimeout(); } catch (error) { window.log.error( - 'cacheAndQueue error trying to add messages to cache:', + 'cacheAndHandleBatch error trying to add messages to cache:', error && error.stack ? error.stack : error ); @@ -733,7 +740,7 @@ class MessageReceiverInner extends EventTarget { } } - cacheAndQueue( + cacheAndHandle( envelope: EnvelopeClass, plaintext: ArrayBuffer, request: IncomingWebSocketRequest @@ -743,7 +750,7 @@ class MessageReceiverInner extends EventTarget { id, version: 2, envelope: MessageReceiverInner.arrayBufferToStringBase64(plaintext), - timestamp: Date.now(), + timestamp: envelope.receivedAtCounter, attempts: 1, }; this.cacheAddBatcher.add({ @@ -754,6 +761,7 @@ class MessageReceiverInner extends EventTarget { } async cacheUpdateBatch(items: Array>) { + window.log.info('MessageReceiver.cacheUpdateBatch', items.length); await window.textsecure.storage.unprocessed.addDecryptedDataToList(items); } @@ -778,43 +786,71 @@ class MessageReceiverInner extends EventTarget { this.cacheRemoveBatcher.add(id); } - async queueDecryptedEnvelope( + // Same as handleEnvelope, just without the decryption step. Necessary for handling + // messages which were successfully decrypted, but application logic didn't finish + // processing. + async handleDecryptedEnvelope( envelope: EnvelopeClass, plaintext: ArrayBuffer - ) { + ): Promise { const id = this.getEnvelopeId(envelope); - window.log.info('queueing decrypted envelope', id); + window.log.info('MessageReceiver.handleDecryptedEnvelope', id); - const task = this.handleDecryptedEnvelope.bind(this, envelope, plaintext); - const taskWithTimeout = window.textsecure.createTaskWithTimeout( - task, - `queueEncryptedEnvelope ${id}` - ); - const promise = this.addToQueue(taskWithTimeout); + try { + if (this.stoppingProcessing) { + return; + } + // No decryption is required for delivery receipts, so the decrypted field of + // the Unprocessed model will never be set - return promise.catch(error => { + if (envelope.content) { + await this.innerHandleContentMessage(envelope, plaintext); + + return; + } + if (envelope.legacyMessage) { + await this.innerHandleLegacyMessage(envelope, plaintext); + + return; + } + + this.removeFromCache(envelope); + throw new Error('Received message with no content and no legacyMessage'); + } catch (error) { window.log.error( - `queueDecryptedEnvelope error handling envelope ${id}:`, + `handleDecryptedEnvelope error handling envelope ${id}:`, error && error.extra ? JSON.stringify(error.extra) : '', error && error.stack ? error.stack : error ); - }); + } } - async queueEnvelope(envelope: EnvelopeClass) { + async handleEnvelope(envelope: EnvelopeClass) { const id = this.getEnvelopeId(envelope); - window.log.info('queueing envelope', id); + window.log.info('MessageReceiver.handleEnvelope', id); - const task = this.handleEnvelope.bind(this, envelope); - const taskWithTimeout = window.textsecure.createTaskWithTimeout( - task, - `queueEnvelope ${id}` - ); - const promise = this.addToQueue(taskWithTimeout); + try { + if (this.stoppingProcessing) { + return Promise.resolve(); + } - return promise.catch(error => { + if (envelope.type === window.textsecure.protobuf.Envelope.Type.RECEIPT) { + return this.onDeliveryReceipt(envelope); + } + + if (envelope.content) { + return this.handleContentMessage(envelope); + } + if (envelope.legacyMessage) { + return this.handleLegacyMessage(envelope); + } + + this.removeFromCache(envelope); + + throw new Error('Received message with no content and no legacyMessage'); + } catch (error) { const args = [ - 'queueEnvelope error handling envelope', + 'handleEnvelope error handling envelope', this.getEnvelopeId(envelope), ':', error && error.extra ? JSON.stringify(error.extra) : '', @@ -825,54 +861,9 @@ class MessageReceiverInner extends EventTarget { } else { window.log.error(...args); } - }); - } - - // Same as handleEnvelope, just without the decryption step. Necessary for handling - // messages which were successfully decrypted, but application logic didn't finish - // processing. - async handleDecryptedEnvelope( - envelope: EnvelopeClass, - plaintext: ArrayBuffer - ): Promise { - if (this.stoppingProcessing) { - return; - } - // No decryption is required for delivery receipts, so the decrypted field of - // the Unprocessed model will never be set - - if (envelope.content) { - await this.innerHandleContentMessage(envelope, plaintext); - - return; - } - if (envelope.legacyMessage) { - await this.innerHandleLegacyMessage(envelope, plaintext); - - return; } - this.removeFromCache(envelope); - throw new Error('Received message with no content and no legacyMessage'); - } - - async handleEnvelope(envelope: EnvelopeClass) { - if (this.stoppingProcessing) { - return Promise.resolve(); - } - - if (envelope.type === window.textsecure.protobuf.Envelope.Type.RECEIPT) { - return this.onDeliveryReceipt(envelope); - } - - if (envelope.content) { - return this.handleContentMessage(envelope); - } - if (envelope.legacyMessage) { - return this.handleLegacyMessage(envelope); - } - this.removeFromCache(envelope); - throw new Error('Received message with no content and no legacyMessage'); + return undefined; } getStatus() { @@ -1257,6 +1248,10 @@ class MessageReceiverInner extends EventTarget { envelope: EnvelopeClass, sentContainer: SyncMessageClass.Sent ) { + window.log.info( + 'MessageReceiver.handleSentMessage', + this.getEnvelopeId(envelope) + ); const { destination, destinationUuid, @@ -1324,6 +1319,8 @@ class MessageReceiverInner extends EventTarget { unidentifiedStatus, message, isRecipientUpdate, + receivedAtCounter: envelope.receivedAtCounter, + receivedAtDate: envelope.receivedAtDate, }; if (expirationStartTimestamp) { ev.data.expirationStartTimestamp = expirationStartTimestamp.toNumber(); @@ -1334,7 +1331,10 @@ class MessageReceiverInner extends EventTarget { } async handleDataMessage(envelope: EnvelopeClass, msg: DataMessageClass) { - window.log.info('data message from', this.getEnvelopeId(envelope)); + window.log.info( + 'MessageReceiver.handleDataMessage', + this.getEnvelopeId(envelope) + ); let p: Promise = Promise.resolve(); // eslint-disable-next-line no-bitwise const destination = envelope.sourceUuid || envelope.source; @@ -1412,6 +1412,8 @@ class MessageReceiverInner extends EventTarget { serverTimestamp: envelope.serverTimestamp, unidentifiedDeliveryReceived: envelope.unidentifiedDeliveryReceived, message, + receivedAtCounter: envelope.receivedAtCounter, + receivedAtDate: envelope.receivedAtDate, }; return this.dispatchAndWait(ev); }) @@ -1419,6 +1421,10 @@ class MessageReceiverInner extends EventTarget { } async handleLegacyMessage(envelope: EnvelopeClass) { + window.log.info( + 'MessageReceiver.handleLegacyMessage', + this.getEnvelopeId(envelope) + ); return this.decrypt(envelope, envelope.legacyMessage).then(plaintext => { if (!plaintext) { window.log.warn('handleLegacyMessage: plaintext was falsey'); @@ -1437,6 +1443,10 @@ class MessageReceiverInner extends EventTarget { } async handleContentMessage(envelope: EnvelopeClass) { + window.log.info( + 'MessageReceiver.handleContentMessage', + this.getEnvelopeId(envelope) + ); return this.decrypt(envelope, envelope.content).then(plaintext => { if (!plaintext) { window.log.warn('handleContentMessage: plaintext was falsey'); @@ -1579,7 +1589,10 @@ class MessageReceiverInner extends EventTarget { } handleNullMessage(envelope: EnvelopeClass) { - window.log.info('null message from', this.getEnvelopeId(envelope)); + window.log.info( + 'MessageReceiver.handleNullMessage', + this.getEnvelopeId(envelope) + ); this.removeFromCache(envelope); } @@ -1778,7 +1791,6 @@ class MessageReceiverInner extends EventTarget { return undefined; } if (syncMessage.read && syncMessage.read.length) { - window.log.info('read messages from', this.getEnvelopeId(envelope)); return this.handleRead(envelope, syncMessage.read); } if (syncMessage.verified) { @@ -1952,6 +1964,7 @@ class MessageReceiverInner extends EventTarget { envelope: EnvelopeClass, read: Array ) { + window.log.info('MessageReceiver.handleRead', this.getEnvelopeId(envelope)); const results = []; for (let i = 0; i < read.length; i += 1) { const ev = new Event('readSync'); diff --git a/ts/util/getMessageTimestamp.ts b/ts/util/getMessageTimestamp.ts new file mode 100644 index 000000000..ed57415fa --- /dev/null +++ b/ts/util/getMessageTimestamp.ts @@ -0,0 +1,8 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { Message } from '../components/conversation/media-gallery/types/Message'; + +export function getMessageTimestamp(message: Message): number { + return message.received_at_ms || message.received_at; +} diff --git a/ts/util/incrementMessageCounter.ts b/ts/util/incrementMessageCounter.ts new file mode 100644 index 000000000..ab910d45f --- /dev/null +++ b/ts/util/incrementMessageCounter.ts @@ -0,0 +1,23 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { debounce } from 'lodash'; + +export function incrementMessageCounter(): number { + if (!window.receivedAtCounter) { + window.receivedAtCounter = + Number(localStorage.getItem('lastReceivedAtCounter')) || Date.now(); + } + + window.receivedAtCounter += 1; + debouncedUpdateLastReceivedAt(); + + return window.receivedAtCounter; +} + +const debouncedUpdateLastReceivedAt = debounce(() => { + localStorage.setItem( + 'lastReceivedAtCounter', + String(window.receivedAtCounter) + ); +}, 500); diff --git a/ts/util/index.ts b/ts/util/index.ts index c550e7e21..fb75f1992 100644 --- a/ts/util/index.ts +++ b/ts/util/index.ts @@ -14,8 +14,10 @@ import { getStringForProfileChange } from './getStringForProfileChange'; import { getTextWithMentions } from './getTextWithMentions'; import { getUserAgent } from './getUserAgent'; import { hasExpired } from './hasExpired'; +import { incrementMessageCounter } from './incrementMessageCounter'; import { isFileDangerous } from './isFileDangerous'; import { makeLookup } from './makeLookup'; +import { saveNewMessageBatcher, updateMessageBatcher } from './messageBatcher'; import { missingCaseError } from './missingCaseError'; import { parseRemoteClientExpiration } from './parseRemoteClientExpiration'; import { sleep } from './sleep'; @@ -29,6 +31,8 @@ import { import * as zkgroup from './zkgroup'; export { + GoogleChrome, + Registration, arrayBufferToObjectURL, combineNames, createBatcher, @@ -40,18 +44,19 @@ export { getStringForProfileChange, getTextWithMentions, getUserAgent, - GoogleChrome, hasExpired, + incrementMessageCounter, isFileDangerous, longRunningTaskWrapper, makeLookup, mapToSupportLocale, missingCaseError, parseRemoteClientExpiration, - Registration, + saveNewMessageBatcher, sessionRecordToProtobuf, sessionStructureToArrayBuffer, sleep, toWebSafeBase64, + updateMessageBatcher, zkgroup, }; diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index 64ae6ce97..9ca326dd0 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -15022,7 +15022,7 @@ "rule": "React-createRef", "path": "ts/components/conversation/media-gallery/MediaGallery.js", "line": " this.focusRef = react_1.default.createRef();", - "lineNumber": 31, + "lineNumber": 32, "reasonCategory": "usageTrusted", "updated": "2019-11-01T22:46:33.013Z", "reasonDetail": "Used for setting focus only" @@ -15031,7 +15031,7 @@ "rule": "React-createRef", "path": "ts/components/conversation/media-gallery/MediaGallery.tsx", "line": " public readonly focusRef: React.RefObject = React.createRef();", - "lineNumber": 71, + "lineNumber": 72, "reasonCategory": "usageTrusted", "updated": "2019-11-01T22:46:33.013Z", "reasonDetail": "Used for setting focus only" diff --git a/ts/util/messageBatcher.ts b/ts/util/messageBatcher.ts new file mode 100644 index 000000000..e59d91820 --- /dev/null +++ b/ts/util/messageBatcher.ts @@ -0,0 +1,24 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { MessageAttributesType } from '../model-types.d'; +import { createBatcher } from './batcher'; +import { createWaitBatcher } from './waitBatcher'; + +export const updateMessageBatcher = createBatcher({ + wait: 500, + maxSize: 50, + processBatch: async (messages: Array) => { + window.log.info('updateMessageBatcher', messages.length); + await window.Signal.Data.saveMessages(messages, {}); + }, +}); + +export const saveNewMessageBatcher = createWaitBatcher({ + wait: 500, + maxSize: 30, + processBatch: async (messages: Array) => { + window.log.info('saveNewMessageBatcher', messages.length); + await window.Signal.Data.saveMessages(messages, { forceSave: true }); + }, +}); diff --git a/ts/window.d.ts b/ts/window.d.ts index d2b953b39..759a42db4 100644 --- a/ts/window.d.ts +++ b/ts/window.d.ts @@ -137,10 +137,12 @@ declare global { WhatIsThis: WhatIsThis; + attachmentDownloadQueue: Array; baseAttachmentsPath: string; baseStickersPath: string; baseTempPath: string; dcodeIO: DCodeIOType; + receivedAtCounter: number; enterKeyboardMode: () => void; enterMouseMode: () => void; getAccountManager: () => AccountManager | undefined; @@ -246,6 +248,9 @@ declare global { titleBarDoubleClick: () => void; unregisterForActive: (handler: () => void) => void; updateTrayIcon: (count: number) => void; + sqlInitializer: { + initialize: () => Promise; + }; Backbone: typeof Backbone; Signal: { @@ -561,6 +566,8 @@ export type DCodeIOType = { }; type MessageControllerType = { + findBySender: (sender: string) => MessageModel | null; + findBySentAt: (sentAt: number) => MessageModel | null; register: (id: string, model: MessageModel) => MessageModel; unregister: (id: string) => void; };