diff --git a/package.json b/package.json index 84d30f5bc..e32b0eae0 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "@popperjs/core": "2.11.6", "@react-aria/utils": "3.16.0", "@react-spring/web": "9.5.5", - "@signalapp/better-sqlite3": "8.4.3", + "@signalapp/better-sqlite3": "8.5.1", "@signalapp/libsignal-client": "0.31.0", "@signalapp/ringrtc": "2.31.2", "@signalapp/windows-dummy-keystroke": "1.0.0", diff --git a/ts/background.ts b/ts/background.ts index ddbc0bafc..d81c7e012 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -53,7 +53,6 @@ import { senderCertificateService } from './services/senderCertificate'; import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher'; import * as KeyboardLayout from './services/keyboardLayout'; import * as StorageService from './services/storage'; -import { optimizeFTS } from './services/ftsOptimizer'; import { RoutineProfileRefresher } from './routineProfileRefresh'; import { isOlderThan, toDayMillis } from './util/timestamp'; import { isValidReactionEmoji } from './reactions/isValidReactionEmoji'; @@ -983,8 +982,6 @@ export async function startApp(): Promise { if (newVersion) { await window.Signal.Data.cleanupOrphanedAttachments(); - optimizeFTS(); - drop(window.Signal.Data.ensureFilePermissions()); } diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index 6eaf279cb..ad720a835 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -87,7 +87,6 @@ import { notificationService, } from '../services/notifications'; import { storageServiceUploadJob } from '../services/storage'; -import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { getSendOptions } from '../util/getSendOptions'; import { isConversationAccepted } from '../util/isConversationAccepted'; import { @@ -4760,8 +4759,6 @@ export class ConversationModel extends window.Backbone await window.Signal.Data.removeAllMessagesInConversation(this.id, { logId: this.idForLogging(), }); - - scheduleOptimizeFTS(); } getTitle(options?: { isShort?: boolean }): string { diff --git a/ts/models/messages.ts b/ts/models/messages.ts index 021c2ecc6..8427257e1 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -69,7 +69,6 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus'; import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes'; import { getOwn } from '../util/getOwn'; import { markRead, markViewed } from '../services/MessageUpdater'; -import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { isDirectConversation, isGroup, @@ -1157,8 +1156,6 @@ export class MessageModel extends window.Backbone.Model { } await window.Signal.Data.deleteSentProtoByMessageId(this.id); - - scheduleOptimizeFTS(); } override isEmpty(): boolean { diff --git a/ts/services/expiringMessagesDeletion.ts b/ts/services/expiringMessagesDeletion.ts index ceee06d72..2a538b5cd 100644 --- a/ts/services/expiringMessagesDeletion.ts +++ b/ts/services/expiringMessagesDeletion.ts @@ -8,7 +8,6 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary'; import { sleep } from '../util/sleep'; import { SECOND } from '../util/durations'; import * as Errors from '../types/errors'; -import { scheduleOptimizeFTS } from './ftsOptimizer'; class ExpiringMessagesDeletionService { public update: typeof this.checkExpiringMessages; @@ -57,10 +56,6 @@ class ExpiringMessagesDeletionService { message.trigger('expired'); window.reduxActions.conversations.messageExpired(message.id); }); - - if (messages.length > 0) { - scheduleOptimizeFTS(); - } } catch (error) { window.SignalContext.log.error( 'destroyExpiredMessages: Error deleting expired messages', diff --git a/ts/services/ftsOptimizer.ts b/ts/services/ftsOptimizer.ts deleted file mode 100644 index 3add63450..000000000 --- a/ts/services/ftsOptimizer.ts +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2023 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only - -import { debounce } from 'lodash'; - -import { SECOND } from '../util/durations'; -import { sleep } from '../util/sleep'; -import { drop } from '../util/drop'; -import dataInterface from '../sql/Client'; -import type { FTSOptimizationStateType } from '../sql/Interface'; -import * as log from '../logging/log'; - -const INTERACTIVITY_DELAY_MS = 50; - -class FTSOptimizer { - private isRunning = false; - - public async run(): Promise { - if (this.isRunning) { - return; - } - this.isRunning = true; - - log.info('ftsOptimizer: starting'); - - let state: FTSOptimizationStateType | undefined; - - const start = Date.now(); - - try { - do { - if (state !== undefined) { - // eslint-disable-next-line no-await-in-loop - await sleep(INTERACTIVITY_DELAY_MS); - } - - // eslint-disable-next-line no-await-in-loop - state = await dataInterface.optimizeFTS(state); - } while (!state?.done); - } finally { - this.isRunning = false; - } - - const duration = Date.now() - start; - - if (!state) { - log.warn('ftsOptimizer: no final state'); - return; - } - - log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`); - } -} - -const optimizer = new FTSOptimizer(); - -export const optimizeFTS = (): void => { - drop(optimizer.run()); -}; - -export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, { - maxWait: 5 * SECOND, -}); diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 5819730ec..9fc9c5bef 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -411,11 +411,6 @@ export type GetAllStoriesResultType = ReadonlyArray< } >; -export type FTSOptimizationStateType = Readonly<{ - steps: number; - done?: boolean; -}>; - export type EditedMessageType = Readonly<{ conversationId: string; messageId: string; @@ -823,10 +818,6 @@ export type DataInterface = { getMaxMessageCounter(): Promise; getStatisticsForLogging(): Promise>; - - optimizeFTS: ( - state?: FTSOptimizationStateType - ) => Promise; }; export type ServerInterface = DataInterface & { diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index fd414de34..138a61721 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -93,7 +93,6 @@ import type { DeleteSentProtoRecipientResultType, EditedMessageType, EmojiType, - FTSOptimizationStateType, GetAllStoriesResultType, GetConversationRangeCenteredOnMessageResultType, GetKnownMessageAttachmentsResultType, @@ -403,8 +402,6 @@ const dataInterface: ServerInterface = { getStatisticsForLogging, - optimizeFTS, - // Server-only initialize, @@ -2222,7 +2219,6 @@ async function _removeAllMessages(): Promise { const db = getInstance(); db.exec(` DELETE FROM messages; - INSERT INTO messages_fts(messages_fts) VALUES('optimize'); `); } @@ -5548,8 +5544,6 @@ async function removeAll(): Promise { DELETE FROM unprocessed; DELETE FROM uninstalled_sticker_packs; - INSERT INTO messages_fts(messages_fts) VALUES('optimize'); - --- Re-create the messages delete trigger --- See migration 45 CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN @@ -6133,48 +6127,6 @@ async function removeKnownDraftAttachments( return Object.keys(lookup); } -const OPTIMIZE_FTS_PAGE_COUNT = 64; - -// This query is incremental. It gets the `state` from the return value of -// previous `optimizeFTS` call. When `state.done` is `true` - optimization is -// complete. -async function optimizeFTS( - state?: FTSOptimizationStateType -): Promise { - // See https://www.sqlite.org/fts5.html#the_merge_command - let pageCount = OPTIMIZE_FTS_PAGE_COUNT; - if (state === undefined) { - pageCount = -pageCount; - } - const db = getInstance(); - const getChanges = prepare(db, 'SELECT total_changes() as changes;', { - pluck: true, - }); - - const changeDifference = db.transaction(() => { - const before: number = getChanges.get({}); - - prepare( - db, - ` - INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount); - ` - ).run({ pageCount }); - - const after: number = getChanges.get({}); - - return after - before; - })(); - - const nextSteps = (state?.steps ?? 0) + 1; - - // From documentation: - // "If the difference is less than 2, then the 'merge' command was a no-op" - const done = changeDifference < 2; - - return { steps: nextSteps, done }; -} - async function getJobsInQueue(queueType: string): Promise> { const db = getInstance(); return getJobsInQueueSync(db, queueType); diff --git a/ts/sql/migrations/930-fts5-secure-delete.ts b/ts/sql/migrations/930-fts5-secure-delete.ts new file mode 100644 index 000000000..983c7b5bc --- /dev/null +++ b/ts/sql/migrations/930-fts5-secure-delete.ts @@ -0,0 +1,30 @@ +// Copyright 2023 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import type { Database } from '@signalapp/better-sqlite3'; + +import type { LoggerType } from '../../types/Logging'; + +export const version = 930; + +export function updateToSchemaVersion930( + currentVersion: number, + db: Database, + logger: LoggerType +): void { + if (currentVersion >= 930) { + return; + } + + db.transaction(() => { + db.exec(` + INSERT INTO messages_fts + (messages_fts, rank) + VALUES + ('secure-delete', 1); + `); + db.pragma('user_version = 930'); + })(); + + logger.info('updateToSchemaVersion930: success!'); +} diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index 7a17ed2b7..b867128e3 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -66,10 +66,11 @@ import updateToSchemaVersion88 from './88-service-ids'; import updateToSchemaVersion89 from './89-call-history'; import updateToSchemaVersion90 from './90-delete-story-reply-screenshot'; import updateToSchemaVersion91 from './91-clean-keys'; +import { updateToSchemaVersion920 } from './920-clean-more-keys'; import { version as MAX_VERSION, - updateToSchemaVersion920, -} from './920-clean-more-keys'; + updateToSchemaVersion930, +} from './930-fts5-secure-delete'; function updateToSchemaVersion1( currentVersion: number, @@ -2010,6 +2011,7 @@ export const SCHEMA_VERSIONS = [ updateToSchemaVersion91, // From here forward, all migrations should be multiples of 10 updateToSchemaVersion920, + updateToSchemaVersion930, ]; export function updateSchema(db: Database, logger: LoggerType): void { diff --git a/yarn.lock b/yarn.lock index e162b058d..eeda5c802 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3355,10 +3355,10 @@ "@react-types/overlays" "^3.7.1" "@react-types/shared" "^3.18.0" -"@signalapp/better-sqlite3@8.4.3": - version "8.4.3" - resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.4.3.tgz#7ffa8d03d2a12543247936bfb7b9f74cdbc6fe9b" - integrity sha512-6HaN8a90fWHBPIIZRRmSG7wc3BPczLx3Mb9MJ8wsatYqf8C1+2NdFae5AzII8Oe9YE9SSHzCjdPd8ST1m9+Qag== +"@signalapp/better-sqlite3@8.5.1": + version "8.5.1" + resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.5.1.tgz#b2bcf69d29f4d01cfecf2ad940ecd138f32470b3" + integrity sha512-s9NDufkbTAuKShFmEg7eOew4cprdLI7mn5ISQFKgcFSiA/TG3wgLJW5wUff9S2vednIV9utyqRJZXrwcSAEQUg== dependencies: bindings "^1.5.0" tar "^6.1.0"