diff --git a/ts/data/data.ts b/ts/data/data.ts index d4e9ab035..07acd7a16 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -497,27 +497,36 @@ async function getSeenMessagesByHashList(hashes: Array): Promise { } async function removeAllMessagesInConversation(conversationId: string): Promise { - let messages; - do { - // Yes, we really want the await in the loop. We're deleting 500 at a - // time so we don't use too much memory. - // eslint-disable-next-line no-await-in-loop - messages = await getLastMessagesByConversation(conversationId, 500, false); - if (!messages.length) { - return; - } - - const ids = messages.map(message => message.id); + let start = Date.now(); + const messages = await getLastMessagesByConversation(conversationId, 50, false); + window.log.info( + `removeAllMessagesInConversation ${conversationId} ${messages.length} took ${Date.now() - + start}` + ); + if (!messages.length) { + return; + } - // Note: It's very important that these models are fully hydrated because - // we need to delete all associated on-disk files along with the database delete. - // eslint-disable-next-line no-await-in-loop + // Note: It's very important that these models are fully hydrated because + // we need to delete all associated on-disk files along with the database delete. + // eslint-disable-next-line no-await-in-loop - await Promise.all(messages.map(message => message.cleanup())); + start = Date.now(); + for (let index = 0; index < messages.length; index++) { + const message = messages.at(index); + await message.cleanup(); + } + window.log.info( + `removeAllMessagesInConversation messages.cleanup() ${conversationId} took ${Date.now() - + start}ms` + ); + start = Date.now(); - // eslint-disable-next-line no-await-in-loop - await channels.removeMessagesByIds(ids); - } while (messages.length > 0); + // eslint-disable-next-line no-await-in-loop + await channels.removeAllMessagesInConversation(conversationId); + window.log.info( + `removeAllMessagesInConversation: ${conversationId} took ${Date.now() - start}ms` + ); } async function getMessagesBySentAt(sentAt: number): Promise { diff --git a/ts/node/database_utility.ts b/ts/node/database_utility.ts index 428d7754b..f36f15135 100644 --- a/ts/node/database_utility.ts +++ b/ts/node/database_utility.ts @@ -242,29 +242,29 @@ export function rebuildFtsTable(db: BetterSqlite3.Database) { db.exec(` -- Then we create our full-text search table and populate it CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE} - USING fts5(id UNINDEXED, body); - INSERT INTO ${MESSAGES_FTS_TABLE}(id, body) - SELECT id, body FROM ${MESSAGES_TABLE}; + USING fts5(body); + INSERT INTO ${MESSAGES_FTS_TABLE}(rowid, body) + SELECT rowid, body FROM ${MESSAGES_TABLE}; -- Then we set up triggers to keep the full-text search table up to date CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN INSERT INTO ${MESSAGES_FTS_TABLE} ( - id, + rowid, body ) VALUES ( - new.id, + new.rowid, new.body ); END; CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN - DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE rowid = old.rowid; END; CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} WHEN new.body <> old.body BEGIN - DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE rowid = old.rowid; INSERT INTO ${MESSAGES_FTS_TABLE}( - id, + rowid, body ) VALUES ( - new.id, + new.rowid, new.body ); END; diff --git a/ts/node/migration/sessionMigrations.ts b/ts/node/migration/sessionMigrations.ts index 6f3efd3ea..9c25bf3f4 100644 --- a/ts/node/migration/sessionMigrations.ts +++ b/ts/node/migration/sessionMigrations.ts @@ -33,6 +33,7 @@ import { } from '../database_utility'; import { getIdentityKeys, sqlNode } from '../sql'; +import { sleepFor } from '../../session/utils/Promise'; const hasDebugEnvVariable = Boolean(process.env.SESSION_DEBUG); @@ -100,6 +101,7 @@ const LOKI_SCHEMA_VERSIONS = [ updateToSessionSchemaVersion29, updateToSessionSchemaVersion30, updateToSessionSchemaVersion31, + updateToSessionSchemaVersion32, ]; function updateToSessionSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { @@ -1204,7 +1206,6 @@ function updateToSessionSchemaVersion29(currentVersion: number, db: BetterSqlite conversationId );`); rebuildFtsTable(db); - // Keeping this empty migration because some people updated to this already, even if it is not needed anymore writeSessionSchemaVersion(targetVersion, db); })(); @@ -1835,6 +1836,26 @@ function updateToSessionSchemaVersion31(currentVersion: number, db: BetterSqlite })(); } +function updateToSessionSchemaVersion32(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 32; + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(`CREATE INDEX messages_conversationId ON ${MESSAGES_TABLE} ( + conversationId + );`); + dropFtsAndTriggers(db); + rebuildFtsTable(db); + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + export function printTableColumns(table: string, db: BetterSqlite3.Database) { console.info(db.pragma(`table_info('${table}');`)); } @@ -1849,7 +1870,7 @@ function writeSessionSchemaVersion(newVersion: number, db: BetterSqlite3.Databas ).run({ newVersion }); } -export function updateSessionSchema(db: BetterSqlite3.Database) { +export async function updateSessionSchema(db: BetterSqlite3.Database) { const result = db .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';`) .get(); @@ -1866,5 +1887,8 @@ export function updateSessionSchema(db: BetterSqlite3.Database) { for (let index = 0, max = LOKI_SCHEMA_VERSIONS.length; index < max; index += 1) { const runSchemaUpdate = LOKI_SCHEMA_VERSIONS[index]; runSchemaUpdate(lokiSchemaVersion, db); + if (index > lokiSchemaVersion) { + await sleepFor(200); // give some time for the UI to not freeze between 2 migrations + } } } diff --git a/ts/node/migration/signalMigrations.ts b/ts/node/migration/signalMigrations.ts index 0c23a7258..ff4caed75 100644 --- a/ts/node/migration/signalMigrations.ts +++ b/ts/node/migration/signalMigrations.ts @@ -527,7 +527,7 @@ const SCHEMA_VERSIONS = [ updateToSchemaVersion11, ]; -export function updateSchema(db: BetterSqlite3.Database) { +export async function updateSchema(db: BetterSqlite3.Database) { const sqliteVersion = getSQLiteVersion(db); const sqlcipherVersion = getSQLCipherVersion(db); const userVersion = getUserVersion(db); @@ -545,7 +545,7 @@ export function updateSchema(db: BetterSqlite3.Database) { const runSchemaUpdate = SCHEMA_VERSIONS[index]; runSchemaUpdate(schemaVersion, db); } - updateSessionSchema(db); + await updateSessionSchema(db); } function migrateSchemaVersion(db: BetterSqlite3.Database) { diff --git a/ts/node/sql.ts b/ts/node/sql.ts index 863f29e6e..6b227126e 100644 --- a/ts/node/sql.ts +++ b/ts/node/sql.ts @@ -28,7 +28,6 @@ import { ATTACHMENT_DOWNLOADS_TABLE, CLOSED_GROUP_V2_KEY_PAIRS_TABLE, CONVERSATIONS_TABLE, - dropFtsAndTriggers, formatRowOfConversation, GUARD_NODE_TABLE, HEX_KEY, @@ -41,7 +40,6 @@ import { NODES_FOR_PUBKEY_TABLE, objectToJSON, OPEN_GROUP_ROOMS_V2_TABLE, - rebuildFtsTable, toSqliteBoolean, } from './database_utility'; import { LocaleMessagesType } from './locale'; // checked - only node @@ -166,7 +164,7 @@ async function initializeSql({ if (!db) { throw new Error('db is not set'); } - updateSchema(db); + await updateSchema(db); // test database @@ -698,9 +696,9 @@ function searchMessages(query: string, limit: number) { ${MESSAGES_TABLE}.json, snippet(${MESSAGES_FTS_TABLE}, -1, '<>', '<>', '...', 5) as snippet FROM ${MESSAGES_FTS_TABLE} - INNER JOIN ${MESSAGES_TABLE} on ${MESSAGES_FTS_TABLE}.id = ${MESSAGES_TABLE}.id + INNER JOIN ${MESSAGES_TABLE} on ${MESSAGES_FTS_TABLE}.rowid = ${MESSAGES_TABLE}.rowid WHERE - ${MESSAGES_FTS_TABLE} match $query + ${MESSAGES_FTS_TABLE}.body match $query ${orderByMessageCoalesceClause} LIMIT $limit;` ) @@ -958,11 +956,13 @@ function removeMessagesByIds(ids: Array, instance?: BetterSqlite3.Databa if (!ids.length) { throw new Error('removeMessagesByIds: No ids to delete!'); } + const start = Date.now(); - // Our node interface doesn't seem to allow you to replace one single ? with an array + // TODO we might need to do the same thing as assertGlobalInstanceOrInstance(instance) .prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE id IN ( ${ids.map(() => '?').join(', ')} );`) .run(ids); + console.log(`removeMessagesByIds of length ${ids.length} took ${Date.now() - start}ms`); } function removeAllMessagesInConversation( @@ -972,11 +972,13 @@ function removeAllMessagesInConversation( if (!conversationId) { return; } + const inst = assertGlobalInstanceOrInstance(instance); - // Our node interface doesn't seem to allow you to replace one single ? with an array - assertGlobalInstanceOrInstance(instance) - .prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId`) - .run({ conversationId }); + inst.transaction(() => { + inst + .prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId`) + .run({ conversationId }); + })(); } function getMessageIdsFromServerIds(serverIds: Array, conversationId: string) { @@ -2233,7 +2235,6 @@ function cleanUpOldOpengroupsOnStart() { // first remove very old messages for each opengroups const db = assertGlobalInstance(); db.transaction(() => { - dropFtsAndTriggers(db); v2ConvosIds.forEach(convoId => { const messagesInConvoBefore = getMessagesCountByConversation(convoId); @@ -2316,8 +2317,6 @@ function cleanUpOldOpengroupsOnStart() { } cleanUpMessagesJson(); - - rebuildFtsTable(db); })(); } diff --git a/ts/session/utils/job_runners/jobs/ConfigurationSyncJob.ts b/ts/session/utils/job_runners/jobs/ConfigurationSyncJob.ts index f50135a14..0cef20c86 100644 --- a/ts/session/utils/job_runners/jobs/ConfigurationSyncJob.ts +++ b/ts/session/utils/job_runners/jobs/ConfigurationSyncJob.ts @@ -294,7 +294,7 @@ async function queueNewJobIfNeeded() { !lastRunConfigSyncJobTimestamp || lastRunConfigSyncJobTimestamp < Date.now() - defaultMsBetweenRetries ) { - window.log.debug('Scheduling ConfSyncJob: ASAP'); + // window.log.debug('Scheduling ConfSyncJob: ASAP'); // we postpone by 1000ms to make sure whoever is adding this job is done with what is needs to do first // this call will make sure that there is only one configuration sync job at all times await runners.configurationSyncRunner.addJob( @@ -305,7 +305,7 @@ async function queueNewJobIfNeeded() { const diff = Math.max(Date.now() - lastRunConfigSyncJobTimestamp, 0); // but we want to run every 30, so what we need is actually `30-10` from now = 20 const leftBeforeNextTick = Math.max(defaultMsBetweenRetries - diff, 1000); - window.log.debug('Scheduling ConfSyncJob: LATER'); + // window.log.debug('Scheduling ConfSyncJob: LATER'); await runners.configurationSyncRunner.addJob( new ConfigurationSyncJob({ nextAttemptTimestamp: Date.now() + leftBeforeNextTick })