keejef
/
session-desktop
mirror of https://github.com/oxen-io/session-desktop
2
0
Fork 0
Code Issues 8 Projects Releases Wiki Activity

fix: improve delete messages perfs and search logic

Browse Source
pull/2795/head
Audric Ackermann 2 years ago
parent 4280d83ba8
commit 524debb307
6 changed files with 78 additions and 46 deletions
Show Stats Download Patch File Download Diff File

45
ts/data/data.ts
Unescape Escape View File

@ -497,27 +497,36 @@ async function getSeenMessagesByHashList(hashes: Array<string>): Promise<any> {
} }
async function removeAllMessagesInConversation(conversationId: string): Promise<void> { async function removeAllMessagesInConversation(conversationId: string): Promise<void> {
let messages; let start = Date.now();
do { const messages = await getLastMessagesByConversation(conversationId, 50, false);
// Yes, we really want the await in the loop. We're deleting 500 at a window.log.info(
// time so we don't use too much memory. `removeAllMessagesInConversation ${conversationId} ${messages.length} took ${Date.now() -
// eslint-disable-next-line no-await-in-loop start}`
messages = await getLastMessagesByConversation(conversationId, 500, false); );
if (!messages.length) { if (!messages.length) {
return; return;
} }
const ids = messages.map(message => message.id);
// Note: It's very important that these models are fully hydrated because // Note: It's very important that these models are fully hydrated because
// we need to delete all associated on-disk files along with the database delete. // we need to delete all associated on-disk files along with the database delete.
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
await Promise.all(messages.map(message => message.cleanup())); start = Date.now();
for (let index = 0; index < messages.length; index++) {
const message = messages.at(index);
await message.cleanup();
}
window.log.info(
`removeAllMessagesInConversation messages.cleanup() ${conversationId} took ${Date.now() -
start}ms`
);
start = Date.now();
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
await channels.removeMessagesByIds(ids); await channels.removeAllMessagesInConversation(conversationId);
} while (messages.length > 0); window.log.info(
`removeAllMessagesInConversation: ${conversationId} took ${Date.now() - start}ms`
);
} }
async function getMessagesBySentAt(sentAt: number): Promise<MessageCollection> { async function getMessagesBySentAt(sentAt: number): Promise<MessageCollection> {

18
ts/node/database_utility.ts
Unescape Escape View File

@ -242,29 +242,29 @@ export function rebuildFtsTable(db: BetterSqlite3.Database) {
db.exec(` db.exec(`
-- Then we create our full-text search table and populate it -- Then we create our full-text search table and populate it
CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE} CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE}
USING fts5(id UNINDEXED, body); USING fts5(body);
INSERT INTO ${MESSAGES_FTS_TABLE}(id, body) INSERT INTO ${MESSAGES_FTS_TABLE}(rowid, body)
SELECT id, body FROM ${MESSAGES_TABLE}; SELECT rowid, body FROM ${MESSAGES_TABLE};
-- Then we set up triggers to keep the full-text search table up to date -- Then we set up triggers to keep the full-text search table up to date
CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN
INSERT INTO ${MESSAGES_FTS_TABLE} ( INSERT INTO ${MESSAGES_FTS_TABLE} (
id, rowid,
body body
) VALUES ( ) VALUES (
new.id, new.rowid,
new.body new.body
); );
END; END;
CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN
DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; DELETE FROM ${MESSAGES_FTS_TABLE} WHERE rowid = old.rowid;
END; END;
CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} WHEN new.body <> old.body BEGIN CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} WHEN new.body <> old.body BEGIN
DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; DELETE FROM ${MESSAGES_FTS_TABLE} WHERE rowid = old.rowid;
INSERT INTO ${MESSAGES_FTS_TABLE}( INSERT INTO ${MESSAGES_FTS_TABLE}(
id, rowid,
body body
) VALUES ( ) VALUES (
new.id, new.rowid,
new.body new.body
); );
END; END;

28
ts/node/migration/sessionMigrations.ts
Unescape Escape View File

@ -33,6 +33,7 @@ import {
} from '../database_utility'; } from '../database_utility';
import { getIdentityKeys, sqlNode } from '../sql'; import { getIdentityKeys, sqlNode } from '../sql';
import { sleepFor } from '../../session/utils/Promise';
const hasDebugEnvVariable = Boolean(process.env.SESSION_DEBUG); const hasDebugEnvVariable = Boolean(process.env.SESSION_DEBUG);
@ -100,6 +101,7 @@ const LOKI_SCHEMA_VERSIONS = [
updateToSessionSchemaVersion29, updateToSessionSchemaVersion29,
updateToSessionSchemaVersion30, updateToSessionSchemaVersion30,
updateToSessionSchemaVersion31, updateToSessionSchemaVersion31,
updateToSessionSchemaVersion32,
]; ];
function updateToSessionSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { function updateToSessionSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) {
@ -1204,7 +1206,6 @@ function updateToSessionSchemaVersion29(currentVersion: number, db: BetterSqlite
conversationId conversationId
);`); );`);
rebuildFtsTable(db); rebuildFtsTable(db);
// Keeping this empty migration because some people updated to this already, even if it is not needed anymore
writeSessionSchemaVersion(targetVersion, db); writeSessionSchemaVersion(targetVersion, db);
})(); })();
@ -1835,6 +1836,26 @@ function updateToSessionSchemaVersion31(currentVersion: number, db: BetterSqlite
})(); })();
} }
function updateToSessionSchemaVersion32(currentVersion: number, db: BetterSqlite3.Database) {
const targetVersion = 32;
if (currentVersion >= targetVersion) {
return;
}
console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`);
db.transaction(() => {
db.exec(`CREATE INDEX messages_conversationId ON ${MESSAGES_TABLE} (
conversationId
);`);
dropFtsAndTriggers(db);
rebuildFtsTable(db);
writeSessionSchemaVersion(targetVersion, db);
})();
console.log(`updateToSessionSchemaVersion${targetVersion}: success!`);
}
export function printTableColumns(table: string, db: BetterSqlite3.Database) { export function printTableColumns(table: string, db: BetterSqlite3.Database) {
console.info(db.pragma(`table_info('${table}');`)); console.info(db.pragma(`table_info('${table}');`));
} }
@ -1849,7 +1870,7 @@ function writeSessionSchemaVersion(newVersion: number, db: BetterSqlite3.Databas
).run({ newVersion }); ).run({ newVersion });
} }
export function updateSessionSchema(db: BetterSqlite3.Database) { export async function updateSessionSchema(db: BetterSqlite3.Database) {
const result = db const result = db
.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';`) .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';`)
.get(); .get();
@ -1866,5 +1887,8 @@ export function updateSessionSchema(db: BetterSqlite3.Database) {
for (let index = 0, max = LOKI_SCHEMA_VERSIONS.length; index < max; index += 1) { for (let index = 0, max = LOKI_SCHEMA_VERSIONS.length; index < max; index += 1) {
const runSchemaUpdate = LOKI_SCHEMA_VERSIONS[index]; const runSchemaUpdate = LOKI_SCHEMA_VERSIONS[index];
runSchemaUpdate(lokiSchemaVersion, db); runSchemaUpdate(lokiSchemaVersion, db);
if (index > lokiSchemaVersion) {
await sleepFor(200); // give some time for the UI to not freeze between 2 migrations
}
} }
} }

4
ts/node/migration/signalMigrations.ts
Unescape Escape View File

@ -527,7 +527,7 @@ const SCHEMA_VERSIONS = [
updateToSchemaVersion11, updateToSchemaVersion11,
]; ];
export function updateSchema(db: BetterSqlite3.Database) { export async function updateSchema(db: BetterSqlite3.Database) {
const sqliteVersion = getSQLiteVersion(db); const sqliteVersion = getSQLiteVersion(db);
const sqlcipherVersion = getSQLCipherVersion(db); const sqlcipherVersion = getSQLCipherVersion(db);
const userVersion = getUserVersion(db); const userVersion = getUserVersion(db);
@ -545,7 +545,7 @@ export function updateSchema(db: BetterSqlite3.Database) {
const runSchemaUpdate = SCHEMA_VERSIONS[index]; const runSchemaUpdate = SCHEMA_VERSIONS[index];
runSchemaUpdate(schemaVersion, db); runSchemaUpdate(schemaVersion, db);
} }
updateSessionSchema(db); await updateSessionSchema(db);
} }
function migrateSchemaVersion(db: BetterSqlite3.Database) { function migrateSchemaVersion(db: BetterSqlite3.Database) {

25
ts/node/sql.ts
Unescape Escape View File

@ -28,7 +28,6 @@ import {
ATTACHMENT_DOWNLOADS_TABLE, ATTACHMENT_DOWNLOADS_TABLE,
CLOSED_GROUP_V2_KEY_PAIRS_TABLE, CLOSED_GROUP_V2_KEY_PAIRS_TABLE,
CONVERSATIONS_TABLE, CONVERSATIONS_TABLE,
dropFtsAndTriggers,
formatRowOfConversation, formatRowOfConversation,
GUARD_NODE_TABLE, GUARD_NODE_TABLE,
HEX_KEY, HEX_KEY,
@ -41,7 +40,6 @@ import {
NODES_FOR_PUBKEY_TABLE, NODES_FOR_PUBKEY_TABLE,
objectToJSON, objectToJSON,
OPEN_GROUP_ROOMS_V2_TABLE, OPEN_GROUP_ROOMS_V2_TABLE,
rebuildFtsTable,
toSqliteBoolean, toSqliteBoolean,
} from './database_utility'; } from './database_utility';
import { LocaleMessagesType } from './locale'; // checked - only node import { LocaleMessagesType } from './locale'; // checked - only node
@ -166,7 +164,7 @@ async function initializeSql({
if (!db) { if (!db) {
throw new Error('db is not set'); throw new Error('db is not set');
} }
updateSchema(db); await updateSchema(db);
// test database // test database
@ -698,9 +696,9 @@ function searchMessages(query: string, limit: number) {
${MESSAGES_TABLE}.json, ${MESSAGES_TABLE}.json,
snippet(${MESSAGES_FTS_TABLE}, -1, '<<left>>', '<<right>>', '...', 5) as snippet snippet(${MESSAGES_FTS_TABLE}, -1, '<<left>>', '<<right>>', '...', 5) as snippet
FROM ${MESSAGES_FTS_TABLE} FROM ${MESSAGES_FTS_TABLE}
INNER JOIN ${MESSAGES_TABLE} on ${MESSAGES_FTS_TABLE}.id = ${MESSAGES_TABLE}.id INNER JOIN ${MESSAGES_TABLE} on ${MESSAGES_FTS_TABLE}.rowid = ${MESSAGES_TABLE}.rowid
WHERE WHERE
${MESSAGES_FTS_TABLE} match $query ${MESSAGES_FTS_TABLE}.body match $query
${orderByMessageCoalesceClause} ${orderByMessageCoalesceClause}
LIMIT $limit;` LIMIT $limit;`
) )
@ -958,11 +956,13 @@ function removeMessagesByIds(ids: Array<string>, instance?: BetterSqlite3.Databa
if (!ids.length) { if (!ids.length) {
throw new Error('removeMessagesByIds: No ids to delete!'); throw new Error('removeMessagesByIds: No ids to delete!');
} }
const start = Date.now();
// Our node interface doesn't seem to allow you to replace one single ? with an array // TODO we might need to do the same thing as
assertGlobalInstanceOrInstance(instance) assertGlobalInstanceOrInstance(instance)
.prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE id IN ( ${ids.map(() => '?').join(', ')} );`) .prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE id IN ( ${ids.map(() => '?').join(', ')} );`)
.run(ids); .run(ids);
console.log(`removeMessagesByIds of length ${ids.length} took ${Date.now() - start}ms`);
} }
function removeAllMessagesInConversation( function removeAllMessagesInConversation(
@ -972,11 +972,13 @@ function removeAllMessagesInConversation(
if (!conversationId) { if (!conversationId) {
return; return;
} }
const inst = assertGlobalInstanceOrInstance(instance);
// Our node interface doesn't seem to allow you to replace one single ? with an array inst.transaction(() => {
assertGlobalInstanceOrInstance(instance) inst
.prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId`) .prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId`)
.run({ conversationId }); .run({ conversationId });
})();
} }
function getMessageIdsFromServerIds(serverIds: Array<string | number>, conversationId: string) { function getMessageIdsFromServerIds(serverIds: Array<string | number>, conversationId: string) {
@ -2233,7 +2235,6 @@ function cleanUpOldOpengroupsOnStart() {
// first remove very old messages for each opengroups // first remove very old messages for each opengroups
const db = assertGlobalInstance(); const db = assertGlobalInstance();
db.transaction(() => { db.transaction(() => {
dropFtsAndTriggers(db);
v2ConvosIds.forEach(convoId => { v2ConvosIds.forEach(convoId => {
const messagesInConvoBefore = getMessagesCountByConversation(convoId); const messagesInConvoBefore = getMessagesCountByConversation(convoId);
@ -2316,8 +2317,6 @@ function cleanUpOldOpengroupsOnStart() {
} }
cleanUpMessagesJson(); cleanUpMessagesJson();
rebuildFtsTable(db);
})(); })();
} }

4
ts/session/utils/job_runners/jobs/ConfigurationSyncJob.ts
Unescape Escape View File

@ -294,7 +294,7 @@ async function queueNewJobIfNeeded() {
!lastRunConfigSyncJobTimestamp || !lastRunConfigSyncJobTimestamp ||
lastRunConfigSyncJobTimestamp < Date.now() - defaultMsBetweenRetries lastRunConfigSyncJobTimestamp < Date.now() - defaultMsBetweenRetries
) { ) {
window.log.debug('Scheduling ConfSyncJob: ASAP'); // window.log.debug('Scheduling ConfSyncJob: ASAP');
// we postpone by 1000ms to make sure whoever is adding this job is done with what is needs to do first // we postpone by 1000ms to make sure whoever is adding this job is done with what is needs to do first
// this call will make sure that there is only one configuration sync job at all times // this call will make sure that there is only one configuration sync job at all times
await runners.configurationSyncRunner.addJob( await runners.configurationSyncRunner.addJob(
@ -305,7 +305,7 @@ async function queueNewJobIfNeeded() {
const diff = Math.max(Date.now() - lastRunConfigSyncJobTimestamp, 0); const diff = Math.max(Date.now() - lastRunConfigSyncJobTimestamp, 0);
// but we want to run every 30, so what we need is actually `30-10` from now = 20 // but we want to run every 30, so what we need is actually `30-10` from now = 20
const leftBeforeNextTick = Math.max(defaultMsBetweenRetries - diff, 1000); const leftBeforeNextTick = Math.max(defaultMsBetweenRetries - diff, 1000);
window.log.debug('Scheduling ConfSyncJob: LATER'); // window.log.debug('Scheduling ConfSyncJob: LATER');
await runners.configurationSyncRunner.addJob( await runners.configurationSyncRunner.addJob(
new ConfigurationSyncJob({ nextAttemptTimestamp: Date.now() + leftBeforeNextTick }) new ConfigurationSyncJob({ nextAttemptTimestamp: Date.now() + leftBeforeNextTick })

Write Preview
Loading…
Cancel
Save