keejef
/
session-desktop
mirror of https://github.com/oxen-io/session-desktop
2
0
Fork 0
Code Issues 8 Projects Releases Wiki Activity

fix: improve delete messages perfs and search logic

Browse Source
pull/2795/head
Audric Ackermann 2 years ago
parent 4280d83ba8
commit 524debb307
6 changed files with 78 additions and 46 deletions
Show Stats Download Patch File Download Diff File

45
ts/data/data.ts
Unescape Escape View File

@ -497,27 +497,36 @@ async function getSeenMessagesByHashList(hashes: Array<string>): Promise<any> {
}
async function removeAllMessagesInConversation(conversationId: string): Promise<void> {
let messages;
do {
// Yes, we really want the await in the loop. We're deleting 500 at a
// time so we don't use too much memory.
// eslint-disable-next-line no-await-in-loop
messages = await getLastMessagesByConversation(conversationId, 500, false);
if (!messages.length) {
return;
}
const ids = messages.map(message => message.id);
let start = Date.now();
const messages = await getLastMessagesByConversation(conversationId, 50, false);
window.log.info(
`removeAllMessagesInConversation ${conversationId} ${messages.length} took ${Date.now() -
start}`
);
if (!messages.length) {
return;
}
// Note: It's very important that these models are fully hydrated because
// we need to delete all associated on-disk files along with the database delete.
// eslint-disable-next-line no-await-in-loop
// Note: It's very important that these models are fully hydrated because
// we need to delete all associated on-disk files along with the database delete.
// eslint-disable-next-line no-await-in-loop
await Promise.all(messages.map(message => message.cleanup()));
start = Date.now();
for (let index = 0; index < messages.length; index++) {
const message = messages.at(index);
await message.cleanup();
}
window.log.info(
`removeAllMessagesInConversation messages.cleanup() ${conversationId} took ${Date.now() -
start}ms`
);
start = Date.now();
// eslint-disable-next-line no-await-in-loop
await channels.removeMessagesByIds(ids);
} while (messages.length > 0);
// eslint-disable-next-line no-await-in-loop
await channels.removeAllMessagesInConversation(conversationId);
window.log.info(
`removeAllMessagesInConversation: ${conversationId} took ${Date.now() - start}ms`
);
}
async function getMessagesBySentAt(sentAt: number): Promise<MessageCollection> {

18
ts/node/database_utility.ts
Unescape Escape View File

@ -242,29 +242,29 @@ export function rebuildFtsTable(db: BetterSqlite3.Database) {
db.exec(`
-- Then we create our full-text search table and populate it
CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE}
USING fts5(id UNINDEXED, body);
INSERT INTO ${MESSAGES_FTS_TABLE}(id, body)
SELECT id, body FROM ${MESSAGES_TABLE};
USING fts5(body);
INSERT INTO ${MESSAGES_FTS_TABLE}(rowid, body)
SELECT rowid, body FROM ${MESSAGES_TABLE};
-- Then we set up triggers to keep the full-text search table up to date
CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN
INSERT INTO ${MESSAGES_FTS_TABLE} (
id,
rowid,
body
) VALUES (
new.id,
new.rowid,
new.body
);
END;
CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN
DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id;
DELETE FROM ${MESSAGES_FTS_TABLE} WHERE rowid = old.rowid;
END;
CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} WHEN new.body <> old.body BEGIN
DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id;
DELETE FROM ${MESSAGES_FTS_TABLE} WHERE rowid = old.rowid;
INSERT INTO ${MESSAGES_FTS_TABLE}(
id,
rowid,
body
) VALUES (
new.id,
new.rowid,
new.body
);
END;

28
ts/node/migration/sessionMigrations.ts
Unescape Escape View File

@ -33,6 +33,7 @@ import {
} from '../database_utility';
import { getIdentityKeys, sqlNode } from '../sql';
import { sleepFor } from '../../session/utils/Promise';
const hasDebugEnvVariable = Boolean(process.env.SESSION_DEBUG);
@ -100,6 +101,7 @@ const LOKI_SCHEMA_VERSIONS = [
updateToSessionSchemaVersion29,
updateToSessionSchemaVersion30,
updateToSessionSchemaVersion31,
updateToSessionSchemaVersion32,
];
function updateToSessionSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) {
@ -1204,7 +1206,6 @@ function updateToSessionSchemaVersion29(currentVersion: number, db: BetterSqlite
conversationId
);`);
rebuildFtsTable(db);
// Keeping this empty migration because some people updated to this already, even if it is not needed anymore
writeSessionSchemaVersion(targetVersion, db);
})();
@ -1835,6 +1836,26 @@ function updateToSessionSchemaVersion31(currentVersion: number, db: BetterSqlite
})();
}
function updateToSessionSchemaVersion32(currentVersion: number, db: BetterSqlite3.Database) {
const targetVersion = 32;
if (currentVersion >= targetVersion) {
return;
}
console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`);
db.transaction(() => {
db.exec(`CREATE INDEX messages_conversationId ON ${MESSAGES_TABLE} (
conversationId
);`);
dropFtsAndTriggers(db);
rebuildFtsTable(db);
writeSessionSchemaVersion(targetVersion, db);
})();
console.log(`updateToSessionSchemaVersion${targetVersion}: success!`);
}
export function printTableColumns(table: string, db: BetterSqlite3.Database) {
console.info(db.pragma(`table_info('${table}');`));
}
@ -1849,7 +1870,7 @@ function writeSessionSchemaVersion(newVersion: number, db: BetterSqlite3.Databas
).run({ newVersion });
}
export function updateSessionSchema(db: BetterSqlite3.Database) {
export async function updateSessionSchema(db: BetterSqlite3.Database) {
const result = db
.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';`)
.get();
@ -1866,5 +1887,8 @@ export function updateSessionSchema(db: BetterSqlite3.Database) {
for (let index = 0, max = LOKI_SCHEMA_VERSIONS.length; index < max; index += 1) {
const runSchemaUpdate = LOKI_SCHEMA_VERSIONS[index];
runSchemaUpdate(lokiSchemaVersion, db);
if (index > lokiSchemaVersion) {
await sleepFor(200); // give some time for the UI to not freeze between 2 migrations
}
}
}

4
ts/node/migration/signalMigrations.ts
Unescape Escape View File

@ -527,7 +527,7 @@ const SCHEMA_VERSIONS = [
updateToSchemaVersion11,
];
export function updateSchema(db: BetterSqlite3.Database) {
export async function updateSchema(db: BetterSqlite3.Database) {
const sqliteVersion = getSQLiteVersion(db);
const sqlcipherVersion = getSQLCipherVersion(db);
const userVersion = getUserVersion(db);
@ -545,7 +545,7 @@ export function updateSchema(db: BetterSqlite3.Database) {
const runSchemaUpdate = SCHEMA_VERSIONS[index];
runSchemaUpdate(schemaVersion, db);
}
updateSessionSchema(db);
await updateSessionSchema(db);
}
function migrateSchemaVersion(db: BetterSqlite3.Database) {

25
ts/node/sql.ts
Unescape Escape View File

@ -28,7 +28,6 @@ import {
ATTACHMENT_DOWNLOADS_TABLE,
CLOSED_GROUP_V2_KEY_PAIRS_TABLE,
CONVERSATIONS_TABLE,
dropFtsAndTriggers,
formatRowOfConversation,
GUARD_NODE_TABLE,
HEX_KEY,
@ -41,7 +40,6 @@ import {
NODES_FOR_PUBKEY_TABLE,
objectToJSON,
OPEN_GROUP_ROOMS_V2_TABLE,
rebuildFtsTable,
toSqliteBoolean,
} from './database_utility';
import { LocaleMessagesType } from './locale'; // checked - only node
@ -166,7 +164,7 @@ async function initializeSql({
if (!db) {
throw new Error('db is not set');
}
updateSchema(db);
await updateSchema(db);
// test database
@ -698,9 +696,9 @@ function searchMessages(query: string, limit: number) {
${MESSAGES_TABLE}.json,
snippet(${MESSAGES_FTS_TABLE}, -1, '<<left>>', '<<right>>', '...', 5) as snippet
FROM ${MESSAGES_FTS_TABLE}
INNER JOIN ${MESSAGES_TABLE} on ${MESSAGES_FTS_TABLE}.id = ${MESSAGES_TABLE}.id
INNER JOIN ${MESSAGES_TABLE} on ${MESSAGES_FTS_TABLE}.rowid = ${MESSAGES_TABLE}.rowid
WHERE
${MESSAGES_FTS_TABLE} match $query
${MESSAGES_FTS_TABLE}.body match $query
${orderByMessageCoalesceClause}
LIMIT $limit;`
)
@ -958,11 +956,13 @@ function removeMessagesByIds(ids: Array<string>, instance?: BetterSqlite3.Databa
if (!ids.length) {
throw new Error('removeMessagesByIds: No ids to delete!');
}
const start = Date.now();
// Our node interface doesn't seem to allow you to replace one single ? with an array
// TODO we might need to do the same thing as
assertGlobalInstanceOrInstance(instance)
.prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE id IN ( ${ids.map(() => '?').join(', ')} );`)
.run(ids);
console.log(`removeMessagesByIds of length ${ids.length} took ${Date.now() - start}ms`);
}
function removeAllMessagesInConversation(
@ -972,11 +972,13 @@ function removeAllMessagesInConversation(
if (!conversationId) {
return;
}
const inst = assertGlobalInstanceOrInstance(instance);
// Our node interface doesn't seem to allow you to replace one single ? with an array
assertGlobalInstanceOrInstance(instance)
.prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId`)
.run({ conversationId });
inst.transaction(() => {
inst
.prepare(`DELETE FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId`)
.run({ conversationId });
})();
}
function getMessageIdsFromServerIds(serverIds: Array<string | number>, conversationId: string) {
@ -2233,7 +2235,6 @@ function cleanUpOldOpengroupsOnStart() {
// first remove very old messages for each opengroups
const db = assertGlobalInstance();
db.transaction(() => {
dropFtsAndTriggers(db);
v2ConvosIds.forEach(convoId => {
const messagesInConvoBefore = getMessagesCountByConversation(convoId);
@ -2316,8 +2317,6 @@ function cleanUpOldOpengroupsOnStart() {
}
cleanUpMessagesJson();
rebuildFtsTable(db);
})();
}

4
ts/session/utils/job_runners/jobs/ConfigurationSyncJob.ts
Unescape Escape View File

@ -294,7 +294,7 @@ async function queueNewJobIfNeeded() {
!lastRunConfigSyncJobTimestamp ||
lastRunConfigSyncJobTimestamp < Date.now() - defaultMsBetweenRetries
) {
window.log.debug('Scheduling ConfSyncJob: ASAP');
// window.log.debug('Scheduling ConfSyncJob: ASAP');
// we postpone by 1000ms to make sure whoever is adding this job is done with what is needs to do first
// this call will make sure that there is only one configuration sync job at all times
await runners.configurationSyncRunner.addJob(
@ -305,7 +305,7 @@ async function queueNewJobIfNeeded() {
const diff = Math.max(Date.now() - lastRunConfigSyncJobTimestamp, 0);
// but we want to run every 30, so what we need is actually `30-10` from now = 20
const leftBeforeNextTick = Math.max(defaultMsBetweenRetries - diff, 1000);
window.log.debug('Scheduling ConfSyncJob: LATER');
// window.log.debug('Scheduling ConfSyncJob: LATER');
await runners.configurationSyncRunner.addJob(
new ConfigurationSyncJob({ nextAttemptTimestamp: Date.now() + leftBeforeNextTick })

Write Preview
Loading…
Cancel
Save