From 323b7ec45c67d83829402352877640984d009140 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 14 Apr 2022 10:56:34 +1000 Subject: [PATCH] cleanup >2 months old messages in opengroups --- ts/node/sql.ts | 102 ++++++++++++++++++++++++++++++++++++++- ts/node/sql_channel.ts | 1 - ts/receiver/queuedJob.ts | 4 ++ 3 files changed, 104 insertions(+), 3 deletions(-) diff --git a/ts/node/sql.ts b/ts/node/sql.ts index 5b1e27b4f..d8e7551bb 100644 --- a/ts/node/sql.ts +++ b/ts/node/sql.ts @@ -1469,9 +1469,11 @@ async function initializeSql({ // Clear any already deleted db entries on each app start. vacuumDatabase(db); + + cleanUpOldOpengroups(); + const msgCount = getMessageCount(); const convoCount = getConversationCount(); - console.info('total message count: ', msgCount); console.info('total conversation count: ', convoCount); } catch (error) { @@ -2212,6 +2214,26 @@ function getMessageBySenderAndSentAt({ source, sentAt }: { source: string; sentA return map(rows, row => jsonToObject(row.json)); } +function getMessagesCountBySender({ source }: { source: string }) { + if (!source) { + throw new Error('source must be set'); + } + const count = assertGlobalInstance() + .prepare( + `SELECT count(*) FROM ${MESSAGES_TABLE} WHERE + source = $source;` + ) + .get({ + source, + }); + console.info('count messages of ', source, ' :', count); + if (!count) { + return 0; + } + + return count['count(*)'] || 0; +} + function getMessageBySenderAndTimestamp({ source, timestamp, @@ -3126,7 +3148,7 @@ function removeKnownAttachments(allAttachments: any) { function getMessagesCountByConversation( conversationId: string, instance?: BetterSqlite3.Database | null -) { +): number { const row = assertGlobalInstanceOrInstance(instance) .prepare(`SELECT count(*) from ${MESSAGES_TABLE} WHERE conversationId = $conversationId;`) .get({ conversationId }); @@ -3334,6 +3356,81 @@ function removeV2OpenGroupRoom(conversationId: string) { }); } +function cleanUpOldOpengroups() { + const v2Convos = getAllOpenGroupV2Conversations(); + + // For each opengroups, if it has more than 1000 messages, we remove all the messages older than 2 months. + // So this does not limit the size of opengroup history to 1000 messages but to 2 months. + // This is the only way we can cleanup conversations objects from users which just sent messages a while ago and with whom we never interacted. + // This is only for opengroups, and is because ALL the conversations are cached in the redux store. Having a very large number of conversations (unused) is deteriorating a lot the performance of the app. + // Another fix would be to not cache all the conversations in the redux store, but it ain't going to happen anytime soon as it would a pretty big change of the way we do things and would break a lot of the app. + const maxMessagePerOpengroupConvo = 1000; + + // first remove very old messages for each opengroups + + v2Convos.forEach(convo => { + const convoId = convo.id; + const messagesInConvo = getMessagesCountByConversation(convoId); + + if (messagesInConvo >= maxMessagePerOpengroupConvo) { + const minute = 1000 * 60; + const twoMonths = minute * 60 * 24 * 60; + console.info( + `too many message: ${messagesInConvo} in convo: ${convoId}. Limit is ${maxMessagePerOpengroupConvo}` + ); + assertGlobalInstance() + .prepare( + ` + DELETE FROM ${MESSAGES_TABLE} WHERE serverTimestamp <= $serverTimestamp AND conversationId = $conversationId` + ) + .run({ conversationId: convoId, serverTimestamp: Date.now() - twoMonths }); // delete messages older than twoMonths + + const messagesInConvoAfter = getMessagesCountByConversation(convoId); + console.info( + `after leaning old history, we have ${messagesInConvoAfter} messages left in convo: ${convoId}` + ); + + const unreadCount = getUnreadCountByConversation(convoId); + const convoProps = getConversationById(convoId); + if (convoProps) { + convoProps.unreadCount = unreadCount; + updateConversation(convoProps); + } + } + }); + + // now, we might have a bunch of private conversation, without any interaction and no messages + // those are the conversation of the old members in the opengroups we just cleaned. + const allInactiveConvos = assertGlobalInstance() + .prepare( + ` + SELECT id FROM ${CONVERSATIONS_TABLE} WHERE type = 'private' AND (active_at IS NULL OR active_at = 0)` + ) + .all(); + + const ourNumber = getItemById('number_id'); + if (!ourNumber || !ourNumber.value) { + return; + } + const ourPubkey = ourNumber.value.split('.')[0]; + + const allInactiveAndWithoutMessagesConvo = allInactiveConvos + .map(c => c.id as string) + .filter(convoId => { + return convoId !== ourPubkey && getMessagesCountBySender({ source: convoId }) === 0 + ? true + : false; + }); + if (allInactiveAndWithoutMessagesConvo.length) { + allInactiveAndWithoutMessagesConvo.forEach(convoId => { + console.info(`${convoId} is not active and has 0 message in the history. Removing it`); + assertGlobalInstance() + .prepare(`DELETE FROM ${CONVERSATIONS_TABLE} WHERE id = $id;`) + .run({ id: convoId }); + }); + } +} + // tslint:disable: binary-expression-operand-order // tslint:disable: insecure-random @@ -3524,6 +3621,7 @@ export const sqlNode = { getUnreadByConversation, getUnreadCountByConversation, getMessageCountByType, + getMessageBySenderAndSentAt, filterAlreadyFetchedOpengroupMessage, getMessageBySenderAndTimestamp, diff --git a/ts/node/sql_channel.ts b/ts/node/sql_channel.ts index a4e578a91..4e4b7d8d3 100644 --- a/ts/node/sql_channel.ts +++ b/ts/node/sql_channel.ts @@ -42,7 +42,6 @@ export function initializeSqlChannel() { } }); - console.warn('********* registering get-user-data-path'); ipcMain.handle('get-user-data-path', () => { return app.getPath('userData'); }); diff --git a/ts/receiver/queuedJob.ts b/ts/receiver/queuedJob.ts index fb2edf970..334c942c5 100644 --- a/ts/receiver/queuedJob.ts +++ b/ts/receiver/queuedJob.ts @@ -374,6 +374,10 @@ export async function handleMessageJob( conversation.updateLastMessage(); await conversation.commit(); + if (conversation.id !== sendingDeviceConversation.id) { + await sendingDeviceConversation.commit(); + } + void queueAttachmentDownloads(messageModel, conversation); // Check if we need to update any profile names // the only profile we don't update with what is coming here is ours,