From 368c0cd01bcf409ae6630552242531759633ad11 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 21 Apr 2022 16:06:20 +1000 Subject: [PATCH] drop sourceDevice and other unused json fields from msg table --- ts/models/message.ts | 12 +++++++-- ts/node/sql.ts | 58 +++++++++++++++++++++++++++++++--------- ts/receiver/queuedJob.ts | 4 +-- ts/receiver/receiver.ts | 1 - ts/types/Message.ts | 1 - 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/ts/models/message.ts b/ts/models/message.ts index f6332ccf2..00a1734a4 100644 --- a/ts/models/message.ts +++ b/ts/models/message.ts @@ -51,7 +51,7 @@ import { getV2OpenGroupRoom } from '../data/opengroups'; import { isUsFromCache } from '../session/utils/User'; import { perfEnd, perfStart } from '../session/utils/Performance'; import { AttachmentTypeWithPath, isVoiceMessage } from '../types/Attachment'; -import _ from 'lodash'; +import _, { isEmpty } from 'lodash'; import { SettingsKey } from '../data/settings-key'; import { deleteExternalMessageFiles, @@ -611,7 +611,7 @@ export class MessageModel extends Backbone.Model { if (quote.text) { // do not show text of not found messages. // if the message was deleted better not show it's text content in the message - quoteProps.text = this.createNonBreakingLastSeparator(quote.text); + quoteProps.text = this.createNonBreakingLastSeparator(sliceQuoteText(quote.text)); } const quoteAttachment = firstAttachment @@ -1313,6 +1313,14 @@ export class MessageModel extends Backbone.Model { } } +// this is to avoid saving 2k chars for just the quote object inside a message +export function sliceQuoteText(quotedText: string | undefined | null) { + if (!quotedText || isEmpty(quotedText)) { + return ''; + } + return quotedText.slice(0, 60); +} + const trotthledAllMessagesDispatch = _.debounce( () => { if (updatesToDispatch.size === 0) { diff --git a/ts/node/sql.ts b/ts/node/sql.ts index 9a434b633..47df2b949 100644 --- a/ts/node/sql.ts +++ b/ts/node/sql.ts @@ -758,6 +758,7 @@ const LOKI_SCHEMA_VERSIONS = [ updateToLokiSchemaVersion19, updateToLokiSchemaVersion20, updateToLokiSchemaVersion21, + updateToLokiSchemaVersion22, ]; function updateToLokiSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { @@ -1241,11 +1242,6 @@ function updateToLokiSchemaVersion19(currentVersion: number, db: BetterSqlite3.D DROP INDEX messages_schemaVersion; ALTER TABLE ${MESSAGES_TABLE} DROP COLUMN schemaVersion; `); - // this is way to slow for now... - // db.exec(` - // UPDATE ${MESSAGES_TABLE} SET - // json = json_remove(json, '$.schemaVersion') - // `); writeLokiSchemaVersion(targetVersion, db); })(); @@ -1319,6 +1315,36 @@ function updateToLokiSchemaVersion21(currentVersion: number, db: BetterSqlite3.D console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } +function updateToLokiSchemaVersion22(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 22; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + DROP INDEX messages_duplicate_check; + `); + + db.exec(` + ALTER TABLE ${MESSAGES_TABLE} DROP sourceDevice; + `); + db.exec(` + ALTER TABLE unprocessed DROP sourceDevice; + `); + db.exec(` + CREATE INDEX messages_duplicate_check ON ${MESSAGES_TABLE} ( + source, + sent_at + ); + `); + + writeLokiSchemaVersion(targetVersion, db); + })(); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); +} + function writeLokiSchemaVersion(newVersion: number, db: BetterSqlite3.Database) { db.prepare( `INSERT INTO loki_schema( @@ -1982,7 +2008,6 @@ function saveMessage(data: any) { // eslint-disable-next-line camelcase sent_at, source, - sourceDevice, type, unread, expireTimer, @@ -2015,7 +2040,6 @@ function saveMessage(data: any) { sent, sent_at, source, - sourceDevice, type: type || '', unread, }; @@ -2039,7 +2063,6 @@ function saveMessage(data: any) { sent, sent_at, source, - sourceDevice, type, unread ) values ( @@ -2059,7 +2082,6 @@ function saveMessage(data: any) { $sent, $sent_at, $source, - $sourceDevice, $type, $unread );` @@ -2793,13 +2815,12 @@ function updateUnprocessedAttempts(id: string, attempts: number) { }); } function updateUnprocessedWithData(id: string, data: any = {}) { - const { source, sourceDevice, serverTimestamp, decrypted, senderIdentity } = data; + const { source, serverTimestamp, decrypted, senderIdentity } = data; assertGlobalInstance() .prepare( `UPDATE unprocessed SET source = $source, - sourceDevice = $sourceDevice, serverTimestamp = $serverTimestamp, decrypted = $decrypted, senderIdentity = $senderIdentity @@ -2808,7 +2829,6 @@ function updateUnprocessedWithData(id: string, data: any = {}) { .run({ id, source, - sourceDevice, serverTimestamp, decrypted, senderIdentity, @@ -3442,6 +3462,19 @@ function cleanUpUnusedNodeForKeyEntries() { } } +function cleanUpMessagesJson() { + console.info('cleanUpMessagesJson '); + const start = Date.now(); + assertGlobalInstance().transaction(() => { + assertGlobalInstance().exec(` + UPDATE ${MESSAGES_TABLE} SET + json = json_remove(json, '$.schemaVersion', '$.recipients', '$.decrypted_at', '$.sourceDevice') + `); + })(); + + console.info(`cleanUpMessagesJson took ${Date.now() - start}ms`); +} + function cleanUpOldOpengroups() { const v2Convos = getAllOpenGroupV2Conversations(); @@ -3642,7 +3675,6 @@ function fillWithTestData(numConvosToAdd: number, numMsgsToAdd: number) { // eslint-disable-next-line camelcase sent_at: Date.now(), source: `${convoId}`, - sourceDevice: 1, type: 'outgoing', unread: 1, expireTimer: 0, diff --git a/ts/receiver/queuedJob.ts b/ts/receiver/queuedJob.ts index 334c942c5..6ed507256 100644 --- a/ts/receiver/queuedJob.ts +++ b/ts/receiver/queuedJob.ts @@ -5,7 +5,7 @@ import { PubKey } from '../session/types'; import _ from 'lodash'; import { getConversationController } from '../session/conversations'; import { ConversationModel, ConversationTypeEnum } from '../models/conversation'; -import { MessageModel } from '../models/message'; +import { MessageModel, sliceQuoteText } from '../models/message'; import { getMessageCountByType, getMessagesBySentAt } from '../../ts/data/data'; import { SignalService } from '../protobuf'; @@ -66,7 +66,7 @@ async function copyFromQuotedMessage( window?.log?.info(`Found quoted message id: ${id}`); quoteLocal.referencedMessageNotFound = false; - quoteLocal.text = found.get('body') || ''; + quoteLocal.text = sliceQuoteText(found.get('body') || ''); // no attachments, just save the quote with the body if ( diff --git a/ts/receiver/receiver.ts b/ts/receiver/receiver.ts index ce2b85edd..8aabb27e8 100644 --- a/ts/receiver/receiver.ts +++ b/ts/receiver/receiver.ts @@ -174,7 +174,6 @@ async function queueCached(item: any) { envelope.source = envelope.source || item.source; // Why do we need to do this??? - envelope.sourceDevice = 1; envelope.senderIdentity = envelope.senderIdentity || item.senderIdentity; envelope.serverTimestamp = envelope.serverTimestamp || item.serverTimestamp; diff --git a/ts/types/Message.ts b/ts/types/Message.ts index 1c0edf933..d21f6a037 100644 --- a/ts/types/Message.ts +++ b/ts/types/Message.ts @@ -17,7 +17,6 @@ export type IncomingMessage = Readonly< expireTimer?: number; flags?: number; source?: string; - sourceDevice?: number; } & SharedMessageProperties & ExpirationTimerUpdate >;