drop sourceDevice and other unused json fields from msg table

ts/models/message.ts

@@ -51,7 +51,7 @@ import { getV2OpenGroupRoom } from '../data/opengroups';
 import { isUsFromCache } from '../session/utils/User';
 import { perfEnd, perfStart } from '../session/utils/Performance';
 import { AttachmentTypeWithPath, isVoiceMessage } from '../types/Attachment';
-import _ from 'lodash';
+import _, { isEmpty } from 'lodash';
 import { SettingsKey } from '../data/settings-key';
 import {
   deleteExternalMessageFiles,
@@ -611,7 +611,7 @@ export class MessageModel extends Backbone.Model<MessageAttributes> {
       if (quote.text) {
         // do not show text of not found messages.
         // if the message was deleted better not show it's text content in the message
-        quoteProps.text = this.createNonBreakingLastSeparator(quote.text);
+        quoteProps.text = this.createNonBreakingLastSeparator(sliceQuoteText(quote.text));
       }
 
       const quoteAttachment = firstAttachment
@@ -1313,6 +1313,14 @@ export class MessageModel extends Backbone.Model<MessageAttributes> {
   }
 }
 
+// this is to avoid saving 2k chars for just the quote object inside a message
+export function sliceQuoteText(quotedText: string | undefined | null) {
+  if (!quotedText || isEmpty(quotedText)) {
+    return '';
+  }
+  return quotedText.slice(0, 60);
+}
+
 const trotthledAllMessagesDispatch = _.debounce(
   () => {
     if (updatesToDispatch.size === 0) {

ts/node/sql.ts

@@ -758,6 +758,7 @@ const LOKI_SCHEMA_VERSIONS = [
   updateToLokiSchemaVersion19,
   updateToLokiSchemaVersion20,
   updateToLokiSchemaVersion21,
+  updateToLokiSchemaVersion22,
 ];
 
 function updateToLokiSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) {
@@ -1241,11 +1242,6 @@ function updateToLokiSchemaVersion19(currentVersion: number, db: BetterSqlite3.D
       DROP INDEX messages_schemaVersion;
       ALTER TABLE ${MESSAGES_TABLE} DROP COLUMN schemaVersion;
     `);
-    // this is way to slow for now...
-    // db.exec(`
-    //   UPDATE ${MESSAGES_TABLE} SET
-    //   json = json_remove(json, '$.schemaVersion')
-    // `);
     writeLokiSchemaVersion(targetVersion, db);
   })();
 
@@ -1319,6 +1315,36 @@ function updateToLokiSchemaVersion21(currentVersion: number, db: BetterSqlite3.D
   console.log(`updateToLokiSchemaVersion${targetVersion}: success!`);
 }
 
+function updateToLokiSchemaVersion22(currentVersion: number, db: BetterSqlite3.Database) {
+  const targetVersion = 22;
+  if (currentVersion >= targetVersion) {
+    return;
+  }
+  console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`);
+
+  db.transaction(() => {
+    db.exec(`
+        DROP INDEX messages_duplicate_check;
+      `);
+
+    db.exec(`
+    ALTER TABLE ${MESSAGES_TABLE} DROP sourceDevice;
+    `);
+    db.exec(`
+    ALTER TABLE unprocessed DROP sourceDevice;
+    `);
+    db.exec(`
+    CREATE INDEX messages_duplicate_check ON ${MESSAGES_TABLE} (
+      source,
+      sent_at
+    );
+    `);
+
+    writeLokiSchemaVersion(targetVersion, db);
+  })();
+  console.log(`updateToLokiSchemaVersion${targetVersion}: success!`);
+}
+
 function writeLokiSchemaVersion(newVersion: number, db: BetterSqlite3.Database) {
   db.prepare(
     `INSERT INTO loki_schema(
@@ -1982,7 +2008,6 @@ function saveMessage(data: any) {
     // eslint-disable-next-line camelcase
     sent_at,
     source,
-    sourceDevice,
     type,
     unread,
     expireTimer,
@@ -2015,7 +2040,6 @@ function saveMessage(data: any) {
     sent,
     sent_at,
     source,
-    sourceDevice,
     type: type || '',
     unread,
   };
@@ -2039,7 +2063,6 @@ function saveMessage(data: any) {
     sent,
     sent_at,
     source,
-    sourceDevice,
     type,
     unread
   ) values (
@@ -2059,7 +2082,6 @@ function saveMessage(data: any) {
     $sent,
     $sent_at,
     $source,
-    $sourceDevice,
     $type,
     $unread
   );`
@@ -2793,13 +2815,12 @@ function updateUnprocessedAttempts(id: string, attempts: number) {
     });
 }
 function updateUnprocessedWithData(id: string, data: any = {}) {
-  const { source, sourceDevice, serverTimestamp, decrypted, senderIdentity } = data;
+  const { source, serverTimestamp, decrypted, senderIdentity } = data;
 
   assertGlobalInstance()
     .prepare(
       `UPDATE unprocessed SET
       source = $source,
-      sourceDevice = $sourceDevice,
       serverTimestamp = $serverTimestamp,
       decrypted = $decrypted,
       senderIdentity = $senderIdentity
@@ -2808,7 +2829,6 @@ function updateUnprocessedWithData(id: string, data: any = {}) {
     .run({
       id,
       source,
-      sourceDevice,
       serverTimestamp,
       decrypted,
       senderIdentity,
@@ -3442,6 +3462,19 @@ function cleanUpUnusedNodeForKeyEntries() {
   }
 }
 
+function cleanUpMessagesJson() {
+  console.info('cleanUpMessagesJson ');
+  const start = Date.now();
+  assertGlobalInstance().transaction(() => {
+    assertGlobalInstance().exec(`
+      UPDATE ${MESSAGES_TABLE} SET
+      json = json_remove(json, '$.schemaVersion', '$.recipients', '$.decrypted_at', '$.sourceDevice')
+    `);
+  })();
+
+  console.info(`cleanUpMessagesJson took ${Date.now() - start}ms`);
+}
+
 function cleanUpOldOpengroups() {
   const v2Convos = getAllOpenGroupV2Conversations();
 
@@ -3642,7 +3675,6 @@ function fillWithTestData(numConvosToAdd: number, numMsgsToAdd: number) {
       // eslint-disable-next-line camelcase
       sent_at: Date.now(),
       source: `${convoId}`,
-      sourceDevice: 1,
       type: 'outgoing',
       unread: 1,
       expireTimer: 0,

ts/receiver/queuedJob.ts

@@ -5,7 +5,7 @@ import { PubKey } from '../session/types';
 import _ from 'lodash';
 import { getConversationController } from '../session/conversations';
 import { ConversationModel, ConversationTypeEnum } from '../models/conversation';
-import { MessageModel } from '../models/message';
+import { MessageModel, sliceQuoteText } from '../models/message';
 import { getMessageCountByType, getMessagesBySentAt } from '../../ts/data/data';
 
 import { SignalService } from '../protobuf';
@@ -66,7 +66,7 @@ async function copyFromQuotedMessage(
   window?.log?.info(`Found quoted message id: ${id}`);
   quoteLocal.referencedMessageNotFound = false;
 
-  quoteLocal.text = found.get('body') || '';
+  quoteLocal.text = sliceQuoteText(found.get('body') || '');
 
   // no attachments, just save the quote with the body
   if (

ts/receiver/receiver.ts

@@ -174,7 +174,6 @@ async function queueCached(item: any) {
     envelope.source = envelope.source || item.source;
 
     // Why do we need to do this???
-    envelope.sourceDevice = 1;
     envelope.senderIdentity = envelope.senderIdentity || item.senderIdentity;
     envelope.serverTimestamp = envelope.serverTimestamp || item.serverTimestamp;
 

ts/types/Message.ts

@@ -17,7 +17,6 @@ export type IncomingMessage = Readonly<
     expireTimer?: number;
     flags?: number;
     source?: string;
-    sourceDevice?: number;
   } & SharedMessageProperties &
     ExpirationTimerUpdate
 >;