fix deduplication using a hash of recent messages

ts/receiver/dataMessage.ts

@@ -620,12 +620,6 @@ export async function handleMessageEvent(event: MessageEvent): Promise<void> {
   // if the message is `sent` (from secondary device) we have to set the sender manually... (at least for now)
   source = source || msg.get('source');
 
-  if (await isMessageDuplicate(data)) {
-    window?.log?.info('Received duplicate message. Dropping it.');
-    confirm();
-    return;
-  }
-
   const isOurDevice = UserUtils.isUsFromCache(source);
 
   const shouldSendReceipt = isIncoming && !isGroupMessage && !isOurDevice;
@@ -664,10 +658,16 @@ export async function handleMessageEvent(event: MessageEvent): Promise<void> {
 
   if (!conversation) {
     window?.log?.warn('Skipping handleJob for unknown convo: ', conversationId);
+    confirm();
     return;
   }
 
   conversation.queueJob(async () => {
+    if (await isMessageDuplicate(data)) {
+      window?.log?.info('Received duplicate message. Dropping it.');
+      confirm();
+      return;
+    }
     await handleMessageJob(msg, conversation, message, ourNumber, confirm, source);
   });
 }

ts/receiver/hashDuplicateFilter.ts

@@ -0,0 +1,48 @@
+import _ from 'lodash';
+import { SignalService } from '../protobuf';
+import { sha256 } from '../session/crypto';
+
+const recentHashByConvo = new Map<string, Array<string>>();
+
+const maxHashToKeepPerConvo = 50;
+
+export function isDuplicateBasedOnHash(
+  dataMessage: SignalService.DataMessage,
+  conversationId: string,
+  sender: string
+): boolean {
+  const toUseForHash = {
+    ..._.omit(
+      SignalService.DataMessage.toObject(dataMessage),
+      'timestamp',
+      'profile',
+      'preview',
+      'profileKey'
+    ),
+    conversationId,
+    sender,
+  };
+
+  if (!recentHashByConvo.has(conversationId)) {
+    recentHashByConvo.set(conversationId, new Array());
+  }
+  const newHash = sha256(JSON.stringify(toUseForHash));
+
+  // this can only be set based on the .set above()
+  let recentHashForConvo = recentHashByConvo.get(conversationId) as Array<string>;
+
+  // this hash already exists for this convo
+  if (recentHashForConvo.some(n => n === newHash)) {
+    return true;
+  }
+  // push the new hash at the end
+  recentHashForConvo.push(newHash);
+  if (recentHashForConvo.length > maxHashToKeepPerConvo) {
+    // slice the last maxHashToKeepPerConvo hashes
+    recentHashForConvo = recentHashForConvo?.slice(-maxHashToKeepPerConvo);
+  }
+  recentHashByConvo.set(conversationId, recentHashForConvo);
+  return false;
+}
+
+// build a hash of the data and check against recent messages

ts/receiver/receiver.ts

@@ -37,6 +37,7 @@ import { OpenGroupRequestCommonType } from '../opengroup/opengroupV2/ApiUtil';
 import { handleMessageJob } from './queuedJob';
 import { fromBase64ToArray } from '../session/utils/String';
 import { removeMessagePadding } from '../session/crypto/BufferPadding';
+import { isDuplicateBasedOnHash } from './hashDuplicateFilter';
 
 // TODO: check if some of these exports no longer needed
 
@@ -330,11 +331,12 @@ export async function handleOpenGroupV2Message(
     window?.log?.error('We cannot handle a message without a conversationId');
     return;
   }
-  const dataMessage = decoded?.dataMessage;
-  if (!dataMessage) {
+  const idataMessage = decoded?.dataMessage;
+  if (!idataMessage) {
     window?.log?.error('Invalid decoded opengroup message: no dataMessage');
     return;
   }
+  const dataMessage = idataMessage as SignalService.DataMessage;
 
   if (!ConversationController.getInstance().get(conversationId)) {
     window?.log?.error('Received a message for an unknown convo. Skipping');
@@ -372,11 +374,16 @@ export async function handleOpenGroupV2Message(
     };
     // WARNING this is very important that the isMessageDuplicate is made in the conversation.queueJob
     const isDuplicate = await isMessageDuplicate(messageCreationData);
+
     if (isDuplicate) {
       window?.log?.info('Received duplicate message. Dropping it.');
       return;
     }
 
+    if (isDuplicateBasedOnHash(dataMessage, conversationId, sender)) {
+      window?.log?.info('Received duplicate message based on hash. Dropping it.');
+      return;
+    }
     // this line just create an empty message with some basic stuff set.
     // the whole decoding of data is happening in handleMessageJob()
     const msg = createMessage(messageCreationData, !isMe);