From 28892ce8cc56e6074b1f49bec6e35627ec419c8e Mon Sep 17 00:00:00 2001
From: Audric Ackermann <audric@loki.network>
Date: Tue, 21 Dec 2021 15:41:49 +1100
Subject: [PATCH] remove the deduplication by hash for opengroup messages

Relates #2069
---
 ts/receiver/dataMessage.ts         | 25 ++++++++--------
 ts/receiver/hashDuplicateFilter.ts | 48 ------------------------------
 ts/receiver/receiver.ts            |  7 +----
 3 files changed, 13 insertions(+), 67 deletions(-)
 delete mode 100644 ts/receiver/hashDuplicateFilter.ts

diff --git a/ts/receiver/dataMessage.ts b/ts/receiver/dataMessage.ts
index 1be4dafac..739ca67c4 100644
--- a/ts/receiver/dataMessage.ts
+++ b/ts/receiver/dataMessage.ts
@@ -362,29 +362,28 @@ export async function isMessageDuplicate({
   serverTimestamp,
 }: MessageId) {
   const { Errors } = window.Signal.Types;
-  // serverId is only used for opengroupv2
+  // serverTimestamp is only used for opengroupv2
   try {
     let result;
     if (serverTimestamp) {
       // first try to find a duplicate with the same serverTimestamp from this sender
-      if (serverTimestamp) {
-        result = await getMessageBySenderAndServerTimestamp({
-          source,
-          serverTimestamp,
-        });
-      }
+
+      result = await getMessageBySenderAndServerTimestamp({
+        source,
+        serverTimestamp,
+      });
+
       // if we have a result, it means a specific user sent two messages either with the same serverTimestamp.
       // no need to do anything else, those messages must be the same
       // Note: this test is not based on which conversation the user sent the message
       // but we consider that a user sending two messages with the same serverTimestamp is unlikely
       return Boolean(result);
-    } else {
-      result = await getMessageBySender({
-        source,
-        sourceDevice,
-        sentAt: timestamp,
-      });
     }
+    result = await getMessageBySender({
+      source,
+      sourceDevice,
+      sentAt: timestamp,
+    });
 
     if (!result) {
       return false;
diff --git a/ts/receiver/hashDuplicateFilter.ts b/ts/receiver/hashDuplicateFilter.ts
deleted file mode 100644
index b7680cc41..000000000
--- a/ts/receiver/hashDuplicateFilter.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-import _ from 'lodash';
-import { SignalService } from '../protobuf';
-import { sha256 } from '../session/crypto';
-
-const recentHashByConvo = new Map<string, Array<string>>();
-
-const maxHashToKeepPerConvo = 10;
-
-export function isDuplicateBasedOnHash(
-  dataMessage: SignalService.DataMessage,
-  conversationId: string,
-  sender: string
-): boolean {
-  const toUseForHash = {
-    ..._.omit(
-      SignalService.DataMessage.toObject(dataMessage),
-      'timestamp',
-      'profile',
-      'preview',
-      'profileKey'
-    ),
-    conversationId,
-    sender,
-  };
-
-  if (!recentHashByConvo.has(conversationId)) {
-    recentHashByConvo.set(conversationId, new Array());
-  }
-  const newHash = sha256(JSON.stringify(toUseForHash));
-
-  // this can only be set based on the .set above()
-  let recentHashForConvo = recentHashByConvo.get(conversationId) as Array<string>;
-
-  // this hash already exists for this convo
-  if (recentHashForConvo.some(n => n === newHash)) {
-    return true;
-  }
-  // push the new hash at the end
-  recentHashForConvo.push(newHash);
-  if (recentHashForConvo.length > maxHashToKeepPerConvo) {
-    // slice the last maxHashToKeepPerConvo hashes
-    recentHashForConvo = recentHashForConvo?.slice(-maxHashToKeepPerConvo);
-  }
-  recentHashByConvo.set(conversationId, recentHashForConvo);
-  return false;
-}
-
-// build a hash of the data and check against recent messages
diff --git a/ts/receiver/receiver.ts b/ts/receiver/receiver.ts
index abb9c2f93..0e2212ce1 100644
--- a/ts/receiver/receiver.ts
+++ b/ts/receiver/receiver.ts
@@ -26,7 +26,6 @@ import { OpenGroupRequestCommonType } from '../session/apis/open_group_api/openg
 import { handleMessageJob } from './queuedJob';
 import { fromBase64ToArray } from '../session/utils/String';
 import { removeMessagePadding } from '../session/crypto/BufferPadding';
-import { isDuplicateBasedOnHash } from './hashDuplicateFilter';
 import { createTaskWithTimeout } from '../session/utils/TaskWithTimeout';
 import { perfEnd, perfStart } from '../session/utils/Performance';
 
@@ -317,7 +316,7 @@ export async function handleOpenGroupV2Message(
       source: sender,
       message: dataMessage,
     };
-    // WARNING this is very important that the isMessageDuplicate is made in the conversation.queueJob
+    // WARNING this is important that the isMessageDuplicate is made in the conversation.queueJob
     const isDuplicate = await isMessageDuplicate(messageCreationData);
 
     if (isDuplicate) {
@@ -325,10 +324,6 @@ export async function handleOpenGroupV2Message(
       return;
     }
 
-    if (isDuplicateBasedOnHash(dataMessage, conversationId, sender)) {
-      window?.log?.info('Received duplicate message based on hash. Dropping it.');
-      return;
-    }
     // this line just create an empty message with some basic stuff set.
     // the whole decoding of data is happening in handleMessageJob()
     const msg = createMessage(messageCreationData, !isMe);