From 712566ef3bd224dd91099ce85758cbd76899f639 Mon Sep 17 00:00:00 2001
From: Beaudan <beau@loki.network>
Date: Mon, 19 Nov 2018 14:51:21 +1100
Subject: [PATCH] Created new table to store the received message hashes.
 Checking this table when receiving messages to look for duplicates. Should be
 cleared of expired messages on app start and every hour after

---
 app/sql.js                      | 63 +++++++++++++++++++++++++++++++++
 js/background.js                |  6 ++++
 js/modules/data.js              | 23 ++++++++++++
 libtextsecure/http-resources.js | 14 +++++++-
 4 files changed, 105 insertions(+), 1 deletion(-)

diff --git a/app/sql.js b/app/sql.js
index e87df9130..a448be12a 100644
--- a/app/sql.js
+++ b/app/sql.js
@@ -89,6 +89,9 @@ module.exports = {
 
   getMessageCount,
   saveMessage,
+  cleanSeenMessages,
+  saveSeenMessageHashes,
+  saveSeenMessageHash,
   saveMessages,
   removeMessage,
   getUnreadByConversation,
@@ -97,6 +100,7 @@ module.exports = {
   getAllMessages,
   getAllMessageIds,
   getMessagesBySentAt,
+  getSeenMessagesByHashList,
   getExpiredMessages,
   getOutgoingWithoutExpiresAt,
   getNextExpiringMessage,
@@ -388,6 +392,14 @@ async function updateToSchemaVersion6(currentVersion, instance) {
   console.log('updateToSchemaVersion6: starting...');
   await instance.run('BEGIN TRANSACTION;');
 
+  await instance.run(
+    `CREATE TABLE seenMessages(
+      id STRING PRIMARY KEY ASC,
+      expiresAt INTEGER,
+      hash STRING
+    );`
+  );
+
   // key-value, ids are strings, one extra column
   await instance.run(
     `CREATE TABLE sessions(
@@ -1223,6 +1235,48 @@ async function saveMessage(data, { forceSave } = {}) {
   return toCreate.id;
 }
 
+async function saveSeenMessageHashes(arrayOfHashes) {
+  let promise;
+
+  db.serialize(() => {
+    promise = Promise.all([
+      db.run('BEGIN TRANSACTION;'),
+      ...map(arrayOfHashes, hashData => saveSeenMessageHash(hashData)),
+      db.run('COMMIT TRANSACTION;'),
+    ]);
+  });
+
+  await promise;
+}
+
+async function saveSeenMessageHash(data) {
+  const {
+    expiresAt,
+    hash,
+  } = data;
+  await db.run(
+    `INSERT INTO seenMessages (
+      id,
+      expiresAt,
+      hash
+    ) values (
+      $id,
+      $expiresAt,
+      $hash
+    );`, {
+      $id: generateUUID(),
+      $expiresAt: expiresAt,
+      $hash: hash,
+    }
+  );
+}
+
+async function cleanSeenMessages() {
+  await db.run('DELETE FROM seenMessages WHERE expiresAt <= $now;', {
+    $now: Date.now(),
+  });
+}
+
 async function saveMessages(arrayOfMessages, { forceSave } = {}) {
   let promise;
 
@@ -1343,6 +1397,15 @@ async function getMessagesBySentAt(sentAt) {
   return map(rows, row => jsonToObject(row.json));
 }
 
+async function getSeenMessagesByHashList(hashes) {
+  const rows = await db.all(
+    `SELECT * FROM seenMessages WHERE hash IN ( ${hashes.map(() => '?').join(', ')} );`,
+     hashes
+  );
+
+  return map(rows, row => row.hash);
+}
+
 async function getExpiredMessages() {
   const now = Date.now();
 
diff --git a/js/background.js b/js/background.js
index 99920f195..ee550a0d3 100644
--- a/js/background.js
+++ b/js/background.js
@@ -456,7 +456,13 @@
     }
   });
 
+  function manageSeenMessages() {
+    window.Signal.Data.cleanSeenMessages();
+    setTimeout(manageSeenMessages, 1000 * 60 * 60);
+  }
+
   async function start() {
+    manageSeenMessages();
     window.dispatchEvent(new Event('storage_ready'));
 
     window.log.info('listening for registration events');
diff --git a/js/modules/data.js b/js/modules/data.js
index 5d4bfac86..47856323c 100644
--- a/js/modules/data.js
+++ b/js/modules/data.js
@@ -122,6 +122,9 @@ module.exports = {
 
   getMessageCount,
   saveMessage,
+  cleanSeenMessages,
+  saveSeenMessageHash,
+  saveSeenMessageHashes,
   saveLegacyMessage,
   saveMessages,
   removeMessage,
@@ -139,6 +142,7 @@ module.exports = {
   getOutgoingWithoutExpiresAt,
   getNextExpiringMessage,
   getMessagesByConversation,
+  getSeenMessagesByHashList,
 
   getUnprocessedCount,
   getAllUnprocessed,
@@ -727,6 +731,18 @@ async function getMessageCount() {
   return channels.getMessageCount();
 }
 
+async function cleanSeenMessages() {
+  await channels.cleanSeenMessages();
+}
+
+async function saveSeenMessageHashes(data) {
+  await channels.saveSeenMessageHashes(_cleanData(data));
+}
+
+async function saveSeenMessageHash(data) {
+  await channels.saveSeenMessageHash(_cleanData(data));
+}
+
 async function saveMessage(data, { forceSave, Message } = {}) {
   const updated = keysFromArrayBuffer(MESSAGE_PRE_KEYS, data);
   const id = await channels.saveMessage(_cleanData(updated), { forceSave });
@@ -854,6 +870,13 @@ async function getMessagesByConversation(
   return new MessageCollection(encoded);
 }
 
+async function getSeenMessagesByHashList(
+  hashes
+) {
+  const seenMessages = await channels.getSeenMessagesByHashList(hashes);
+  return seenMessages;
+}
+
 async function removeAllMessagesInConversation(
   conversationId,
   { MessageCollection }
diff --git a/libtextsecure/http-resources.js b/libtextsecure/http-resources.js
index bb1fa9ee0..08934dc39 100644
--- a/libtextsecure/http-resources.js
+++ b/libtextsecure/http-resources.js
@@ -68,7 +68,19 @@
         setTimeout(() => { pollServer(callBack); }, 5000);
         return;
       }
-      result.messages.forEach(async message => {
+      const incomingHashes = result.messages.map(m => m.hash);
+      const dupHashes = await window.Signal.Data.getSeenMessagesByHashList(incomingHashes);
+      if (incomingHashes.length === dupHashes.length) {
+        setTimeout(() => { pollServer(callBack); }, 5000);
+        return;
+      }
+      const NewMessages = result.messages.filter(m => !dupHashes.includes(m.hash));
+      const NewHashes = NewMessages.map(m => ({
+        expiresAt: m.expiration,
+        hash: m.hash,
+      }));
+      await window.Signal.Data.saveMessageHashes(NewHashes);
+      NewMessages.forEach(async message => {
         const { data } = message;
         const dataPlaintext = stringToArrayBufferBase64(data);
         const messageBuf = textsecure.protobuf.WebSocketMessage.decode(dataPlaintext);