Heap of linting, eslint warning/error removal, and fixed a couple small bugs found along the way

Created new table to store the received message hashes. Checking this table when receiving messages to look for duplicates. Should be cleared of expired messages on app start and every hour after

Removed id which was not needed for seen messages. Refactored filter logic into function and found function name error

create unique index for contact prekeys (to allow using REPLACE)

Fixed lint stuff that merge brought back

app/sql.js

@@ -89,6 +89,9 @@ module.exports = {
 
   getMessageCount,
   saveMessage,
+  cleanSeenMessages,
+  saveSeenMessageHashes,
+  saveSeenMessageHash,
   saveMessages,
   removeMessage,
   getUnreadByConversation,
@@ -98,6 +101,7 @@ module.exports = {
   getAllMessageIds,
   getAllUnsentMessages,
   getMessagesBySentAt,
+  getSeenMessagesByHashList,
   getExpiredMessages,
   getOutgoingWithoutExpiresAt,
   getNextExpiringMessage,
@@ -390,6 +394,13 @@ async function updateToSchemaVersion6(currentVersion, instance) {
   console.log('updateToSchemaVersion6: starting...');
   await instance.run('BEGIN TRANSACTION;');
 
+  await instance.run(
+    `CREATE TABLE seenMessages(
+      hash STRING PRIMARY KEY,
+      expiresAt INTEGER
+    );`
+  );
+
   // key-value, ids are strings, one extra column
   await instance.run(
     `CREATE TABLE sessions(
@@ -447,6 +458,11 @@ async function updateToSchemaVersion6(currentVersion, instance) {
     );`
   );
 
+  await instance.run(`CREATE UNIQUE INDEX contact_prekey_identity_key_string_keyid ON contactPreKeys (
+    identityKeyString,
+    keyId
+  );`);
+
   await instance.run(
     `CREATE TABLE contactSignedPreKeys(
       id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
@@ -456,6 +472,11 @@ async function updateToSchemaVersion6(currentVersion, instance) {
     );`
   );
 
+  await instance.run(`CREATE UNIQUE INDEX contact_signed_prekey_identity_key_string_keyid ON contactSignedPreKeys (
+    identityKeyString,
+    keyId
+  );`);
+
   await instance.run('PRAGMA schema_version = 6;');
   await instance.run('COMMIT TRANSACTION;');
   console.log('updateToSchemaVersion6: success!');
@@ -1118,6 +1139,7 @@ async function saveMessage(data, { forceSave } = {}) {
     schemaVersion,
     // eslint-disable-next-line camelcase
     sent,
+    // eslint-disable-next-line camelcase
     sent_at,
     source,
     sourceDevice,
@@ -1230,6 +1252,45 @@ async function saveMessage(data, { forceSave } = {}) {
   return toCreate.id;
 }
 
+async function saveSeenMessageHashes(arrayOfHashes) {
+  let promise;
+
+  db.serialize(() => {
+    promise = Promise.all([
+      db.run('BEGIN TRANSACTION;'),
+      ...map(arrayOfHashes, hashData => saveSeenMessageHash(hashData)),
+      db.run('COMMIT TRANSACTION;'),
+    ]);
+  });
+
+  await promise;
+}
+
+async function saveSeenMessageHash(data) {
+  const {
+    expiresAt,
+    hash,
+  } = data;
+  await db.run(
+    `INSERT INTO seenMessages (
+      expiresAt,
+      hash
+    ) values (
+      $expiresAt,
+      $hash
+    );`, {
+      $expiresAt: expiresAt,
+      $hash: hash,
+    }
+  );
+}
+
+async function cleanSeenMessages() {
+  await db.run('DELETE FROM seenMessages WHERE expiresAt <= $now;', {
+    $now: Date.now(),
+  });
+}
+
 async function saveMessages(arrayOfMessages, { forceSave } = {}) {
   let promise;
 
@@ -1360,6 +1421,15 @@ async function getMessagesBySentAt(sentAt) {
   return map(rows, row => jsonToObject(row.json));
 }
 
+async function getSeenMessagesByHashList(hashes) {
+  const rows = await db.all(
+    `SELECT * FROM seenMessages WHERE hash IN ( ${hashes.map(() => '?').join(', ')} );`,
+    hashes
+  );
+
+  return map(rows, row => row.hash);
+}
+
 async function getExpiredMessages() {
   const now = Date.now();
 

js/background.js

@@ -7,7 +7,6 @@
   Signal,
   storage,
   textsecure,
-  WebAPI
   Whisper,
 */
 
@@ -464,7 +463,13 @@
     }
   });
 
+  function manageSeenMessages() {
+    window.Signal.Data.cleanSeenMessages();
+    setTimeout(manageSeenMessages, 1000 * 60 * 60);
+  }
+
   async function start() {
+    manageSeenMessages();
     window.dispatchEvent(new Event('storage_ready'));
 
     window.log.info('listening for registration events');
@@ -575,7 +580,9 @@
       try {
         const conversation = ConversationController.get(pubKey);
         conversation.onCalculatingPoW(pubKey, timestamp);
-      } catch (e) {}
+      } catch (e) {
+        window.log.error('Error showing PoW cog');
+      }
     });
   }
 

js/models/conversations.js

@@ -1,8 +1,7 @@
 /* global _: false */
 /* global Backbone: false */
-/* global libphonenumber: false */
-
 /* global ConversationController: false */
+/* global i18n: false */
 /* global libsignal: false */
 /* global storage: false */
 /* global textsecure: false */
@@ -239,7 +238,7 @@
       }
     },
     // This goes through all our message history and finds a friend request
-    // But this is not a concurrent operation and thus `updatePendingFriendRequests` is used
+    // But this is not a concurrent operation and thus updatePendingFriendRequests is used
     async hasPendingFriendRequests() {
       // Go through the messages and check for any pending friend requests
       const messages = await window.Signal.Data.getMessagesByConversation(
@@ -249,15 +248,16 @@
           MessageCollection: Whisper.MessageCollection,
         }
       );
-
+      const pendingFriendRequest =
-      for (const message of messages.models) {
+        messages.models.find(message =>
-        if (message.isFriendRequest() && message.attributes.friendStatus === 'pending') return true;
+          message.isFriendRequest() &&
-      }
+          message.attributes.friendStatus === 'pending'
-
+        );
-      return false;
+      return pendingFriendRequest !== undefined;
     },
     async getPendingFriendRequests(direction) {
-      // Theoretically all ouur messages could be friend requests, thus we have to unfortunately go through each one :(
+      // Theoretically all our messages could be friend requests,
+      // thus we have to unfortunately go through each one :(
       const messages = await window.Signal.Data.getMessagesByConversation(
         this.id,
         {
@@ -267,9 +267,10 @@
       );
 
       // Get the messages that are matching the direction and the friendStatus
-      return messages.models.filter(m => {
+      return messages.models.filter(m =>
-        return (m.attributes.direction === direction && m.attributes.friendStatus === 'pending')
+        m.attributes.direction === direction &&
-      });
+        m.attributes.friendStatus === 'pending'
+      );
     },
     getPropsForListItem() {
       const result = {
@@ -469,7 +470,9 @@
     },
     async onFriendRequestAccepted({ updateUnread }) {
       // Make sure we don't keep incrementing the unread count
-      const unreadCount = this.isKeyExchangeCompleted() || !updateUnread ? {} : { unreadCount: this.get('unreadCount') + 1 };
+      const unreadCount = !updateUnread || this.isKeyExchangeCompleted()
+        ? {}
+        : { unreadCount: this.get('unreadCount') + 1 };
       this.set({
         friendRequestStatus: null,
         keyExchangeCompleted: true,
@@ -739,16 +742,15 @@
       if (_options.direction === 'incoming') {
         const requests = await this.getPendingFriendRequests('incoming');
 
-        for (const request of requests) {
         // Delete the old message if it's pending
-          await this._removeMessage(request.id);
+        await Promise.all(requests.map(async request => this._removeMessage(request.id)));
-        }
         // Trigger an update if we removed messages
         if (requests.length > 0)
           this.trigger('change');
       }
 
       // Add the new message
+      // eslint-disable-next-line camelcase
       const received_at = _options.received_at || Date.now();
       const message = {
         conversationId: this.id,
@@ -1024,26 +1026,31 @@
             recipients,
           });
         } else {
-            // We also need to make sure we don't send a new friend request if we already have an existing one
+          // We also need to make sure we don't send a new friend request
+          // if we already have an existing one
           const incomingRequests = await this.getPendingFriendRequests('incoming');
-            if (incomingRequests.length > 0) return;
+          if (incomingRequests.length > 0) return null;
 
           // Otherwise check if we have sent a friend request
           const outgoingRequests = await this.getPendingFriendRequests('outgoing');
           if (outgoingRequests.length > 0) {
-            // Check if the requests have errored, if so then remove them and send the new request if possible
+            // Check if the requests have errored, if so then remove them
-            const friendRequestSent = false;
+            // and send the new request if possible
-            for (const outgoing of outgoingRequests) {
+            let friendRequestSent = false;
+            const promises = [];
+            outgoingRequests.forEach(async outgoing => {
               if (outgoing.hasErrors()) {
-                await this._removeMessage(outgoing.id);
+                promises.push(this._removeMessage(outgoing.id));
               } else {
                 // No errors = we have sent over the friend request
                 friendRequestSent = true;
               }
-            }
+            });
+            await Promise.all(promises);
 
-            // If the requests didn't error then don't add a new friend request because one of them was sent successfully
+            // If the requests didn't error then don't add a new friend request
-            if (friendRequestSent) return;
+            // because one of them was sent successfully
+            if (friendRequestSent) return null;
           }
 
           // Send the friend request!
@@ -1114,8 +1121,8 @@
         const options = this.getSendOptions();
 
         // Add the message sending on another queue so that our UI doesn't get blocked
-        this.queueMessageSend(async () => {
+        this.queueMessageSend(async () =>
-          return message.send(
+          message.send(
             this.wrapSend(
               sendFunction(
                 destination,
@@ -1128,8 +1135,8 @@
                 options
               )
             )
+          )
         );
-        });
 
         return true;
       });
@@ -1148,13 +1155,12 @@
           this.trigger('disable:input', true);
           this.trigger('change:placeholder', 'disabled');
           return;
-        } else {
+        }
         // Tell the user to introduce themselves
         this.trigger('disable:input', false);
         this.trigger('change:placeholder', 'friend-request');
         return;
       }
-      }
       this.trigger('disable:input', false);
       this.trigger('change:placeholder', 'chat');
     },
@@ -2104,8 +2110,9 @@
     // Notification for friend request received
     async notifyFriendRequest(source, type) {
       // Data validation
-      if (!source) return Promise.reject('Invalid source');
+      if (!source) return Promise.reject(new Error('Invalid source'));
-      if (!['accepted', 'requested'].includes(type)) return Promise.reject('Type must be accepted or requested.');
+      if (!['accepted', 'requested'].includes(type))
+        return Promise.reject(new Error('Type must be accepted or requested.'));
 
       // Call the notification on the right conversation
       let conversation = this;
@@ -2115,17 +2122,22 @@
             source,
             'private'
           );
-          window.log.info(`Notify called on a different conversation. expected: ${this.id}. actual: ${conversation.id}`);
+          window.log.info(`Notify called on a different conversation.
+                           Expected: ${this.id}. Actual: ${conversation.id}`);
         } catch (e) {
-          return Promise.reject('Failed to fetch conversation');
+          return Promise.reject(new Error('Failed to fetch conversation'));
         }
       }
 
       const isTypeAccepted = type === 'accepted';
-      const title = isTypeAccepted ? 'friendRequestAcceptedNotificationTitle' : 'friendRequestNotificationTitle';
+      const title = isTypeAccepted
-      const message = isTypeAccepted ? 'friendRequestAcceptedNotificationMessage' : 'friendRequestNotificationMessage';
+        ? 'friendRequestAcceptedNotificationTitle'
-
+        : 'friendRequestNotificationTitle';
-      conversation.getNotificationIcon().then(iconUrl => {
+      const message = isTypeAccepted
+        ? 'friendRequestAcceptedNotificationMessage'
+        : 'friendRequestNotificationMessage';
+
+      const iconUrl = await conversation.getNotificationIcon();
       window.log.info('Add notification for friend request updated', {
         conversationId: conversation.idForLogging(),
       });
@@ -2137,7 +2149,7 @@
         messageSentAt: Date.now(),
         title: i18n(title),
       });
-      }); 
+      return Promise.resolve();
     },
   });
 

js/modules/data.js

@@ -122,6 +122,9 @@ module.exports = {
 
   getMessageCount,
   saveMessage,
+  cleanSeenMessages,
+  saveSeenMessageHash,
+  saveSeenMessageHashes,
   saveLegacyMessage,
   saveMessages,
   removeMessage,
@@ -140,6 +143,7 @@ module.exports = {
   getOutgoingWithoutExpiresAt,
   getNextExpiringMessage,
   getMessagesByConversation,
+  getSeenMessagesByHashList,
 
   getUnprocessedCount,
   getAllUnprocessed,
@@ -728,6 +732,18 @@ async function getMessageCount() {
   return channels.getMessageCount();
 }
 
+async function cleanSeenMessages() {
+  await channels.cleanSeenMessages();
+}
+
+async function saveSeenMessageHashes(data) {
+  await channels.saveSeenMessageHashes(_cleanData(data));
+}
+
+async function saveSeenMessageHash(data) {
+  await channels.saveSeenMessageHash(_cleanData(data));
+}
+
 async function saveMessage(data, { forceSave, Message } = {}) {
   const updated = keysFromArrayBuffer(MESSAGE_PRE_KEYS, data);
   const id = await channels.saveMessage(_cleanData(updated), { forceSave });
@@ -861,6 +877,13 @@ async function getMessagesByConversation(
   return new MessageCollection(encoded);
 }
 
+async function getSeenMessagesByHashList(
+  hashes
+) {
+  const seenMessages = await channels.getSeenMessagesByHashList(hashes);
+  return seenMessages;
+}
+
 async function removeAllMessagesInConversation(
   conversationId,
   { MessageCollection }

js/modules/loki_message_api.js

@@ -1,4 +1,4 @@
-/* global log, dcodeIO */
+/* global log, dcodeIO, window */
 
 const fetch = require('node-fetch');
 const is = require('@sindresorhus/is');

libtextsecure/http-resources.js

@@ -40,6 +40,17 @@
     };
   };
 
+  const filterIncomingMessages = async function filterIncomingMessages(messages) {
+    const incomingHashes = messages.map(m => m.hash);
+    const dupHashes = await window.Signal.Data.getSeenMessagesByHashList(incomingHashes);
+    const newMessages = messages.filter(m => !dupHashes.includes(m.hash));
+    const newHashes = newMessages.map(m => ({
+      expiresAt: m.expiration,
+      hash: m.hash,
+    }));
+    await window.Signal.Data.saveSeenMessageHashes(newHashes);
+    return newMessages;
+  };
 
   window.HttpResource = function HttpResource(_server, opts = {}) {
     server = _server;
@@ -68,7 +79,8 @@
         setTimeout(() => { pollServer(callBack); }, 5000);
         return;
       }
-      result.messages.forEach(async message => {
+      const newMessages = await filterIncomingMessages(result.messages);
+      newMessages.forEach(async message => {
         const { data } = message;
         const dataPlaintext = stringToArrayBufferBase64(data);
         const messageBuf = textsecure.protobuf.WebSocketMessage.decode(dataPlaintext);

libtextsecure/message_receiver.js

@@ -1,6 +1,7 @@
 /* global window: false */
 /* global textsecure: false */
 /* global StringView: false */
+/* global libloki: false */
 /* global libsignal: false */
 /* global WebSocket: false */
 /* global Event: false */
@@ -10,8 +11,10 @@
 /* global ContactBuffer: false */
 /* global GroupBuffer: false */
 /* global Worker: false */
+/* global WebSocketResource: false */
 
 /* eslint-disable more/no-then */
+/* eslint-disable no-unreachable */
 
 const WORKER_TIMEOUT = 60 * 1000; // one minute
 
@@ -251,7 +254,7 @@ MessageReceiver.prototype.extend({
       this.calledClose
     );
     // TODO: handle properly
-    return;
+    return Promise.resolve();
     this.shutdown();
 
     if (this.calledClose) {
@@ -708,7 +711,7 @@ MessageReceiver.prototype.extend({
         promise = sessionCipher.decryptWhisperMessage(ciphertext)
           .then(this.unpad);
         break;
-      case textsecure.protobuf.Envelope.Type.FRIEND_REQUEST:
+      case textsecure.protobuf.Envelope.Type.FRIEND_REQUEST: {
         window.log.info('friend-request message from ', envelope.source);
         const fallBackSessionCipher = new libloki.FallBackSessionCipher(
           address
@@ -716,6 +719,7 @@ MessageReceiver.prototype.extend({
         promise = fallBackSessionCipher.decrypt(ciphertext.toArrayBuffer())
           .then(this.unpad);
         break;
+      }
       case textsecure.protobuf.Envelope.Type.PREKEY_BUNDLE:
         window.log.info('prekey message from', this.getEnvelopeId(envelope));
         promise = this.decryptPreKeyWhisperMessage(
@@ -971,7 +975,7 @@ MessageReceiver.prototype.extend({
     if (!message || !message.direction || !message.friendStatus) return;
 
     // Update the conversation
-    const conversation = ConversationController.get(pubKey);
+    const conversation = window.ConversationController.get(pubKey);
     if (conversation) {
       // Update the conversation friend request indicator
       conversation.updatePendingFriendRequests();
@@ -990,9 +994,9 @@ MessageReceiver.prototype.extend({
         );
       }
 
-      await conversation.onFriendRequestAccepted();
+      await conversation.onFriendRequestAccepted({ updateUnread: false });
     }
-    console.log(`Friend request for ${pubKey} was ${message.friendStatus}`, message);
+    window.log.info(`Friend request for ${pubKey} was ${message.friendStatus}`, message);
   },
   async innerHandleContentMessage(envelope, plaintext) {
     const content = textsecure.protobuf.Content.decode(plaintext);
@@ -1000,15 +1004,17 @@ MessageReceiver.prototype.extend({
     if (envelope.type === textsecure.protobuf.Envelope.Type.FRIEND_REQUEST) {
       let conversation;
       try {
-        conversation = ConversationController.get(envelope.source);
+        conversation = window.ConversationController.get(envelope.source);
-      } catch (e) { }
+      } catch (e) {
+        throw new Error('Error getting conversation for message.')
+      }
 
       // only prompt friend request if there is no conversation yet
       if (!conversation) {
         this.promptUserToAcceptFriendRequest(
           envelope,
           content.dataMessage.body,
-          content.preKeyBundleMessage,
+          content.preKeyBundleMessage
         );
       } else {
         const keyExchangeComplete = conversation.isKeyExchangeCompleted();
@@ -1017,7 +1023,8 @@ MessageReceiver.prototype.extend({
         // We are certain that other user accepted the friend request IF:
         // - The message has a preKeyBundleMessage
         // - We have an outgoing friend request that is pending
-        // The second check is crucial because it makes sure we don't save the preKeys of the incoming friend request (which is saved only when we press accept)
+        // The second check is crucial because it makes sure we don't save the preKeys of
+        // the incoming friend request (which is saved only when we press accept)
         if (!keyExchangeComplete && content.preKeyBundleMessage) {
           // Check for any outgoing friend requests
           const pending = await conversation.getPendingFriendRequests('outgoing');
@@ -1040,7 +1047,7 @@ MessageReceiver.prototype.extend({
       }
 
       // Exit early since the friend request reply will be a regular empty message
-      return;
+      return Promise.resolve();
     }
 
     if (content.syncMessage) {
@@ -1054,9 +1061,7 @@ MessageReceiver.prototype.extend({
     } else if (content.receiptMessage) {
       return this.handleReceiptMessage(envelope, content.receiptMessage);
     }
-    if (!content.preKeyBundleMessage) {
     throw new Error('Unsupported content message');
-    }
   },
   handleCallMessage(envelope) {
     window.log.info('call message from', this.getEnvelopeId(envelope));
@@ -1284,13 +1289,13 @@ MessageReceiver.prototype.extend({
       signature,
     } = preKeyBundleMessage;
 
-    if (pubKey != StringView.arrayBufferToHex(identityKey)) {
+    if (pubKey !== StringView.arrayBufferToHex(identityKey)) {
       throw new Error(
         'Error in handlePreKeyBundleMessage: envelope pubkey does not match pubkey in prekey bundle'
       );
     }
 
-    return await libloki.savePreKeyBundleForNumber({
+    return libloki.savePreKeyBundleForNumber({
       pubKey,
       preKeyId,
       signedKeyId,
@@ -1306,8 +1311,8 @@ MessageReceiver.prototype.extend({
     return textsecure.storage.get('blocked-groups', []).indexOf(groupId) >= 0;
   },
   handleAttachment(attachment) {
-    console.log("Not handling attachments.");
+    window.log.info('Not handling attachments.');
-    return;
+    return Promise.reject();
     // eslint-disable-next-line no-param-reassign
     attachment.id = attachment.id.toString();
     // eslint-disable-next-line no-param-reassign

libtextsecure/outgoing_message.js

@@ -7,11 +7,10 @@
   StringView,
   dcodeIO,
   log,
-  btoa,
-  _
 */
 
 /* eslint-disable more/no-then */
+/* eslint-disable no-unreachable */
 
 function OutgoingMessage(
   server,
@@ -249,7 +248,8 @@ OutgoingMessage.prototype = {
     if (accessKey && !senderCertificate) {
       return Promise.reject(
         new Error(
-          'OutgoingMessage.doSendMessage: accessKey was provided, but senderCertificate was not'
+          'OutgoingMessage.doSendMessage: accessKey was provided, ' +
+          'but senderCertificate was not'
         )
       );
     }