use server created_at timestamp to order messages

Also update the way we check for duplicated message to allow a 10s
window with the same body rather than an exact match of timestamp.

This is needed as the timestamp of the message pulled is now the one of
the creation of the server, and not the same we have locally (sent at).

js/modules/data.d.ts

@@ -267,6 +267,10 @@ export function getMessageBySender(
   }: { source: any; sourceDevice: any; sent_at: any },
   { Message }: { Message: any }
 ): Promise<any>;
+export function getMessagesBySender(
+  { source, sourceDevice }: { source: any; sourceDevice: any },
+  { Message }: { Message: any }
+): Promise<Whisper.MessageCollection>;
 export function getMessageIdsFromServerIds(
   serverIds: any,
   conversationId: any

js/modules/data.js

@@ -151,6 +151,7 @@ module.exports = {
   removeAllMessagesInConversation,
 
   getMessageBySender,
+  getMessagesBySender,
   getMessageIdsFromServerIds,
   getMessageById,
   getAllMessages,
@@ -1015,6 +1016,23 @@ async function getMessageBySender(
   return new Message(messages[0]);
 }
 
+async function getMessagesBySender(
+  // eslint-disable-next-line camelcase
+  { source, sourceDevice, sent_at },
+  { Message }
+) {
+  const messages = await channels.getMessageBySender({
+    source,
+    sourceDevice,
+    sent_at,
+  });
+  if (!messages || !messages.length) {
+    return null;
+  }
+
+  return messages.map(m => new Message(m));
+}
+
 async function getUnreadByConversation(conversationId, { MessageCollection }) {
   const messages = await channels.getUnreadByConversation(conversationId);
   return new MessageCollection(messages);

js/modules/loki_app_dot_net_api.js

@@ -6,13 +6,13 @@ const { URL, URLSearchParams } = require('url');
 const FormData = require('form-data');
 const https = require('https');
 const path = require('path');
+const dataMessage = require('../../ts/receiver/dataMessage');
 
 // Can't be less than 1200 if we have unauth'd requests
 const PUBLICCHAT_MSG_POLL_EVERY = 1.5 * 1000; // 1.5s
 const PUBLICCHAT_CHAN_POLL_EVERY = 20 * 1000; // 20s
 const PUBLICCHAT_DELETION_POLL_EVERY = 5 * 1000; // 5s
 const PUBLICCHAT_MOD_POLL_EVERY = 30 * 1000; // 30s
-const PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES = 10 * 1000; // 10s
 
 // FIXME: replace with something on urlPubkeyMap...
 const FILESERVER_HOSTS = [
@@ -1799,7 +1799,7 @@ class LokiPublicChannelAPI {
     }
 
     return {
-      timestamp,
+      timestamp: new Date(`${adnMessage.created_at}`).getTime() || timestamp,
       attachments,
       preview,
       quote,
@@ -1913,7 +1913,10 @@ class LokiPublicChannelAPI {
         if (messengerData === false) {
           return false;
         }
-
+        // eslint-disable-next-line no-param-reassign
+        adnMessage.timestamp = messengerData.timestamp;
+        // eslint-disable-next-line no-param-reassign
+        adnMessage.body = messengerData.text;
         const {
           timestamp,
           quote,
@@ -1927,20 +1930,15 @@ class LokiPublicChannelAPI {
         }
 
         // Duplicate check
-        const isDuplicate = message => {
-          // The username in this case is the users pubKey
-          const sameUsername = message.username === pubKey;
-          const sameText = message.text === adnMessage.text;
-          // Don't filter out messages that are too far apart from each other
-          const timestampsSimilar =
-            Math.abs(message.timestamp - timestamp) <=
-            PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES;
-
-          return sameUsername && sameText && timestampsSimilar;
-        };
+        const isDuplicate = (message, testedMessage) =>
+          dataMessage.isDuplicate(
+            message,
+            testedMessage,
+            testedMessage.user.username
+          );
 
         // Filter out any messages that we got previously
-        if (this.lastMessagesCache.some(isDuplicate)) {
+        if (this.lastMessagesCache.some(m => isDuplicate(m, adnMessage))) {
           return false; // Duplicate message
         }
 
@@ -1949,9 +1947,11 @@ class LokiPublicChannelAPI {
         this.lastMessagesCache = [
           ...this.lastMessagesCache,
           {
-            username: pubKey,
-            text: adnMessage.text,
-            timestamp,
+            propsForMessage: {
+              authorPhoneNumber: pubKey,
+              text: adnMessage.text,
+              timestamp,
+            },
           },
         ].splice(-5);
 

ts/receiver/dataMessage.ts

@@ -356,30 +356,49 @@ interface MessageId {
   source: any;
   sourceDevice: any;
   timestamp: any;
+  message: any;
 }
+const PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES = 10 * 1000; // 10s
 
 async function isMessageDuplicate({
   source,
   sourceDevice,
   timestamp,
+  message,
 }: MessageId) {
   const { Errors } = window.Signal.Types;
 
   try {
-    const result = await window.Signal.Data.getMessageBySender(
-      { source, sourceDevice, sent_at: timestamp },
+    const result = await window.Signal.Data.getMessagesBySender(
+      { source, sourceDevice },
       {
         Message: window.Whisper.Message,
       }
     );
+    if (!result) {
+      return false;
+    }
 
-    return Boolean(result);
+    const isSimilar = result.some((m: any) => isDuplicate(m, message, source));
+    return isSimilar;
   } catch (error) {
     window.log.error('isMessageDuplicate error:', Errors.toLogFormat(error));
     return false;
   }
 }
 
+export const isDuplicate = (m: any, testedMessage: any, source: string) => {
+  // The username in this case is the users pubKey
+  const sameUsername = m.propsForMessage.authorPhoneNumber === source;
+  const sameText = m.propsForMessage.text === testedMessage.body;
+  // Don't filter out messages that are too far apart from each other
+  const timestampsSimilar =
+    Math.abs(m.propsForMessage.timestamp - testedMessage.timestamp) <=
+    PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES;
+
+  return sameUsername && sameText && timestampsSimilar;
+};
+
 async function handleProfileUpdate(
   profileKeyBuffer: Uint8Array,
   convoId: string,
@@ -588,9 +607,7 @@ export async function handleMessageEvent(event: MessageEvent): Promise<void> {
   // if the message is `sent` (from secondary device) we have to set the sender manually... (at least for now)
   source = source || msg.get('source');
 
-  const isDuplicate = await isMessageDuplicate(data);
-
-  if (isDuplicate) {
+  if (await isMessageDuplicate(data)) {
     // RSS expects duplicates, so squelch log
     if (!source.match(/^rss:/)) {
       window.log.warn('Received duplicate message', msg.idForLogging());