From cd686269cb30931cae7469a7f1145917fa9e53bb Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Wed, 2 Sep 2020 12:09:23 +1000 Subject: [PATCH] use server created_at timestamp to order messages Also update the way we check for duplicated message to allow a 10s window with the same body rather than an exact match of timestamp. This is needed as the timestamp of the message pulled is now the one of the creation of the server, and not the same we have locally (sent at). --- js/modules/data.d.ts | 4 ++++ js/modules/data.js | 18 +++++++++++++++ js/modules/loki_app_dot_net_api.js | 36 +++++++++++++++--------------- ts/receiver/dataMessage.ts | 29 +++++++++++++++++++----- 4 files changed, 63 insertions(+), 24 deletions(-) diff --git a/js/modules/data.d.ts b/js/modules/data.d.ts index 35a87b0f5..17ac87ebc 100644 --- a/js/modules/data.d.ts +++ b/js/modules/data.d.ts @@ -267,6 +267,10 @@ export function getMessageBySender( }: { source: any; sourceDevice: any; sent_at: any }, { Message }: { Message: any } ): Promise; +export function getMessagesBySender( + { source, sourceDevice }: { source: any; sourceDevice: any }, + { Message }: { Message: any } +): Promise; export function getMessageIdsFromServerIds( serverIds: any, conversationId: any diff --git a/js/modules/data.js b/js/modules/data.js index e532978e1..6de00db8c 100644 --- a/js/modules/data.js +++ b/js/modules/data.js @@ -151,6 +151,7 @@ module.exports = { removeAllMessagesInConversation, getMessageBySender, + getMessagesBySender, getMessageIdsFromServerIds, getMessageById, getAllMessages, @@ -1015,6 +1016,23 @@ async function getMessageBySender( return new Message(messages[0]); } +async function getMessagesBySender( + // eslint-disable-next-line camelcase + { source, sourceDevice, sent_at }, + { Message } +) { + const messages = await channels.getMessageBySender({ + source, + sourceDevice, + sent_at, + }); + if (!messages || !messages.length) { + return null; + } + + return messages.map(m => new Message(m)); +} + async function getUnreadByConversation(conversationId, { MessageCollection }) { const messages = await channels.getUnreadByConversation(conversationId); return new MessageCollection(messages); diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index ae606fb35..369de615a 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -6,13 +6,13 @@ const { URL, URLSearchParams } = require('url'); const FormData = require('form-data'); const https = require('https'); const path = require('path'); +const dataMessage = require('../../ts/receiver/dataMessage'); // Can't be less than 1200 if we have unauth'd requests const PUBLICCHAT_MSG_POLL_EVERY = 1.5 * 1000; // 1.5s const PUBLICCHAT_CHAN_POLL_EVERY = 20 * 1000; // 20s const PUBLICCHAT_DELETION_POLL_EVERY = 5 * 1000; // 5s const PUBLICCHAT_MOD_POLL_EVERY = 30 * 1000; // 30s -const PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES = 10 * 1000; // 10s // FIXME: replace with something on urlPubkeyMap... const FILESERVER_HOSTS = [ @@ -1799,7 +1799,7 @@ class LokiPublicChannelAPI { } return { - timestamp, + timestamp: new Date(`${adnMessage.created_at}`).getTime() || timestamp, attachments, preview, quote, @@ -1913,7 +1913,10 @@ class LokiPublicChannelAPI { if (messengerData === false) { return false; } - + // eslint-disable-next-line no-param-reassign + adnMessage.timestamp = messengerData.timestamp; + // eslint-disable-next-line no-param-reassign + adnMessage.body = messengerData.text; const { timestamp, quote, @@ -1927,20 +1930,15 @@ class LokiPublicChannelAPI { } // Duplicate check - const isDuplicate = message => { - // The username in this case is the users pubKey - const sameUsername = message.username === pubKey; - const sameText = message.text === adnMessage.text; - // Don't filter out messages that are too far apart from each other - const timestampsSimilar = - Math.abs(message.timestamp - timestamp) <= - PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES; - - return sameUsername && sameText && timestampsSimilar; - }; + const isDuplicate = (message, testedMessage) => + dataMessage.isDuplicate( + message, + testedMessage, + testedMessage.user.username + ); // Filter out any messages that we got previously - if (this.lastMessagesCache.some(isDuplicate)) { + if (this.lastMessagesCache.some(m => isDuplicate(m, adnMessage))) { return false; // Duplicate message } @@ -1949,9 +1947,11 @@ class LokiPublicChannelAPI { this.lastMessagesCache = [ ...this.lastMessagesCache, { - username: pubKey, - text: adnMessage.text, - timestamp, + propsForMessage: { + authorPhoneNumber: pubKey, + text: adnMessage.text, + timestamp, + }, }, ].splice(-5); diff --git a/ts/receiver/dataMessage.ts b/ts/receiver/dataMessage.ts index dddde9cfb..971d4ccc3 100644 --- a/ts/receiver/dataMessage.ts +++ b/ts/receiver/dataMessage.ts @@ -356,30 +356,49 @@ interface MessageId { source: any; sourceDevice: any; timestamp: any; + message: any; } +const PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES = 10 * 1000; // 10s async function isMessageDuplicate({ source, sourceDevice, timestamp, + message, }: MessageId) { const { Errors } = window.Signal.Types; try { - const result = await window.Signal.Data.getMessageBySender( - { source, sourceDevice, sent_at: timestamp }, + const result = await window.Signal.Data.getMessagesBySender( + { source, sourceDevice }, { Message: window.Whisper.Message, } ); + if (!result) { + return false; + } - return Boolean(result); + const isSimilar = result.some((m: any) => isDuplicate(m, message, source)); + return isSimilar; } catch (error) { window.log.error('isMessageDuplicate error:', Errors.toLogFormat(error)); return false; } } +export const isDuplicate = (m: any, testedMessage: any, source: string) => { + // The username in this case is the users pubKey + const sameUsername = m.propsForMessage.authorPhoneNumber === source; + const sameText = m.propsForMessage.text === testedMessage.body; + // Don't filter out messages that are too far apart from each other + const timestampsSimilar = + Math.abs(m.propsForMessage.timestamp - testedMessage.timestamp) <= + PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES; + + return sameUsername && sameText && timestampsSimilar; +}; + async function handleProfileUpdate( profileKeyBuffer: Uint8Array, convoId: string, @@ -588,9 +607,7 @@ export async function handleMessageEvent(event: MessageEvent): Promise { // if the message is `sent` (from secondary device) we have to set the sender manually... (at least for now) source = source || msg.get('source'); - const isDuplicate = await isMessageDuplicate(data); - - if (isDuplicate) { + if (await isMessageDuplicate(data)) { // RSS expects duplicates, so squelch log if (!source.match(/^rss:/)) { window.log.warn('Received duplicate message', msg.idForLogging());