From 4aeec224b4445863a09419235f0d0b32e0738f11 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Tue, 27 Apr 2021 11:14:07 +1000 Subject: [PATCH] add download of attachments for opengroupv2 --- ts/receiver/attachments.ts | 85 +++++++------------------ ts/receiver/dataMessage.ts | 21 +++++- ts/session/snode_api/onions.ts | 9 ++- ts/session/utils/AttachmentsDownload.ts | 22 ++++++- ts/session/utils/AttachmentsV2.ts | 7 +- 5 files changed, 73 insertions(+), 71 deletions(-) diff --git a/ts/receiver/attachments.ts b/ts/receiver/attachments.ts index 43fbe9f97..489a2414f 100644 --- a/ts/receiver/attachments.ts +++ b/ts/receiver/attachments.ts @@ -5,6 +5,8 @@ import { saveMessage } from '../../ts/data/data'; import { fromBase64ToArrayBuffer } from '../session/utils/String'; import { AttachmentDownloads, AttachmentUtils } from '../session/utils'; import { ConversationModel } from '../models/conversation'; +import { downloadFileOpenGroupV2 } from '../opengroup/opengroupV2/OpenGroupAPIV2'; +import { OpenGroupRequestCommonType } from '../opengroup/opengroupV2/ApiUtil'; export async function downloadAttachment(attachment: any) { const serverUrl = new URL(attachment.url).origin; @@ -83,75 +85,32 @@ export async function downloadAttachment(attachment: any) { }; } -export async function downloadAttachmentOpenGrouPV2(attachment: any) { - const serverUrl = new URL(attachment.url).origin; - - // The fileserver adds the `-static` part for some reason - const defaultFileserver = _.includes( - ['https://file-static.lokinet.org', 'https://file.getsession.org'], - serverUrl - ); - - let res: ArrayBuffer | null = null; - - // TODO: we need attachments to remember which API should be used to retrieve them - if (!defaultFileserver) { - const serverAPI = await window.lokiPublicChatAPI.findOrCreateServer(serverUrl); - - if (serverAPI) { - res = await serverAPI.downloadAttachment(attachment.url); - } - } - - // Fallback to using the default fileserver - if (defaultFileserver || !res || res.byteLength === 0) { - res = await window.lokiFileServerAPI.downloadAttachment(attachment.url); - } +export async function downloadAttachmentOpenGroupV2( + attachment: any, + roomInfos: OpenGroupRequestCommonType +) { + const dataUint = await downloadFileOpenGroupV2(attachment.id, roomInfos); - if (res.byteLength === 0) { + if (!dataUint?.length) { window.log.error('Failed to download attachment. Length is 0'); throw new Error(`Failed to download attachment. Length is 0 for ${attachment.url}`); } - // FIXME "178" test to remove once this is fixed server side. - if (!window.lokiFeatureFlags.useFileOnionRequestsV2) { - if (res.byteLength === 178) { - window.log.error( - 'Data of 178 length corresponds of a 404 returned as 200 by file.getsession.org.' + let data = dataUint; + if (attachment.size !== dataUint.length) { + // we might have padding, check that all the remaining bytes are padding bytes + // otherwise we have an error. + if (AttachmentUtils.isLeftOfBufferPaddingOnly(dataUint.buffer, attachment.size)) { + // we can safely remove the padding + data = data.slice(0, attachment.size); + } else { + throw new Error( + `downloadAttachment: Size ${attachment.size} did not match downloaded attachment size ${data.byteLength}` ); - throw new Error(`downloadAttachment: invalid response for ${attachment.url}`); } } else { - // if useFileOnionRequestsV2 is true, we expect an ArrayBuffer not empty - } - - // The attachment id is actually just the absolute url of the attachment - let data = res; - if (!attachment.isRaw) { - const { key, digest, size } = attachment; - - if (!key || !digest) { - throw new Error('Attachment is not raw but we do not have a key to decode it'); - } - - data = await window.textsecure.crypto.decryptAttachment( - data, - fromBase64ToArrayBuffer(key), - fromBase64ToArrayBuffer(digest) - ); - - if (!size || size !== data.byteLength) { - // we might have padding, check that all the remaining bytes are padding bytes - // otherwise we have an error. - if (AttachmentUtils.isLeftOfBufferPaddingOnly(data, size)) { - // we can safely remove the padding - data = data.slice(0, size); - } else { - throw new Error( - `downloadAttachment: Size ${size} did not match downloaded attachment size ${data.byteLength}` - ); - } - } + // nothing to do, the attachment has already the correct size. There is just no padding included, which is bas + window.log.warn('Received opengroupv2 unpadded attachment'); } return { @@ -166,13 +125,15 @@ async function processNormalAttachments( convo: ConversationModel ): Promise { const isOpenGroupV2 = convo.isOpenGroupV2(); + const openGroupV2Details = (isOpenGroupV2 && convo.toOpenGroupV2()) || undefined; const attachments = await Promise.all( - normalAttachments.map((attachment: any, index: any) => { + normalAttachments.map(async (attachment: any, index: any) => { return AttachmentDownloads.addJob(attachment, { messageId: message.id, type: 'attachment', index, isOpenGroupV2, + openGroupV2Details, }); }) ); diff --git a/ts/receiver/dataMessage.ts b/ts/receiver/dataMessage.ts index 0f2b7b8ff..8c9d0346b 100644 --- a/ts/receiver/dataMessage.ts +++ b/ts/receiver/dataMessage.ts @@ -359,7 +359,7 @@ export async function isMessageDuplicate({ } const filteredResult = [result].filter((m: any) => m.attributes.body === message.body); if (serverId) { - return filteredResult.some(m => isDuplicate(m, { ...message, serverId }, source)); + return filteredResult.some(m => isDuplicateServerId(m, { ...message, serverId }, source)); } return filteredResult.some(m => isDuplicate(m, message, source)); } catch (error) { @@ -368,6 +368,25 @@ export async function isMessageDuplicate({ } } +/** + * This function is to be used to check for duplicates for open group v2 messages. + * It just check that the sender and the serverId of a received and an already saved messages are the same + */ +export const isDuplicateServerId = ( + m: MessageModel, + testedMessage: MessageDuplicateSearchType, + source: string +) => { + // The username in this case is the users pubKey + const sameUsername = m.attributes.source === source; + // testedMessage.id is needed as long as we support opengroupv1 + const sameServerId = + m.attributes.serverId !== undefined && + (testedMessage.serverId || testedMessage.id) === m.attributes.serverId; + + return sameUsername && sameServerId; +}; + export const isDuplicate = ( m: MessageModel, testedMessage: MessageDuplicateSearchType, diff --git a/ts/session/snode_api/onions.ts b/ts/session/snode_api/onions.ts index 7d1f81257..b41d270d4 100644 --- a/ts/session/snode_api/onions.ts +++ b/ts/session/snode_api/onions.ts @@ -5,6 +5,7 @@ import { Snode } from './snodePool'; import ByteBuffer from 'bytebuffer'; import { StringUtils } from '../utils'; import { OnionPaths } from '../onions'; +import Long from 'long'; export enum RequestError { BAD_PATH = 'BAD_PATH', @@ -330,7 +331,13 @@ const processOnionResponse = async ( } try { - const jsonRes: SnodeResponse = JSON.parse(plaintext); + const jsonRes: SnodeResponse = JSON.parse(plaintext, (key, value) => { + if (typeof value === 'number' && value > Number.MAX_SAFE_INTEGER) { + window.log.warn('Received an out of bounds js number'); + } + return value; + }); + return jsonRes; } catch (e) { log.error( diff --git a/ts/session/utils/AttachmentsDownload.ts b/ts/session/utils/AttachmentsDownload.ts index 8139f0bf7..19b244010 100644 --- a/ts/session/utils/AttachmentsDownload.ts +++ b/ts/session/utils/AttachmentsDownload.ts @@ -11,7 +11,7 @@ import { setAttachmentDownloadJobPending, } from '../../../ts/data/data'; import { MessageModel } from '../../models/message'; -import { downloadAttachment } from '../../receiver/attachments'; +import { downloadAttachment, downloadAttachmentOpenGroupV2 } from '../../receiver/attachments'; import { MessageController } from '../messages'; const MAX_ATTACHMENT_JOB_PARALLELISM = 3; @@ -128,8 +128,10 @@ async function _maybeStartJob() { } } +// tslint:disable-next-line: cyclomatic-complexity async function _runJob(job: any) { - const { id, messageId, attachment, type, index, attempts, isOpenGroupV2 } = job || {}; + const { id, messageId, attachment, type, index, attempts, isOpenGroupV2, openGroupV2Details } = + job || {}; let message; try { @@ -143,6 +145,16 @@ async function _runJob(job: any) { await _finishJob(null, id); return; } + + if (isOpenGroupV2 && (!openGroupV2Details?.serverUrl || !openGroupV2Details.roomId)) { + window.log.warn( + 'isOpenGroupV2 download attachment, but no valid openGroupV2Details given:', + openGroupV2Details + ); + await _finishJob(null, id); + return; + } + message = MessageController.getInstance().register(found.id, found); const pending = true; @@ -151,7 +163,11 @@ async function _runJob(job: any) { let downloaded; try { - downloaded = await downloadAttachment(attachment); + if (isOpenGroupV2) { + downloaded = await downloadAttachmentOpenGroupV2(attachment, openGroupV2Details); + } else { + downloaded = await downloadAttachment(attachment); + } } catch (error) { // Attachments on the server expire after 60 days, then start returning 404 if (error && error.code === 404) { diff --git a/ts/session/utils/AttachmentsV2.ts b/ts/session/utils/AttachmentsV2.ts index 4486705d9..e877ae0c9 100644 --- a/ts/session/utils/AttachmentsV2.ts +++ b/ts/session/utils/AttachmentsV2.ts @@ -57,10 +57,9 @@ export async function uploadV2(params: UploadParamsV2): Promise