From c7e54c4257fc88c7c806200aee9673c7137aa76e Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 8 Oct 2020 15:36:57 +1100 Subject: [PATCH 1/5] make attachment download use onion routing --- js/modules/loki_app_dot_net_api.js | 8 ++++++++ js/modules/loki_file_server_api.d.ts | 1 + js/modules/loki_file_server_api.js | 5 +++++ libtextsecure/sendmessage.js | 2 +- preload.js | 2 +- ts/receiver/attachments.ts | 9 ++------- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index ca78976a1..923d57daf 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -1240,6 +1240,14 @@ class LokiAppDotNetServerAPI { }); return this.uploadAvatar(formData); } + + async downloadAttachment(url) { + const endpoint = new URL(url).pathname; + + return this.serverRequest(`loki/v1${endpoint}`, { + method: 'GET', + }); + } } // functions to a specific ADN channel on an ADN server diff --git a/js/modules/loki_file_server_api.d.ts b/js/modules/loki_file_server_api.d.ts index f90fec127..9b92f3576 100644 --- a/js/modules/loki_file_server_api.d.ts +++ b/js/modules/loki_file_server_api.d.ts @@ -13,4 +13,5 @@ interface DeviceMappingAnnotation { interface LokiFileServerInstance { getUserDeviceMapping(pubKey: string): Promise; clearOurDeviceMappingAnnotations(): Promise; + downloadAttachment(url: string): Promise; } diff --git a/js/modules/loki_file_server_api.js b/js/modules/loki_file_server_api.js index 9287875c9..65d281434 100644 --- a/js/modules/loki_file_server_api.js +++ b/js/modules/loki_file_server_api.js @@ -198,6 +198,11 @@ class LokiFileServerInstance { result.slaveMap = newSlavePrimaryMap; return result; } + + // for files + async downloadAttachment(url) { + return this._server.downloadAttachment(url); + } } // extends LokiFileServerInstance with functions we'd only perform on our own home server diff --git a/libtextsecure/sendmessage.js b/libtextsecure/sendmessage.js index 0364b5b17..6e1e38975 100644 --- a/libtextsecure/sendmessage.js +++ b/libtextsecure/sendmessage.js @@ -185,8 +185,8 @@ Message.prototype = { }; function MessageSender() { + // Currently only used for getProxiedSize() and makeProxiedRequest(), which are only used for fetching previews this.server = WebAPI.connect(); - this.pendingMessages = {}; } MessageSender.prototype = { diff --git a/preload.js b/preload.js index 86eca4ccc..3a8645236 100644 --- a/preload.js +++ b/preload.js @@ -47,8 +47,8 @@ window.getCommitHash = () => config.commitHash; window.getNodeVersion = () => config.node_version; window.getHostName = () => config.hostname; window.getServerTrustRoot = () => config.serverTrustRoot; -window.isBehindProxy = () => Boolean(config.proxyUrl); window.JobQueue = JobQueue; +window.isBehindProxy = () => Boolean(config.proxyUrl); window.getStoragePubKey = key => window.isDev() ? key.substring(0, key.length - 2) : key; window.getDefaultFileServer = () => config.defaultFileServer; diff --git a/ts/receiver/attachments.ts b/ts/receiver/attachments.ts index bf71c795a..c326f5227 100644 --- a/ts/receiver/attachments.ts +++ b/ts/receiver/attachments.ts @@ -3,16 +3,11 @@ import _ from 'lodash'; import * as Data from '../../js/modules/data'; -// TODO: Might convert it to a class later -let webAPI: any; - export async function downloadAttachment(attachment: any) { - if (!webAPI) { - webAPI = window.WebAPI.connect(); - } + const res = await window.lokiFileServerAPI.downloadAttachment(attachment.url); // The attachment id is actually just the absolute url of the attachment - let data = await webAPI.getAttachment(attachment.url); + let data = new Uint8Array(res.response.data).buffer; if (!attachment.isRaw) { const { key, digest, size } = attachment; From 692a0e8cff43fd713601fa7f0a7d0e4566b21e23 Mon Sep 17 00:00:00 2001 From: Maxim Shishmarev Date: Mon, 21 Sep 2020 16:30:44 +1000 Subject: [PATCH 2/5] Optionally use v2 onions for snode requests --- preload.js | 1 + ts/session/snode_api/onions.ts | 150 +++++++++++++++++++++++++++------ ts/window.d.ts | 1 + 3 files changed, 128 insertions(+), 24 deletions(-) diff --git a/preload.js b/preload.js index 3a8645236..c70b76cc8 100644 --- a/preload.js +++ b/preload.js @@ -450,6 +450,7 @@ window.lokiFeatureFlags = { privateGroupChats: true, useSnodeProxy: !process.env.USE_STUBBED_NETWORK, useOnionRequests: true, + useOnionRequestsV2: false, useFileOnionRequests: true, enableSenderKeys: true, onionRequestHops: 3, diff --git a/ts/session/snode_api/onions.ts b/ts/session/snode_api/onions.ts index a7987e3cc..05e8c0f19 100644 --- a/ts/session/snode_api/onions.ts +++ b/ts/session/snode_api/onions.ts @@ -33,7 +33,7 @@ async function encryptForRelay( destination: any, ctx: any ) { - const { log, dcodeIO, StringView } = window; + const { log, StringView } = window; // ctx contains: ciphertext, symmetricKey, ephemeralKey const payload = ctx.ciphertext; @@ -51,23 +51,80 @@ async function encryptForRelay( return encryptForPubKey(relayX25519hex, reqObj); } -async function makeGuardPayload(guardCtx: any) { +// `ctx` holds info used by `node` to relay further +async function encryptForRelayV2( + relayX25519hex: string, + destination: any, + ctx: any +) { + const { log, StringView } = window; + + if (!destination.host && !destination.destination) { + log.warn('loki_rpc::encryptForRelay - no destination', destination); + } + + const reqObj = { + ...destination, + ephemeral_key: StringView.arrayBufferToHex(ctx.ephemeralKey), + }; + + const plaintext = encodeCiphertextPlusJson(ctx.ciphertext, reqObj); + + return window.libloki.crypto.encryptForPubkey(relayX25519hex, plaintext); +} + +function makeGuardPayload(guardCtx: any): Uint8Array { const ciphertextBase64 = StringUtils.decode(guardCtx.ciphertext, 'base64'); - const guardPayloadObj = { + const payloadObj = { ciphertext: ciphertextBase64, ephemeral_key: StringUtils.decode(guardCtx.ephemeralKey, 'hex'), }; - return guardPayloadObj; + + const payloadStr = JSON.stringify(payloadObj); + + const buffer = ByteBuffer.wrap(payloadStr, 'utf8'); + + return buffer.buffer; } -// we just need the targetNode.pubkey_ed25519 for the encryption -// targetPubKey is ed25519 if snode is the target -async function makeOnionRequest( +/// Encode ciphertext as (len || binary) and append payloadJson as utf8 +function encodeCiphertextPlusJson( + ciphertext: any, + payloadJson: any +): Uint8Array { + const payloadStr = JSON.stringify(payloadJson); + + const bufferJson = ByteBuffer.wrap(payloadStr, 'utf8'); + + const len = ciphertext.length as number; + const arrayLen = bufferJson.buffer.length + 4 + len; + const littleEndian = true; + const buffer = new ByteBuffer(arrayLen, littleEndian); + + buffer.writeInt32(len); + buffer.append(ciphertext); + buffer.append(bufferJson); + + return new Uint8Array(buffer.buffer); +} + +// New "semi-binary" encoding +function makeGuardPayloadV2(guardCtx: any): Uint8Array { + const guardPayloadObj = { + ephemeral_key: StringUtils.decode(guardCtx.ephemeralKey, 'hex'), + }; + + return encodeCiphertextPlusJson(guardCtx.ciphertext, guardPayloadObj); +} + +async function buildOnionCtxs( nodePath: Array, destCtx: any, targetED25519Hex: string, - finalRelayOptions?: any, + // whether to use the new "semi-binary" protocol + useV2: boolean, + fileServerOptions?: any, id = '' ) { const { log } = window; @@ -80,9 +137,9 @@ async function makeOnionRequest( let dest; const relayingToFinalDestination = i === firstPos; // if last position - if (relayingToFinalDestination && finalRelayOptions) { + if (relayingToFinalDestination && fileServerOptions) { dest = { - host: finalRelayOptions.host, + host: fileServerOptions.host, target: '/loki/v1/lsrpc', method: 'POST', }; @@ -107,8 +164,9 @@ async function makeOnionRequest( }; } try { + const encryptFn = useV2 ? encryptForRelayV2 : encryptForRelay; // eslint-disable-next-line no-await-in-loop - const ctx = await encryptForRelay( + const ctx = await encryptFn( nodePath[i].pubkey_x25519, dest, ctxes[ctxes.length - 1] @@ -123,12 +181,38 @@ async function makeOnionRequest( throw e; } } + + return ctxes; +} + +// we just need the targetNode.pubkey_ed25519 for the encryption +// targetPubKey is ed25519 if snode is the target +async function makeOnionRequest( + nodePath: Array, + destCtx: any, + targetED25519Hex: string, + // whether to use the new (v2) protocol + useV2: boolean, + finalRelayOptions?: any, + id = '' +) { + const ctxes = await buildOnionCtxs( + nodePath, + destCtx, + targetED25519Hex, + useV2, + finalRelayOptions, + id + ); + const guardCtx = ctxes[ctxes.length - 1]; // last ctx - const payloadObj = makeGuardPayload(guardCtx); + const payload = useV2 + ? makeGuardPayloadV2(guardCtx) + : makeGuardPayload(guardCtx); // all these requests should use AesGcm - return payloadObj; + return payload; } // Process a response as it arrives from `fetch`, handling @@ -276,13 +360,14 @@ const sendOnionRequest = async ( ) => { const { log, StringView } = window; - // loki-storage may need this to function correctly - // but ADN calls will not always have a body - /* - if (!finalDestOptions.body) { - finalDestOptions.body = ''; - } - */ + let useV2 = window.lokiFeatureFlags.useOnionRequestsV2; + + if (useV2 && finalRelayOptions) { + useV2 = false; + log.error( + 'TODO: v2 onion protocol for the file server is not yet supported' + ); + } let id = ''; if (lsrpcIdx !== undefined) { @@ -317,7 +402,21 @@ const sendOnionRequest = async ( let destCtx; try { - destCtx = await encryptForPubKey(destX25519hex, options); + if (useV2) { + const body = options.body || ''; + delete options.body; + + const textEncoder = new TextEncoder(); + const bodyEncoded = textEncoder.encode(body); + + const plaintext = encodeCiphertextPlusJson(bodyEncoded, options); + destCtx = await window.libloki.crypto.encryptForPubkey( + destX25519hex, + plaintext + ); + } else { + destCtx = await encryptForPubKey(destX25519hex, options); + } } catch (e) { log.error( `loki_rpc::sendOnionRequest ${id} - encryptForPubKey failure [`, @@ -333,22 +432,25 @@ const sendOnionRequest = async ( throw e; } - const payloadObj = await makeOnionRequest( + const payload = await makeOnionRequest( nodePath, destCtx, targetEd25519hex, + useV2, finalRelayOptions, id ); const guardFetchOptions = { method: 'POST', - body: JSON.stringify(payloadObj), + body: payload, // we are talking to a snode... agent: snodeHttpsAgent, }; - const guardUrl = `https://${nodePath[0].ip}:${nodePath[0].port}/onion_req`; + const target = useV2 ? '/onion_req/v2' : '/onion_req'; + + const guardUrl = `https://${nodePath[0].ip}:${nodePath[0].port}${target}`; const response = await fetch(guardUrl, guardFetchOptions); return processOnionResponse( diff --git a/ts/window.d.ts b/ts/window.d.ts index 51387b0d3..8201c274b 100644 --- a/ts/window.d.ts +++ b/ts/window.d.ts @@ -58,6 +58,7 @@ declare global { privateGroupChats: boolean; useSnodeProxy: boolean; useOnionRequests: boolean; + useOnionRequestsV2: boolean; useFileOnionRequests: boolean; enableSenderKeys: boolean; onionRequestHops: number; From c5b2b64d7fb869ee2fa9cfc0d12a9cec5ce2c511 Mon Sep 17 00:00:00 2001 From: Maxim Shishmarev Date: Wed, 21 Oct 2020 14:07:41 +1100 Subject: [PATCH 3/5] Add onions-v2 support for fileserver requests --- config/swarm-testing.json | 3 ++- js/modules/loki_app_dot_net_api.js | 1 + .../AvatarPlaceHolder/AvatarPlaceHolder.tsx | 16 ++++++++-------- ts/session/snode_api/onions.ts | 19 +++++++++---------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/config/swarm-testing.json b/config/swarm-testing.json index 61594f421..bb68a5d00 100644 --- a/config/swarm-testing.json +++ b/config/swarm-testing.json @@ -6,5 +6,6 @@ } ], "openDevTools": true, - "defaultPublicChatServer": "https://team-chat.lokinet.org/" + "defaultPublicChatServer": "https://team-chat.lokinet.org/", + "defaultFileServer": "https://file-dev.getsession.org" } diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 923d57daf..1e6329af1 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -32,6 +32,7 @@ const LOKIFOUNDATION_APNS_PUBKEY = const urlPubkeyMap = { 'https://file-dev.getsession.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY, 'https://file-dev.lokinet.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY, + 'https://file-dev.getsession.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY, 'https://file.getsession.org': LOKIFOUNDATION_FILESERVER_PUBKEY, 'https://file.lokinet.org': LOKIFOUNDATION_FILESERVER_PUBKEY, 'https://dev.apns.getsession.org': LOKIFOUNDATION_APNS_PUBKEY, diff --git a/ts/components/AvatarPlaceHolder/AvatarPlaceHolder.tsx b/ts/components/AvatarPlaceHolder/AvatarPlaceHolder.tsx index 431065f88..8e7d1e8fd 100644 --- a/ts/components/AvatarPlaceHolder/AvatarPlaceHolder.tsx +++ b/ts/components/AvatarPlaceHolder/AvatarPlaceHolder.tsx @@ -61,9 +61,9 @@ export class AvatarPlaceHolder extends React.PureComponent { cy={r} r={rWithoutBorder} fill="#d2d2d3" - shape-rendering="geometricPrecision" + shapeRendering="geometricPrecision" stroke={borderColor} - stroke-width="1" + strokeWidth="1" /> @@ -88,19 +88,19 @@ export class AvatarPlaceHolder extends React.PureComponent { cy={r} r={rWithoutBorder} fill={bgColor} - shape-rendering="geometricPrecision" + shapeRendering="geometricPrecision" stroke={borderColor} - stroke-width="1" + strokeWidth="1" /> {initial} diff --git a/ts/session/snode_api/onions.ts b/ts/session/snode_api/onions.ts index 05e8c0f19..2fcc2c0ad 100644 --- a/ts/session/snode_api/onions.ts +++ b/ts/session/snode_api/onions.ts @@ -138,9 +138,12 @@ async function buildOnionCtxs( const relayingToFinalDestination = i === firstPos; // if last position if (relayingToFinalDestination && fileServerOptions) { + + const target = useV2 ? '/loki/v2/lsrpc' : '/loki/v1/lsrpc'; + dest = { host: fileServerOptions.host, - target: '/loki/v1/lsrpc', + target, method: 'POST', }; } else { @@ -360,14 +363,6 @@ const sendOnionRequest = async ( ) => { const { log, StringView } = window; - let useV2 = window.lokiFeatureFlags.useOnionRequestsV2; - - if (useV2 && finalRelayOptions) { - useV2 = false; - log.error( - 'TODO: v2 onion protocol for the file server is not yet supported' - ); - } let id = ''; if (lsrpcIdx !== undefined) { @@ -400,9 +395,11 @@ const sendOnionRequest = async ( options.headers = ''; } + const useV2 = window.lokiFeatureFlags.useOnionRequestsV2; + let destCtx; try { - if (useV2) { + if (useV2 && !finalRelayOptions) { const body = options.body || ''; delete options.body; @@ -441,6 +438,8 @@ const sendOnionRequest = async ( id ); + log.debug('Onion payload size: ', payload.length); + const guardFetchOptions = { method: 'POST', body: payload, From 61897df7dd6f2ee15c60ab0f9f155ee050bfa033 Mon Sep 17 00:00:00 2001 From: Maxim Shishmarev Date: Wed, 21 Oct 2020 15:51:52 +1100 Subject: [PATCH 4/5] Lint --- js/modules/loki_app_dot_net_api.js | 1 - ts/session/snode_api/onions.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 1e6329af1..923d57daf 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -32,7 +32,6 @@ const LOKIFOUNDATION_APNS_PUBKEY = const urlPubkeyMap = { 'https://file-dev.getsession.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY, 'https://file-dev.lokinet.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY, - 'https://file-dev.getsession.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY, 'https://file.getsession.org': LOKIFOUNDATION_FILESERVER_PUBKEY, 'https://file.lokinet.org': LOKIFOUNDATION_FILESERVER_PUBKEY, 'https://dev.apns.getsession.org': LOKIFOUNDATION_APNS_PUBKEY, diff --git a/ts/session/snode_api/onions.ts b/ts/session/snode_api/onions.ts index 2fcc2c0ad..4cc02bb90 100644 --- a/ts/session/snode_api/onions.ts +++ b/ts/session/snode_api/onions.ts @@ -138,7 +138,6 @@ async function buildOnionCtxs( const relayingToFinalDestination = i === firstPos; // if last position if (relayingToFinalDestination && fileServerOptions) { - const target = useV2 ? '/loki/v2/lsrpc' : '/loki/v1/lsrpc'; dest = { @@ -363,7 +362,6 @@ const sendOnionRequest = async ( ) => { const { log, StringView } = window; - let id = ''; if (lsrpcIdx !== undefined) { id += `${lsrpcIdx}=>`; From 5ecf43c124f38583078ab1b7e796640e557c826c Mon Sep 17 00:00:00 2001 From: Maxim Shishmarev Date: Thu, 22 Oct 2020 17:14:10 +1100 Subject: [PATCH 5/5] Fix open group file uploads using incorrect server --- js/modules/loki_app_dot_net_api.d.ts | 6 ++++++ ts/receiver/attachments.ts | 26 +++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/js/modules/loki_app_dot_net_api.d.ts b/js/modules/loki_app_dot_net_api.d.ts index 68d705850..b898ae4ea 100644 --- a/js/modules/loki_app_dot_net_api.d.ts +++ b/js/modules/loki_app_dot_net_api.d.ts @@ -9,6 +9,11 @@ interface UploadResponse { id?: number; } +interface DownloadResponse { + statucCode: number; + reponse: any; +} + export interface LokiAppDotNetServerInterface { findOrCreateChannel( api: LokiPublicChatFactoryAPI, @@ -19,6 +24,7 @@ export interface LokiAppDotNetServerInterface { uploadAvatar(data: FormData): Promise; putAttachment(data: ArrayBuffer): Promise; putAvatar(data: ArrayBuffer): Promise; + downloadAttachment(url: String): Promise; // todo: add return type } export interface LokiPublicChannelAPI { diff --git a/ts/receiver/attachments.ts b/ts/receiver/attachments.ts index c326f5227..26bf845ee 100644 --- a/ts/receiver/attachments.ts +++ b/ts/receiver/attachments.ts @@ -4,7 +4,31 @@ import _ from 'lodash'; import * as Data from '../../js/modules/data'; export async function downloadAttachment(attachment: any) { - const res = await window.lokiFileServerAPI.downloadAttachment(attachment.url); + const serverUrl = new URL(attachment.url).origin; + + // The fileserver adds the `-static` part for some reason + const defaultFileserver = _.includes( + ['https://file-static.lokinet.org', 'https://file.getsession.org'], + serverUrl + ); + + let res: any; + + // TODO: we need attachments to remember which API should be used to retrieve them + if (!defaultFileserver) { + const serverAPI = await window.lokiPublicChatAPI.findOrCreateServer( + serverUrl + ); + + if (serverAPI) { + res = await serverAPI.downloadAttachment(attachment.url); + } + } + + // Fallback to using the default fileserver + if (defaultFileserver || !res) { + res = await window.lokiFileServerAPI.downloadAttachment(attachment.url); + } // The attachment id is actually just the absolute url of the attachment let data = new Uint8Array(res.response.data).buffer;