From a195c980619a27ecc7c644c5f85aff410c4bf450 Mon Sep 17 00:00:00 2001 From: Maxim Shishmarev Date: Thu, 19 Dec 2019 11:35:52 +1100 Subject: [PATCH 01/17] Add a feature flag for snode proxy --- config/swarm-testing.json | 12 +++++ config/swarm-testing2.json | 12 +++++ js/modules/loki_app_dot_net_api.js | 28 +++++++++- js/modules/loki_message_api.js | 19 ++++--- js/modules/loki_rpc.js | 84 ++++++++++++++++++++++++++---- js/modules/loki_snode_api.js | 16 ++++-- package.json | 2 + preload.js | 1 + 8 files changed, 149 insertions(+), 25 deletions(-) create mode 100644 config/swarm-testing.json create mode 100644 config/swarm-testing2.json diff --git a/config/swarm-testing.json b/config/swarm-testing.json new file mode 100644 index 000000000..9bdd086d9 --- /dev/null +++ b/config/swarm-testing.json @@ -0,0 +1,12 @@ +{ + "storageProfile": "swarm-testing", + "seedNodeList": [ + { + "ip": "localhost", + "port": "22129" + } + ], + "openDevTools": true, + "defaultPublicChatServer": "https://team-chat.lokinet.org/" + } + \ No newline at end of file diff --git a/config/swarm-testing2.json b/config/swarm-testing2.json new file mode 100644 index 000000000..e3799a713 --- /dev/null +++ b/config/swarm-testing2.json @@ -0,0 +1,12 @@ +{ + "storageProfile": "swarm-testing2", + "seedNodeList": [ + { + "ip": "localhost", + "port": "22129" + } + ], + "openDevTools": true, + "defaultPublicChatServer": "https://team-chat.lokinet.org/" + } + \ No newline at end of file diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 97aa8b13d..f50a5e441 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -291,6 +291,26 @@ class LokiAppDotNetServerAPI { } } + async _sendToProxy(fetchOptions, endpoint, method) { + + const rand_snode = await lokiSnodeAPI.getRandomSnodeAddress(); + const url = `https://${rand_snode.ip}:${rand_snode.port}/file_proxy`; + + const body = fetchOptions.body; + + const firstHopOptions = { + method: 'POST', + body, + headers: { + "X-Loki-File-Server-Target": `/${endpoint}`, + "X-Loki-File-Server-Verb": method, + "X-Loki-File-Server-Headers": JSON.stringify(fetchOptions.headers), + } + } + + return await nodeFetch(url, firstHopOptions); + } + // make a request to the server async serverRequest(endpoint, options = {}) { const { @@ -324,7 +344,13 @@ class LokiAppDotNetServerAPI { fetchOptions.body = rawBody; } fetchOptions.headers = new Headers(headers); - result = await nodeFetch(url, fetchOptions || undefined); + + if (window.lokiFeatureFlags.useSnodeProxy && this.baseServerUrl === "https://file.lokinet.org") { + log.info("Sending a proxy request to https://file.lokinet.org"); + result = await this._sendToProxy({...fetchOptions, headers}, endpoint, method); + } else { + result = await nodeFetch(url, fetchOptions || undefined); + } } catch (e) { log.info(`e ${e}`); return { diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index aa9eb430c..9c8a94680 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -3,7 +3,7 @@ /* global log, dcodeIO, window, callWorker, lokiP2pAPI, lokiSnodeAPI, textsecure */ const _ = require('lodash'); -const { rpc } = require('./loki_rpc'); +const { loki_rpc } = require('./loki_rpc'); const DEFAULT_CONNECTIONS = 3; const MAX_ACCEPTABLE_FAILURES = 1; @@ -47,7 +47,7 @@ const trySendP2p = async (pubKey, data64, isPing, messageEventData) => { return false; } try { - await rpc(p2pDetails.address, p2pDetails.port, 'store', { + await loki_rpc(p2pDetails.address, p2pDetails.port, 'store', { data: data64, }); lokiP2pAPI.setContactOnline(pubKey); @@ -213,6 +213,7 @@ class LokiMessageAPI { const successfulSend = await this.sendToNode( snode.ip, snode.port, + snode, params ); if (successfulSend) { @@ -237,12 +238,12 @@ class LokiMessageAPI { return false; } - async sendToNode(address, port, params) { + async sendToNode(address, port, targetNode, params) { let successiveFailures = 0; while (successiveFailures < MAX_ACCEPTABLE_FAILURES) { await sleepFor(successiveFailures * 500); try { - const result = await rpc(`https://${address}`, port, 'store', params); + const result = await loki_rpc(`https://${address}`, port, 'store', params, {}, '/storage_rpc/v1', targetNode); // Make sure we aren't doing too much PoW const currentDifficulty = window.storage.get('PoWDifficulty', null); @@ -365,12 +366,14 @@ class LokiMessageAPI { }, }; - const result = await rpc( + const result = await loki_rpc( `https://${nodeUrl}`, nodeData.port, 'retrieve', params, - options + options, + '/storage_rpc/v1', + nodeData ); return result.messages || []; } @@ -386,10 +389,10 @@ class LokiMessageAPI { const lastHash = await window.Signal.Data.getLastHashBySnode( nodes[i].address ); + this.ourSwarmNodes[nodes[i].address] = { + ...nodes[i], lastHash, - ip: nodes[i].ip, - port: nodes[i].port, }; } diff --git a/js/modules/loki_rpc.js b/js/modules/loki_rpc.js index 13e37e511..4113181c4 100644 --- a/js/modules/loki_rpc.js +++ b/js/modules/loki_rpc.js @@ -1,4 +1,4 @@ -/* global log, libloki, textsecure, getStoragePubKey */ +/* global log, libloki, textsecure, getStoragePubKey, lokiSnodeAPI */ const nodeFetch = require('node-fetch'); const { parse } = require('url'); @@ -21,8 +21,63 @@ const decryptResponse = async (response, address) => { return {}; }; +// TODO: Don't allow arbitrary URLs, only snodes and loki servers +const sendToProxy = async (options = {}, targetNode) => { + + const rand_snode = await lokiSnodeAPI.getRandomSnodeAddress(); + + const url = `https://${rand_snode.ip}:${rand_snode.port}/proxy`; + + log.info(`Proxy snode reqeust to ${targetNode.pubkey_ed25519} via ${rand_snode.pubkey_ed25519}`); + + const sn_pub_key_hex = StringView.hexToArrayBuffer(targetNode.pubkey_x25519); + + const my_keys = window.libloki.crypto.snodeCipher._ephemeralKeyPair; + + const symmetricKey = libsignal.Curve.calculateAgreement( + sn_pub_key_hex, + my_keys.privKey + ); + + const textEncoder = new TextEncoder(); + const body = JSON.stringify(options); + + const plainText = textEncoder.encode(body); + const ivAndCiphertext = await window.libloki.crypto.DHEncrypt(symmetricKey, plainText); + + const firstHopOptions = { + method: 'POST', + body: ivAndCiphertext, + headers: { + "X-Sender-Public-Key": StringView.arrayBufferToHex(my_keys.pubKey), + "X-Target-Snode-Key": targetNode.pubkey_ed25519, + } + } + + const response = await nodeFetch(url, firstHopOptions); + const ciphertext = await response.text(); + + const ciphertextBuffer = dcodeIO.ByteBuffer.wrap( + ciphertext, 'base64' + ).toArrayBuffer(); + + const plaintextBuffer = await window.libloki.crypto.DHDecrypt(symmetricKey, ciphertextBuffer); + + const textDecoder = new TextDecoder(); + const plaintext = textDecoder.decode(plaintextBuffer); + + const json_res = JSON.parse(plaintext); + + json_res.json = () => { + return JSON.parse(json_res.body); + } + + return json_res; + +} + // A small wrapper around node-fetch which deserializes response -const fetch = async (url, options = {}) => { +const loki_fetch = async (url, options = {}, targetNode = null) => { const timeout = options.timeout || 10000; const method = options.method || 'GET'; @@ -47,12 +102,18 @@ const fetch = async (url, options = {}) => { } } + const fetchOptions = { + ...options, timeout, method, + }; + try { - const response = await nodeFetch(url, { - ...options, - timeout, - method, - }); + + if (window.lokiFeatureFlags.useSnodeProxy && targetNode) { + const result = await sendToProxy(fetchOptions, targetNode); + return result.json(); + } + + const response = await nodeFetch(url, fetchOptions); let result; // Wrong swarm @@ -107,13 +168,14 @@ const fetch = async (url, options = {}) => { }; // Wrapper for a JSON RPC request -const rpc = ( +const loki_rpc = ( address, port, method, params, options = {}, - endpoint = endpointBase + endpoint = endpointBase, + targetNode, ) => { const headers = options.headers || {}; const portString = port ? `:${port}` : ''; @@ -144,9 +206,9 @@ const rpc = ( }, }; - return fetch(url, fetchOptions); + return loki_fetch(url, fetchOptions, targetNode); }; module.exports = { - rpc, + loki_rpc, }; diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 51da421a1..8c0370a33 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -4,7 +4,7 @@ const is = require('@sindresorhus/is'); const dns = require('dns'); const process = require('process'); -const { rpc } = require('./loki_rpc'); +const { loki_rpc } = require('./loki_rpc'); const natUpnp = require('nat-upnp'); const resolve4 = url => @@ -94,6 +94,8 @@ class LokiSnodeAPI { fields: { public_ip: true, storage_port: true, + pubkey_x25519: true, + pubkey_ed25519: true, }, }; const seedNode = seedNodes.splice( @@ -101,7 +103,8 @@ class LokiSnodeAPI { 1 )[0]; try { - const result = await rpc( + + const result = await loki_rpc( `http://${seedNode.ip}`, seedNode.port, 'get_n_service_nodes', @@ -116,6 +119,8 @@ class LokiSnodeAPI { this.randomSnodePool = snodes.map(snode => ({ ip: snode.public_ip, port: snode.storage_port, + pubkey_x25519: snode.pubkey_x25519, + pubkey_ed25519: snode.pubkey_ed25519, })); } catch (e) { window.mixpanel.track('Seed Node Failed'); @@ -191,11 +196,12 @@ class LokiSnodeAPI { async getSwarmNodes(pubKey) { // TODO: Hit multiple random nodes and merge lists? - const { ip, port } = await this.getRandomSnodeAddress(); + const snode = await this.getRandomSnodeAddress(); try { - const result = await rpc(`https://${ip}`, port, 'get_snodes_for_pubkey', { + + const result = await loki_rpc(`https://${snode.ip}`, snode.port, 'get_snodes_for_pubkey', { pubKey, - }); + }, {}, '/storage_rpc/v1', snode); const snodes = result.snodes.filter(snode => snode.ip !== '0.0.0.0'); return snodes; } catch (e) { diff --git a/package.json b/package.json index c87bf2840..c9034a848 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "start-multi2": "NODE_APP_INSTANCE=2 electron .", "start-prod": "NODE_ENV=production NODE_APP_INSTANCE=devprod LOKI_DEV=1 electron .", "start-prod-multi": "NODE_ENV=production NODE_APP_INSTANCE=devprod1 LOKI_DEV=1 electron .", + "start-swarm-test": "NODE_ENV=swarm-testing NODE_APP_INSTANCE=test1 LOKI_DEV=1 electron .", + "start-swarm-test-2": "NODE_ENV=swarm-testing2 NODE_APP_INSTANCE=test2 LOKI_DEV=1 electron .", "grunt": "grunt", "icon-gen": "electron-icon-maker --input=images/icon_1024.png --output=./build", "generate": "yarn icon-gen && yarn grunt", diff --git a/preload.js b/preload.js index cc55e5aba..b1a02009e 100644 --- a/preload.js +++ b/preload.js @@ -471,4 +471,5 @@ window.SMALL_GROUP_SIZE_LIMIT = 10; window.lokiFeatureFlags = { multiDeviceUnpairing: true, privateGroupChats: false, + useSnodeProxy: false, }; From 9e10d11e9d298ea2bf8976575b8f73eec8958187 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Mon, 20 Jan 2020 21:19:31 -0800 Subject: [PATCH 02/17] hardcode file server pub key, pass it to _server --- js/modules/loki_file_server_api.js | 46 +++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/js/modules/loki_file_server_api.js b/js/modules/loki_file_server_api.js index 8cd1107f4..78c42f216 100644 --- a/js/modules/loki_file_server_api.js +++ b/js/modules/loki_file_server_api.js @@ -1,4 +1,4 @@ -/* global log, libloki */ +/* global log, libloki, process, window */ /* global storage: false */ /* global Signal: false */ /* global log: false */ @@ -8,11 +8,49 @@ const LokiAppDotNetAPI = require('./loki_app_dot_net_api'); const DEVICE_MAPPING_USER_ANNOTATION_TYPE = 'network.loki.messenger.devicemapping'; +// const LOKIFOUNDATION_DEVFILESERVER_PUBKEY = +// 'BSZiMVxOco/b3sYfaeyiMWv/JnqokxGXkHoclEx8TmZ6'; +const LOKIFOUNDATION_FILESERVER_PUBKEY = + 'BWJQnVm97sQE3Q1InB4Vuo+U/T1hmwHBv0ipkiv8tzEc'; + // can have multiple of these instances as each user can have a // different home server class LokiFileServerInstance { constructor(ourKey) { this.ourKey = ourKey; + + // do we have their pubkey locally? + /* + // get remote pubKey + this._server.serverRequest('loki/v1/public_key').then(keyRes => { + // we don't need to delay to protect identity because the token request + // should only be done over lokinet-lite + this.delayToken = true; + if (keyRes.err || !keyRes.response || !keyRes.response.data) { + if (keyRes.err) { + log.error(`Error ${keyRes.err}`); + } + } else { + // store it + this.pubKey = dcodeIO.ByteBuffer.wrap( + keyRes.response.data, + 'base64' + ).toArrayBuffer(); + // write it to a file + } + }); + */ + // Hard coded + this.pubKey = window.Signal.Crypto.base64ToArrayBuffer( + LOKIFOUNDATION_FILESERVER_PUBKEY + ); + if (this.pubKey.byteLength && this.pubKey.byteLength !== 33) { + log.error( + 'FILESERVER PUBKEY is invalid, length:', + this.pubKey.byteLength + ); + process.exit(1); + } } // FIXME: this is not file-server specific @@ -21,6 +59,10 @@ class LokiFileServerInstance { async establishConnection(serverUrl) { // why don't we extend this? this._server = new LokiAppDotNetAPI(this.ourKey, serverUrl); + + // configure proxy + this._server.pubKey = this.pubKey; + // get a token for multidevice const gotToken = await this._server.getOrRefreshServerToken(); // TODO: Handle this failure gracefully @@ -220,6 +262,8 @@ class LokiHomeServerInstance extends LokiFileServerInstance { return this._setOurDeviceMapping(authorisations, isPrimary); } + // you only upload to your own home server + // you can download from any server... uploadAvatar(data) { return this._server.uploadAvatar(data); } From 1a785a2435a2644bfbacba6544f363b3ceeddddd Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Mon, 20 Jan 2020 21:19:36 -0800 Subject: [PATCH 03/17] fix getSwarmNodes exception handler, lint --- js/modules/loki_snode_api.js | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 8c0370a33..eb2e43294 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -4,7 +4,7 @@ const is = require('@sindresorhus/is'); const dns = require('dns'); const process = require('process'); -const { loki_rpc } = require('./loki_rpc'); +const { lokiRPC } = require('./loki_rpc'); const natUpnp = require('nat-upnp'); const resolve4 = url => @@ -103,8 +103,7 @@ class LokiSnodeAPI { 1 )[0]; try { - - const result = await loki_rpc( + const result = await lokiRPC( `http://${seedNode.ip}`, seedNode.port, 'get_n_service_nodes', @@ -198,16 +197,24 @@ class LokiSnodeAPI { // TODO: Hit multiple random nodes and merge lists? const snode = await this.getRandomSnodeAddress(); try { - - const result = await loki_rpc(`https://${snode.ip}`, snode.port, 'get_snodes_for_pubkey', { - pubKey, - }, {}, '/storage_rpc/v1', snode); - const snodes = result.snodes.filter(snode => snode.ip !== '0.0.0.0'); + const result = await lokiRPC( + `https://${snode.ip}`, + snode.port, + 'get_snodes_for_pubkey', + { + pubKey, + }, + {}, + '/storage_rpc/v1', + snode + ); + const snodes = result.snodes.filter(tSnode => tSnode.ip !== '0.0.0.0'); return snodes; } catch (e) { + // this.randomSnodePool = _.without( this.randomSnodePool, - _.find(this.randomSnodePool, { ip }) + _.find(this.randomSnodePool, { ip: snode.ip }) ); return this.getSwarmNodes(pubKey); } From 3969817a0a188c25ab7778094aceabfaec251c13 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Mon, 20 Jan 2020 21:19:41 -0800 Subject: [PATCH 04/17] lint lint lint --- js/modules/loki_message_api.js | 16 +++-- js/modules/loki_rpc.js | 69 ++++++++++++---------- ts/components/conversation/AddMentions.tsx | 2 +- 3 files changed, 52 insertions(+), 35 deletions(-) diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index 9c8a94680..ce4d55b81 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -3,7 +3,7 @@ /* global log, dcodeIO, window, callWorker, lokiP2pAPI, lokiSnodeAPI, textsecure */ const _ = require('lodash'); -const { loki_rpc } = require('./loki_rpc'); +const { lokiRpc } = require('./loki_rpc'); const DEFAULT_CONNECTIONS = 3; const MAX_ACCEPTABLE_FAILURES = 1; @@ -47,7 +47,7 @@ const trySendP2p = async (pubKey, data64, isPing, messageEventData) => { return false; } try { - await loki_rpc(p2pDetails.address, p2pDetails.port, 'store', { + await lokiRpc(p2pDetails.address, p2pDetails.port, 'store', { data: data64, }); lokiP2pAPI.setContactOnline(pubKey); @@ -243,7 +243,15 @@ class LokiMessageAPI { while (successiveFailures < MAX_ACCEPTABLE_FAILURES) { await sleepFor(successiveFailures * 500); try { - const result = await loki_rpc(`https://${address}`, port, 'store', params, {}, '/storage_rpc/v1', targetNode); + const result = await lokiRpc( + `https://${address}`, + port, + 'store', + params, + {}, + '/storage_rpc/v1', + targetNode + ); // Make sure we aren't doing too much PoW const currentDifficulty = window.storage.get('PoWDifficulty', null); @@ -366,7 +374,7 @@ class LokiMessageAPI { }, }; - const result = await loki_rpc( + const result = await lokiRpc( `https://${nodeUrl}`, nodeData.port, 'retrieve', diff --git a/js/modules/loki_rpc.js b/js/modules/loki_rpc.js index 4113181c4..5f78ec07a 100644 --- a/js/modules/loki_rpc.js +++ b/js/modules/loki_rpc.js @@ -1,4 +1,5 @@ -/* global log, libloki, textsecure, getStoragePubKey, lokiSnodeAPI */ +/* global log, libloki, textsecure, getStoragePubKey, lokiSnodeAPI, StringView, + libsignal, window, TextDecoder, TextEncoder, dcodeIO */ const nodeFetch = require('node-fetch'); const { parse } = require('url'); @@ -23,61 +24,68 @@ const decryptResponse = async (response, address) => { // TODO: Don't allow arbitrary URLs, only snodes and loki servers const sendToProxy = async (options = {}, targetNode) => { + const randSnode = await lokiSnodeAPI.getRandomSnodeAddress(); - const rand_snode = await lokiSnodeAPI.getRandomSnodeAddress(); + const url = `https://${randSnode.ip}:${randSnode.port}/proxy`; - const url = `https://${rand_snode.ip}:${rand_snode.port}/proxy`; - - log.info(`Proxy snode reqeust to ${targetNode.pubkey_ed25519} via ${rand_snode.pubkey_ed25519}`); + log.info( + `Proxy snode request to ${targetNode.pubkey_ed25519} via ${ + randSnode.pubkey_ed25519 + }` + ); - const sn_pub_key_hex = StringView.hexToArrayBuffer(targetNode.pubkey_x25519); + const snPubkeyHex = StringView.hexToArrayBuffer(targetNode.pubkey_x25519); - const my_keys = window.libloki.crypto.snodeCipher._ephemeralKeyPair; + const myKeys = window.libloki.crypto.snodeCipher._ephemeralKeyPair; const symmetricKey = libsignal.Curve.calculateAgreement( - sn_pub_key_hex, - my_keys.privKey + snPubkeyHex, + myKeys.privKey ); const textEncoder = new TextEncoder(); const body = JSON.stringify(options); const plainText = textEncoder.encode(body); - const ivAndCiphertext = await window.libloki.crypto.DHEncrypt(symmetricKey, plainText); + const ivAndCiphertext = await window.libloki.crypto.DHEncrypt( + symmetricKey, + plainText + ); const firstHopOptions = { method: 'POST', body: ivAndCiphertext, headers: { - "X-Sender-Public-Key": StringView.arrayBufferToHex(my_keys.pubKey), - "X-Target-Snode-Key": targetNode.pubkey_ed25519, - } - } + 'X-Sender-Public-Key': StringView.arrayBufferToHex(myKeys.pubKey), + 'X-Target-Snode-Key': targetNode.pubkey_ed25519, + }, + }; const response = await nodeFetch(url, firstHopOptions); const ciphertext = await response.text(); const ciphertextBuffer = dcodeIO.ByteBuffer.wrap( - ciphertext, 'base64' + ciphertext, + 'base64' ).toArrayBuffer(); - const plaintextBuffer = await window.libloki.crypto.DHDecrypt(symmetricKey, ciphertextBuffer); + const plaintextBuffer = await window.libloki.crypto.DHDecrypt( + symmetricKey, + ciphertextBuffer + ); const textDecoder = new TextDecoder(); const plaintext = textDecoder.decode(plaintextBuffer); - const json_res = JSON.parse(plaintext); + const jsonRes = JSON.parse(plaintext); - json_res.json = () => { - return JSON.parse(json_res.body); - } + jsonRes.json = () => JSON.parse(jsonRes.body); - return json_res; - -} + return jsonRes; +}; // A small wrapper around node-fetch which deserializes response -const loki_fetch = async (url, options = {}, targetNode = null) => { +const lokiFetch = async (url, options = {}, targetNode = null) => { const timeout = options.timeout || 10000; const method = options.method || 'GET'; @@ -103,11 +111,12 @@ const loki_fetch = async (url, options = {}, targetNode = null) => { } const fetchOptions = { - ...options, timeout, method, + ...options, + timeout, + method, }; try { - if (window.lokiFeatureFlags.useSnodeProxy && targetNode) { const result = await sendToProxy(fetchOptions, targetNode); return result.json(); @@ -168,14 +177,14 @@ const loki_fetch = async (url, options = {}, targetNode = null) => { }; // Wrapper for a JSON RPC request -const loki_rpc = ( +const lokiRpc = ( address, port, method, params, options = {}, endpoint = endpointBase, - targetNode, + targetNode ) => { const headers = options.headers || {}; const portString = port ? `:${port}` : ''; @@ -206,9 +215,9 @@ const loki_rpc = ( }, }; - return loki_fetch(url, fetchOptions, targetNode); + return lokiFetch(url, fetchOptions, targetNode); }; module.exports = { - loki_rpc, + lokiRpc, }; diff --git a/ts/components/conversation/AddMentions.tsx b/ts/components/conversation/AddMentions.tsx index 8cfa0cb40..05cd170b8 100644 --- a/ts/components/conversation/AddMentions.tsx +++ b/ts/components/conversation/AddMentions.tsx @@ -31,12 +31,12 @@ class Mention extends React.Component { } public componentWillMount() { - this.setState({ found: false }); // TODO: give up after some period of time? this.intervalHandle = setInterval(this.tryRenameMention, 30000); + // tslint:disable-next-line:no-floating-promises this.tryRenameMention(); } From 4198f30b990881d5f99812758b2f0bef56be07b7 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Mon, 20 Jan 2020 21:20:45 -0800 Subject: [PATCH 05/17] Label errors better, make _sendToProxy use ephermal layering --- js/modules/loki_app_dot_net_api.js | 102 ++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 23 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index f50a5e441..24b7dc1db 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -1,6 +1,6 @@ /* global log, textsecure, libloki, Signal, Whisper, Headers, ConversationController, clearTimeout, MessageController, libsignal, StringView, window, _, -dcodeIO, Buffer */ +dcodeIO, Buffer, lokiSnodeAPI */ const nodeFetch = require('node-fetch'); const { URL, URLSearchParams } = require('url'); const FormData = require('form-data'); @@ -19,6 +19,9 @@ const MESSAGE_ATTACHMENT_TYPE = 'net.app.core.oembed'; const LOKI_ATTACHMENT_TYPE = 'attachment'; const LOKI_PREVIEW_TYPE = 'preview'; +// for onion routing +const IV_LENGTH = 16; + // the core ADN class that handles all communication with a specific server class LokiAppDotNetServerAPI { constructor(ourKey, url) { @@ -108,7 +111,7 @@ class LokiAppDotNetServerAPI { // no big deal if it fails... if (res.err || !res.response || !res.response.data) { if (res.err) { - log.error(`Error ${res.err}`); + log.error(`setProfileName Error ${res.err}`); } return []; } @@ -135,7 +138,7 @@ class LokiAppDotNetServerAPI { if (res.err || !res.response || !res.response.data) { if (res.err) { - log.error(`Error ${res.err}`); + log.error(`setHomeServer Error ${res.err}`); } return []; } @@ -292,23 +295,67 @@ class LokiAppDotNetServerAPI { } async _sendToProxy(fetchOptions, endpoint, method) { + const randSnode = await lokiSnodeAPI.getRandomSnodeAddress(); + const url = `https://${randSnode.ip}:${randSnode.port}/file_proxy`; + + // make temporary key for this request/response + const ephemeralKey = libsignal.Curve.generateKeyPair(); - const rand_snode = await lokiSnodeAPI.getRandomSnodeAddress(); - const url = `https://${rand_snode.ip}:${rand_snode.port}/file_proxy`; + // some randomness + const iv = libsignal.crypto.getRandomBytes(IV_LENGTH); - const body = fetchOptions.body; + // mix server pub key with our priv key + const symKey = libsignal.Curve.calculateAgreement( + this.pubKey, // server's pubkey + ephemeralKey.privKey // our privkey + ); + + const payloadObj = { + // I think this is a stream, we may need to collect it all? + body: fetchOptions.body, // might need to b64 if binary... + endpoint, + method, + headers: fetchOptions.headers, + }; + + // convert our payload to binary buffer + const payloadData = Buffer.from( + dcodeIO.ByteBuffer.wrap(JSON.stringify(payloadObj)).toArrayBuffer() + ); + + // encrypt payloadData with symmetric Key using iv + const cipherBody = await libsignal.crypto.encrypt(symKey, payloadData, iv); + + // make final buffer for cipherText + const ivAndCiphertext = new Uint8Array( + iv.byteLength + cipherBody.byteLength + ); + // add iv + ivAndCiphertext.set(new Uint8Array(iv)); + // add ciphertext after iv position + ivAndCiphertext.set(new Uint8Array(cipherBody), iv.byteLength); + + // convert final buffer to base64 + const cipherText64 = dcodeIO.ByteBuffer.wrap(ivAndCiphertext).toString( + 'base64' + ); + const ephemeralPubKey64 = dcodeIO.ByteBuffer.wrap( + ephemeralKey.privKey + ).toString('base64'); const firstHopOptions = { method: 'POST', - body, + cipherText64, headers: { - "X-Loki-File-Server-Target": `/${endpoint}`, - "X-Loki-File-Server-Verb": method, - "X-Loki-File-Server-Headers": JSON.stringify(fetchOptions.headers), - } - } + 'X-Loki-File-Server-Target': `/loki/v1/secure_rpc`, + 'X-Loki-File-Server-Verb': 'POST', + 'X-Loki-File-Server-Headers': JSON.stringify({ + 'X-Loki-File-Server-Ephemeral-Key': ephemeralPubKey64, + }), + }, + }; - return await nodeFetch(url, firstHopOptions); + return nodeFetch(url, firstHopOptions); } // make a request to the server @@ -320,6 +367,7 @@ class LokiAppDotNetServerAPI { objBody, forceFreshToken = false, } = options; + const url = new URL(`${this.baseServerUrl}/${endpoint}`); if (params) { url.search = new URLSearchParams(params); @@ -345,14 +393,22 @@ class LokiAppDotNetServerAPI { } fetchOptions.headers = new Headers(headers); - if (window.lokiFeatureFlags.useSnodeProxy && this.baseServerUrl === "https://file.lokinet.org") { - log.info("Sending a proxy request to https://file.lokinet.org"); - result = await this._sendToProxy({...fetchOptions, headers}, endpoint, method); + if ( + window.lokiFeatureFlags.useSnodeProxy && + (this.baseServerUrl === 'https://file-dev.lokinet.org' || + this.baseServerUrl === 'https://file.lokinet.org') + ) { + log.info('Sending a proxy request to', this.baseServerUrl); + result = await this._sendToProxy( + { ...fetchOptions, headers }, + endpoint, + method + ); } else { result = await nodeFetch(url, fetchOptions || undefined); } } catch (e) { - log.info(`e ${e}`); + log.info(`serverRequest nodeFetch/_sendToProxy error: ${e}`); return { err: e, }; @@ -404,7 +460,7 @@ class LokiAppDotNetServerAPI { if (res.err || !res.response || !res.response.data) { if (res.err) { - log.error(`Error ${res.err}`); + log.error(`getUserAnnotations Error ${res.err}`); } return []; } @@ -523,7 +579,7 @@ class LokiAppDotNetServerAPI { if (res.err || !res.response || !res.response.data) { if (res.err) { - log.error(`Error ${res.err}`); + log.error(`getSubscribers Error ${res.err}`); } return []; } @@ -553,7 +609,7 @@ class LokiAppDotNetServerAPI { if (res.err || !res.response || !res.response.data) { if (res.err) { - log.error(`Error ${res.err}`); + log.error(`getUsers Error ${res.err}`); } return []; } @@ -708,7 +764,7 @@ class LokiPublicChannelAPI { if (res.err || !res.response || !res.response.data) { if (res.err) { - log.error(`Error ${res.err}`); + log.error(`banUser Error ${res.err}`); } return false; } @@ -824,7 +880,7 @@ class LokiPublicChannelAPI { ); if (updateRes.err || !updateRes.response || !updateRes.response.data) { if (updateRes.err) { - log.error(`Error ${updateRes.err}`); + log.error(`setChannelSettings Error ${updateRes.err}`); } return false; } @@ -975,7 +1031,7 @@ class LokiPublicChannelAPI { // if any problems, abort out if (res.err || !res.response) { if (res.err) { - log.error(`Error ${res.err}`); + log.error(`pollOnceForDeletions Error ${res.err}`); } break; } From 6ab16d7e56728e100df83903953688ba3136bf50 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Tue, 21 Jan 2020 00:22:30 -0800 Subject: [PATCH 06/17] fix case so function exists, give a warning when initialiseRandomPool or getSwarmNodes fail --- js/modules/loki_snode_api.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index eb2e43294..6c561f4d6 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -4,7 +4,7 @@ const is = require('@sindresorhus/is'); const dns = require('dns'); const process = require('process'); -const { lokiRPC } = require('./loki_rpc'); +const { lokiRpc } = require('./loki_rpc'); const natUpnp = require('nat-upnp'); const resolve4 = url => @@ -103,7 +103,7 @@ class LokiSnodeAPI { 1 )[0]; try { - const result = await lokiRPC( + const result = await lokiRpc( `http://${seedNode.ip}`, seedNode.port, 'get_n_service_nodes', @@ -122,6 +122,7 @@ class LokiSnodeAPI { pubkey_ed25519: snode.pubkey_ed25519, })); } catch (e) { + log.warn('initialiseRandomPool error', JSON.stringify(e)) window.mixpanel.track('Seed Node Failed'); if (seedNodes.length === 0) { throw new window.textsecure.SeedNodeError( @@ -197,7 +198,7 @@ class LokiSnodeAPI { // TODO: Hit multiple random nodes and merge lists? const snode = await this.getRandomSnodeAddress(); try { - const result = await lokiRPC( + const result = await lokiRpc( `https://${snode.ip}`, snode.port, 'get_snodes_for_pubkey', @@ -211,6 +212,7 @@ class LokiSnodeAPI { const snodes = result.snodes.filter(tSnode => tSnode.ip !== '0.0.0.0'); return snodes; } catch (e) { + log.error('getSwarmNodes', JSON.stringify(e)); // this.randomSnodePool = _.without( this.randomSnodePool, From ce2e85cd38497a5c5742efc7b5b5fbe99c480a4f Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Tue, 21 Jan 2020 02:10:22 -0800 Subject: [PATCH 07/17] send ephermal pubkey not privkey, file_proxy JSON encode body --- js/modules/loki_app_dot_net_api.js | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 24b7dc1db..4f819bfe0 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -301,8 +301,14 @@ class LokiAppDotNetServerAPI { // make temporary key for this request/response const ephemeralKey = libsignal.Curve.generateKeyPair(); + function buf2hex(buffer) { // buffer is an ArrayBuffer + return Array.prototype.map.call(new Uint8Array(buffer), x => ('00' + x.toString(16)).slice(-2)).join(''); + } + // some randomness const iv = libsignal.crypto.getRandomBytes(IV_LENGTH); + // console.log('iv ', buf2hex(iv)) + // console.log('epk', buf2hex(ephemeralKey.pubKey)) // mix server pub key with our priv key const symKey = libsignal.Curve.calculateAgreement( @@ -339,19 +345,30 @@ class LokiAppDotNetServerAPI { const cipherText64 = dcodeIO.ByteBuffer.wrap(ivAndCiphertext).toString( 'base64' ); + + //console.log('ephemeralKey.privKey', ephemeralKey.privKey.toString('hex')) + const ephemeralPubKey64 = dcodeIO.ByteBuffer.wrap( - ephemeralKey.privKey + ephemeralKey.pubKey ).toString('base64'); + // console.log('ephemeralKey', ephemeralPubKey64); + // console.log('cipherText64', cipherText64); + + const finalRequestHeader = { + 'X-Loki-File-Server-Ephemeral-Key': ephemeralPubKey64, + }; + const firstHopOptions = { method: 'POST', - cipherText64, + // not sure why I can't use anything but json... + // text/plain would be preferred... + body: JSON.stringify({ cipherText64 }), headers: { + 'Content-Type': 'application/json', 'X-Loki-File-Server-Target': `/loki/v1/secure_rpc`, 'X-Loki-File-Server-Verb': 'POST', - 'X-Loki-File-Server-Headers': JSON.stringify({ - 'X-Loki-File-Server-Ephemeral-Key': ephemeralPubKey64, - }), + 'X-Loki-File-Server-Headers': JSON.stringify(finalRequestHeader), }, }; @@ -404,11 +421,12 @@ class LokiAppDotNetServerAPI { endpoint, method ); + log.info('Got proxy response', result, 'for', method || 'GET', endpoint); } else { result = await nodeFetch(url, fetchOptions || undefined); } } catch (e) { - log.info(`serverRequest nodeFetch/_sendToProxy error: ${e}`); + log.info('serverRequest nodeFetch/_sendToProxy error:', JSON.stringify(e)); return { err: e, }; From e51a030dc98b78d84742329155920f1261902607 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Wed, 22 Jan 2020 23:52:31 -0800 Subject: [PATCH 08/17] include log --- js/modules/loki_snode_api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 6c561f4d6..dc1a357b3 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -1,5 +1,5 @@ /* eslint-disable class-methods-use-this */ -/* global window, ConversationController, _ */ +/* global window, ConversationController, _, log */ const is = require('@sindresorhus/is'); const dns = require('dns'); From f8252ec1ece36ebaf51528d30a763e76a364b951 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Thu, 23 Jan 2020 00:02:12 -0800 Subject: [PATCH 09/17] decrypt file server response, remove debug, handle crypt before _sendToProxy, improve json parsing failure logging --- js/modules/loki_app_dot_net_api.js | 76 ++++++++++++++++++------------ 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 4f819bfe0..5fbf35354 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -1,6 +1,6 @@ /* global log, textsecure, libloki, Signal, Whisper, Headers, ConversationController, clearTimeout, MessageController, libsignal, StringView, window, _, -dcodeIO, Buffer, lokiSnodeAPI */ +dcodeIO, Buffer, lokiSnodeAPI, TextDecoder */ const nodeFetch = require('node-fetch'); const { URL, URLSearchParams } = require('url'); const FormData = require('form-data'); @@ -294,28 +294,10 @@ class LokiAppDotNetServerAPI { } } - async _sendToProxy(fetchOptions, endpoint, method) { + static async _sendToProxy(fetchOptions, endpoint, method, { ephemeralKey, symKey, iv }) { const randSnode = await lokiSnodeAPI.getRandomSnodeAddress(); const url = `https://${randSnode.ip}:${randSnode.port}/file_proxy`; - // make temporary key for this request/response - const ephemeralKey = libsignal.Curve.generateKeyPair(); - - function buf2hex(buffer) { // buffer is an ArrayBuffer - return Array.prototype.map.call(new Uint8Array(buffer), x => ('00' + x.toString(16)).slice(-2)).join(''); - } - - // some randomness - const iv = libsignal.crypto.getRandomBytes(IV_LENGTH); - // console.log('iv ', buf2hex(iv)) - // console.log('epk', buf2hex(ephemeralKey.pubKey)) - - // mix server pub key with our priv key - const symKey = libsignal.Curve.calculateAgreement( - this.pubKey, // server's pubkey - ephemeralKey.privKey // our privkey - ); - const payloadObj = { // I think this is a stream, we may need to collect it all? body: fetchOptions.body, // might need to b64 if binary... @@ -346,15 +328,10 @@ class LokiAppDotNetServerAPI { 'base64' ); - //console.log('ephemeralKey.privKey', ephemeralKey.privKey.toString('hex')) - const ephemeralPubKey64 = dcodeIO.ByteBuffer.wrap( ephemeralKey.pubKey ).toString('base64'); - // console.log('ephemeralKey', ephemeralPubKey64); - // console.log('cipherText64', cipherText64); - const finalRequestHeader = { 'X-Loki-File-Server-Ephemeral-Key': ephemeralPubKey64, }; @@ -390,6 +367,9 @@ class LokiAppDotNetServerAPI { url.search = new URLSearchParams(params); } let result; + let ephemeralKey; + let symKey; + let iv; try { const fetchOptions = {}; const headers = {}; @@ -415,13 +395,23 @@ class LokiAppDotNetServerAPI { (this.baseServerUrl === 'https://file-dev.lokinet.org' || this.baseServerUrl === 'https://file.lokinet.org') ) { + // make temporary key for this request/response + ephemeralKey = libsignal.Curve.generateKeyPair(); + // mix server pub key with our priv key + symKey = libsignal.Curve.calculateAgreement( + this.pubKey, // server's pubkey + ephemeralKey.privKey // our privkey + ); + // some randomness + iv = libsignal.crypto.getRandomBytes(IV_LENGTH); + log.info('Sending a proxy request to', this.baseServerUrl); result = await this._sendToProxy( { ...fetchOptions, headers }, endpoint, - method + method, + { ephemeralKey, symKey, iv } ); - log.info('Got proxy response', result, 'for', method || 'GET', endpoint); } else { result = await nodeFetch(url, fetchOptions || undefined); } @@ -432,16 +422,44 @@ class LokiAppDotNetServerAPI { }; } let response = null; + let TxtResponse = ''; try { - response = await result.json(); + TxtResponse = await result.text(); + response = JSON.parse(TxtResponse); } catch (e) { - log.warn(`serverRequest json parse ${e}`); + log.warn(`serverRequest json parse ${e} ${TxtResponse}`); return { err: e, statusCode: result.status, }; } + + if ( + window.lokiFeatureFlags.useSnodeProxy && + (this.baseServerUrl === 'https://file-dev.lokinet.org' || + this.baseServerUrl === 'https://file.lokinet.org') + ) { + // log.info('Got proxy response', response, 'for', method || 'GET', endpoint); + if (response.meta && response.meta.code === 200) { + try { + const cipherBuffer = dcodeIO.ByteBuffer.wrap(response.data, 'base64').toArrayBuffer() + const decryped = await libsignal.crypto.decrypt(symKey, cipherBuffer, iv); + const textDecoder = new TextDecoder(); + const json = textDecoder.decode(decryped); + response = JSON.parse(json); + } catch(e) { + // useless with the ephemeralKey and iv + log.warn(`serverRequest useSnodeProxy parse ${e} ${TxtResponse}`); + return { + err: e, + statusCode: result.status, + }; + } + // log.info('decrypted response', response); + } + } + // if it's a response style with a meta if (result.status !== 200) { if (!forceFreshToken && (!response.meta || response.meta.code === 401)) { From 41e35a1647100e6cb39a473a3f602df8590bcba0 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Thu, 23 Jan 2020 03:30:54 -0800 Subject: [PATCH 10/17] support file uploads on file proxy, fix _sendToProxy calling --- js/modules/loki_app_dot_net_api.js | 52 ++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 5fbf35354..4b15b83fd 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -294,7 +294,12 @@ class LokiAppDotNetServerAPI { } } - static async _sendToProxy(fetchOptions, endpoint, method, { ephemeralKey, symKey, iv }) { + static async _sendToProxy( + fetchOptions, + endpoint, + method, + { ephemeralKey, symKey, iv } + ) { const randSnode = await lokiSnodeAPI.getRandomSnodeAddress(); const url = `https://${randSnode.ip}:${randSnode.port}/file_proxy`; @@ -306,10 +311,28 @@ class LokiAppDotNetServerAPI { headers: fetchOptions.headers, }; + // from https://github.com/sindresorhus/is-stream/blob/master/index.js + if ( + payloadObj.body && + typeof payloadObj.body === 'object' && + typeof payloadObj.body.pipe === 'function' + ) { + log.info('detected body is a stream'); + const fData = payloadObj.body.getBuffer(); + const fHeaders = payloadObj.body.getHeaders(); + // update headers for boundary + payloadObj.headers = { ...payloadObj.headers, ...fHeaders }; + // update body with base64 chunk + payloadObj.body = { + fileUpload: fData.toString('base64'), + }; + } + // convert our payload to binary buffer const payloadData = Buffer.from( dcodeIO.ByteBuffer.wrap(JSON.stringify(payloadObj)).toArrayBuffer() ); + payloadObj.body = false; // free memory // encrypt payloadData with symmetric Key using iv const cipherBody = await libsignal.crypto.encrypt(symKey, payloadData, iv); @@ -343,12 +366,11 @@ class LokiAppDotNetServerAPI { body: JSON.stringify({ cipherText64 }), headers: { 'Content-Type': 'application/json', - 'X-Loki-File-Server-Target': `/loki/v1/secure_rpc`, + 'X-Loki-File-Server-Target': '/loki/v1/secure_rpc', 'X-Loki-File-Server-Verb': 'POST', 'X-Loki-File-Server-Headers': JSON.stringify(finalRequestHeader), }, }; - return nodeFetch(url, firstHopOptions); } @@ -406,17 +428,21 @@ class LokiAppDotNetServerAPI { iv = libsignal.crypto.getRandomBytes(IV_LENGTH); log.info('Sending a proxy request to', this.baseServerUrl); - result = await this._sendToProxy( + result = await this.constructor._sendToProxy( { ...fetchOptions, headers }, endpoint, method, { ephemeralKey, symKey, iv } ); + // log.info('sent to proxy') } else { result = await nodeFetch(url, fetchOptions || undefined); } } catch (e) { - log.info('serverRequest nodeFetch/_sendToProxy error:', JSON.stringify(e)); + log.info( + 'serverRequest nodeFetch/_sendToProxy error:', + JSON.stringify(e) + ); return { err: e, }; @@ -434,7 +460,6 @@ class LokiAppDotNetServerAPI { }; } - if ( window.lokiFeatureFlags.useSnodeProxy && (this.baseServerUrl === 'https://file-dev.lokinet.org' || @@ -443,12 +468,19 @@ class LokiAppDotNetServerAPI { // log.info('Got proxy response', response, 'for', method || 'GET', endpoint); if (response.meta && response.meta.code === 200) { try { - const cipherBuffer = dcodeIO.ByteBuffer.wrap(response.data, 'base64').toArrayBuffer() - const decryped = await libsignal.crypto.decrypt(symKey, cipherBuffer, iv); + const cipherBuffer = dcodeIO.ByteBuffer.wrap( + response.data, + 'base64' + ).toArrayBuffer(); + const decryped = await libsignal.crypto.decrypt( + symKey, + cipherBuffer, + iv + ); const textDecoder = new TextDecoder(); const json = textDecoder.decode(decryped); response = JSON.parse(json); - } catch(e) { + } catch (e) { // useless with the ephemeralKey and iv log.warn(`serverRequest useSnodeProxy parse ${e} ${TxtResponse}`); return { @@ -457,6 +489,8 @@ class LokiAppDotNetServerAPI { }; } // log.info('decrypted response', response); + } else { + log.warn('file server secure_rpc gave an non-200 response'); } } From 6293cd468b82b1241620a6b14b8e925739c69b78 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Thu, 23 Jan 2020 03:31:14 -0800 Subject: [PATCH 11/17] bump form-data to 3.0 --- package.json | 2 +- yarn.lock | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index c9034a848..9f239d903 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "emoji-panel": "https://github.com/scottnonnenberg-signal/emoji-panel.git#v0.5.5", "filesize": "3.6.1", "firstline": "1.2.1", - "form-data": "2.3.2", + "form-data": "^3.0.0", "fs-extra": "5.0.0", "glob": "7.1.2", "google-libphonenumber": "3.2.2", diff --git a/yarn.lock b/yarn.lock index 6aa154576..72dd93c2c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1805,6 +1805,13 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + commander@2.11.0: version "2.11.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" @@ -3522,13 +3529,13 @@ forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" -form-data@2.3.2, form-data@~2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099" - integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk= +form-data@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" + integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== dependencies: asynckit "^0.4.0" - combined-stream "1.0.6" + combined-stream "^1.0.8" mime-types "^2.1.12" form-data@~2.1.1: @@ -3547,6 +3554,15 @@ form-data@~2.3.1: combined-stream "^1.0.5" mime-types "^2.1.12" +form-data@~2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099" + integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk= + dependencies: + asynckit "^0.4.0" + combined-stream "1.0.6" + mime-types "^2.1.12" + forwarded@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" From 5ced9a2b85a17ed7a514e12b4c5f9098f21fce76 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Thu, 23 Jan 2020 14:56:18 -0800 Subject: [PATCH 12/17] initial refactor of feaure flag detection statements in serverRequest() --- js/modules/loki_app_dot_net_api.js | 135 ++++++++++++++--------------- 1 file changed, 65 insertions(+), 70 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 4b15b83fd..2b10a8766 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -388,12 +388,8 @@ class LokiAppDotNetServerAPI { if (params) { url.search = new URLSearchParams(params); } - let result; - let ephemeralKey; - let symKey; - let iv; + const fetchOptions = {}; try { - const fetchOptions = {}; const headers = {}; if (forceFreshToken) { await this.getOrRefreshServerToken(true); @@ -411,86 +407,85 @@ class LokiAppDotNetServerAPI { fetchOptions.body = rawBody; } fetchOptions.headers = new Headers(headers); + } catch (e) { + log.info('serverRequest set up error:', JSON.stringify(e)); + return { + err: e, + }; + } - if ( - window.lokiFeatureFlags.useSnodeProxy && - (this.baseServerUrl === 'https://file-dev.lokinet.org' || - this.baseServerUrl === 'https://file.lokinet.org') - ) { + let result; + let response = null; + let TxtResponse = ''; + if ( + window.lokiFeatureFlags.useSnodeProxy && + (this.baseServerUrl === 'https://file-dev.lokinet.org' || + this.baseServerUrl === 'https://file.lokinet.org') + ) { + try { // make temporary key for this request/response - ephemeralKey = libsignal.Curve.generateKeyPair(); + const ephemeralKey = libsignal.Curve.generateKeyPair(); // mix server pub key with our priv key - symKey = libsignal.Curve.calculateAgreement( + const symKey = libsignal.Curve.calculateAgreement( this.pubKey, // server's pubkey ephemeralKey.privKey // our privkey ); + // some randomness - iv = libsignal.crypto.getRandomBytes(IV_LENGTH); + const iv = libsignal.crypto.getRandomBytes(IV_LENGTH); - log.info('Sending a proxy request to', this.baseServerUrl); result = await this.constructor._sendToProxy( - { ...fetchOptions, headers }, + fetchOptions, endpoint, method, { ephemeralKey, symKey, iv } ); - // log.info('sent to proxy') - } else { - result = await nodeFetch(url, fetchOptions || undefined); - } - } catch (e) { - log.info( - 'serverRequest nodeFetch/_sendToProxy error:', - JSON.stringify(e) - ); - return { - err: e, - }; - } - let response = null; - let TxtResponse = ''; - try { - TxtResponse = await result.text(); - response = JSON.parse(TxtResponse); - } catch (e) { - log.warn(`serverRequest json parse ${e} ${TxtResponse}`); - return { - err: e, - statusCode: result.status, - }; - } - if ( - window.lokiFeatureFlags.useSnodeProxy && - (this.baseServerUrl === 'https://file-dev.lokinet.org' || - this.baseServerUrl === 'https://file.lokinet.org') - ) { - // log.info('Got proxy response', response, 'for', method || 'GET', endpoint); - if (response.meta && response.meta.code === 200) { - try { - const cipherBuffer = dcodeIO.ByteBuffer.wrap( - response.data, - 'base64' - ).toArrayBuffer(); - const decryped = await libsignal.crypto.decrypt( - symKey, - cipherBuffer, - iv - ); - const textDecoder = new TextDecoder(); - const json = textDecoder.decode(decryped); - response = JSON.parse(json); - } catch (e) { - // useless with the ephemeralKey and iv - log.warn(`serverRequest useSnodeProxy parse ${e} ${TxtResponse}`); - return { - err: e, - statusCode: result.status, - }; + TxtResponse = await result.text(); + response = JSON.parse(TxtResponse); + + if (response.meta && response.meta.code === 200) { + try { + const cipherBuffer = dcodeIO.ByteBuffer.wrap( + response.data, + 'base64' + ).toArrayBuffer(); + const decryped = await libsignal.crypto.decrypt( + symKey, + cipherBuffer, + iv + ); + const textDecoder = new TextDecoder(); + const json = textDecoder.decode(decryped); + // replace response + response = JSON.parse(json); + } catch (e) { + // useless with the ephemeralKey and iv + log.warn(`serverRequest useSnodeProxy parse ${e} ${TxtResponse}`); + return { + err: e, + statusCode: result.status, + }; + } + } else { + log.warn('file server secure_rpc gave an non-200 response'); } - // log.info('decrypted response', response); - } else { - log.warn('file server secure_rpc gave an non-200 response'); + } catch (e) { + log.info('serverRequest _sendToProxy error:', e); + return { + err: e, + }; + } + } else { + try { + result = await nodeFetch(url, fetchOptions || undefined); + TxtResponse = await result.text(); + response = JSON.parse(TxtResponse); + } catch (e) { + log.info('serverRequest nodeFetch error:', JSON.stringify(e)); + return { + err: e, + }; } } From 56648a6e30e4715e7bb16d275b24b6d7ffacea4d Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Tue, 28 Jan 2020 00:46:16 -0800 Subject: [PATCH 13/17] remove date, we have git --- js/background.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/background.js b/js/background.js index b8a69eebe..ebb899812 100644 --- a/js/background.js +++ b/js/background.js @@ -644,7 +644,7 @@ Whisper.events.on('registration_done', async () => { window.log.info('handling registration event'); - // Disable link previews as default per Kee 20/01/28 + // Disable link previews as default per Kee storage.onready(async () => { storage.put('linkPreviews', false); }); From f2233c20c7fb39c88bbf57b6732d478c4e7245f3 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Tue, 28 Jan 2020 00:47:02 -0800 Subject: [PATCH 14/17] add missing semicolon --- js/modules/loki_snode_api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index dc1a357b3..53ee50c1e 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -122,7 +122,7 @@ class LokiSnodeAPI { pubkey_ed25519: snode.pubkey_ed25519, })); } catch (e) { - log.warn('initialiseRandomPool error', JSON.stringify(e)) + log.warn('initialiseRandomPool error', JSON.stringify(e)); window.mixpanel.track('Seed Node Failed'); if (seedNodes.length === 0) { throw new window.textsecure.SeedNodeError( From ab83f8914201825e206ef8bef7efea0a329f5cf6 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Tue, 28 Jan 2020 00:55:16 -0800 Subject: [PATCH 15/17] _sendToProxy pass headers/handle response refactor, lint --- js/modules/loki_app_dot_net_api.js | 138 +++++++++++++---------------- 1 file changed, 60 insertions(+), 78 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index 1df3418d3..c08b6a112 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -294,12 +294,7 @@ class LokiAppDotNetServerAPI { } } - static async _sendToProxy( - fetchOptions, - endpoint, - method, - { ephemeralKey, symKey, iv } - ) { + async _sendToProxy(fetchOptions, method, headers, endpoint) { const randSnode = await lokiSnodeAPI.getRandomSnodeAddress(); const url = `https://${randSnode.ip}:${randSnode.port}/file_proxy`; @@ -308,7 +303,7 @@ class LokiAppDotNetServerAPI { body: fetchOptions.body, // might need to b64 if binary... endpoint, method, - headers: fetchOptions.headers, + headers, }; // from https://github.com/sindresorhus/is-stream/blob/master/index.js @@ -334,6 +329,18 @@ class LokiAppDotNetServerAPI { ); payloadObj.body = false; // free memory + // make temporary key for this request/response + const ephemeralKey = libsignal.Curve.generateKeyPair(); + + // mix server pub key with our priv key + const symKey = libsignal.Curve.calculateAgreement( + this.pubKey, // server's pubkey + ephemeralKey.privKey // our privkey + ); + + // some randomness + const iv = libsignal.crypto.getRandomBytes(IV_LENGTH); + // encrypt payloadData with symmetric Key using iv const cipherBody = await libsignal.crypto.encrypt(symKey, payloadData, iv); @@ -371,7 +378,25 @@ class LokiAppDotNetServerAPI { 'X-Loki-File-Server-Headers': JSON.stringify(finalRequestHeader), }, }; - return nodeFetch(url, firstHopOptions); + const result = await nodeFetch(url, firstHopOptions); + + const txtResponse = await result.text(); + let response = JSON.parse(txtResponse); + + if (response.meta && response.meta.code === 200) { + const cipherBuffer = dcodeIO.ByteBuffer.wrap( + response.data, + 'base64' + ).toArrayBuffer(); + const decryped = await libsignal.crypto.decrypt(symKey, cipherBuffer, iv); + const textDecoder = new TextDecoder(); + const json = textDecoder.decode(decryped); + // replace response + response = JSON.parse(json); + } else { + log.warn('file server secure_rpc gave an non-200 response'); + } + return { result, txtResponse, response }; } // make a request to the server @@ -389,8 +414,8 @@ class LokiAppDotNetServerAPI { url.search = new URLSearchParams(params); } const fetchOptions = {}; + const headers = {}; try { - const headers = {}; if (forceFreshToken) { await this.getOrRefreshServerToken(true); } @@ -414,81 +439,35 @@ class LokiAppDotNetServerAPI { }; } + let response; let result; - let response = null; - let TxtResponse = ''; - if ( - window.lokiFeatureFlags.useSnodeProxy && - (this.baseServerUrl === 'https://file-dev.lokinet.org' || - this.baseServerUrl === 'https://file.lokinet.org') - ) { - try { - // make temporary key for this request/response - const ephemeralKey = libsignal.Curve.generateKeyPair(); - // mix server pub key with our priv key - const symKey = libsignal.Curve.calculateAgreement( - this.pubKey, // server's pubkey - ephemeralKey.privKey // our privkey - ); - - // some randomness - const iv = libsignal.crypto.getRandomBytes(IV_LENGTH); - - result = await this.constructor._sendToProxy( + let txtResponse; + let mode = 'nodeFetch'; + try { + if ( + window.lokiFeatureFlags.useSnodeProxy && + (this.baseServerUrl === 'https://file-dev.lokinet.org' || + this.baseServerUrl === 'https://file.lokinet.org') + ) { + mode = '_sendToProxy'; + // have to send headers because fetchOptions.headers isn't readable + ({ response, txtResponse, result } = await this._sendToProxy( fetchOptions, - endpoint, method, - { ephemeralKey, symKey, iv } - ); - - TxtResponse = await result.text(); - response = JSON.parse(TxtResponse); - - if (response.meta && response.meta.code === 200) { - try { - const cipherBuffer = dcodeIO.ByteBuffer.wrap( - response.data, - 'base64' - ).toArrayBuffer(); - const decryped = await libsignal.crypto.decrypt( - symKey, - cipherBuffer, - iv - ); - const textDecoder = new TextDecoder(); - const json = textDecoder.decode(decryped); - // replace response - response = JSON.parse(json); - } catch (e) { - // useless with the ephemeralKey and iv - log.warn(`serverRequest useSnodeProxy parse ${e} ${TxtResponse}`); - return { - err: e, - statusCode: result.status, - }; - } - } else { - log.warn('file server secure_rpc gave an non-200 response'); - } - } catch (e) { - log.info('serverRequest _sendToProxy error:', e); - return { - err: e, - }; - } - } else { - try { + headers, + endpoint + )); + } else { result = await nodeFetch(url, fetchOptions || undefined); - TxtResponse = await result.text(); - response = JSON.parse(TxtResponse); - } catch (e) { - log.info('serverRequest nodeFetch error:', JSON.stringify(e)); - return { - err: e, - }; + txtResponse = await result.text(); + response = JSON.parse(txtResponse); } + } catch (e) { + log.info(`serverRequest ${mode} error json: ${txtResponse}`); + return { + err: e, + }; } - // if it's a response style with a meta if (result.status !== 200) { if (!forceFreshToken && (!response.meta || response.meta.code === 401)) { @@ -767,6 +746,7 @@ class LokiAppDotNetServerAPI { contentType: 'application/octet-stream', name: 'content', filename: 'attachment', + knownLength: buffer.byteLength, }); return this.uploadData(formData); @@ -1510,6 +1490,7 @@ class LokiPublicChannelAPI { const primaryPubKey = slavePrimaryMap[slaveKey]; // send out remaining messages for this merged identity + /* eslint-disable no-param-reassign */ if (slavePrimaryMap[slaveKey]) { // rewrite source, profile messageData.source = primaryPubKey; @@ -1520,6 +1501,7 @@ class LokiPublicChannelAPI { messageData.message.profile.avatar = avatar; messageData.message.profileKey = profileKey; } + /* eslint-enable no-param-reassign */ this.chatAPI.emit('publicMessage', { message: messageData, }); From df120e8e5f6a7092fa8c8c4644600b607b2f22b5 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Tue, 28 Jan 2020 00:55:41 -0800 Subject: [PATCH 16/17] fix my yarn conflict/resolve --- yarn.lock | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/yarn.lock b/yarn.lock index 62dfcf124..b7bd6eabc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1991,13 +1991,6 @@ combined-stream@^1.0.6, combined-stream@~1.0.5, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -combined-stream@^1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" - integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== - dependencies: - delayed-stream "~1.0.0" - commander@2.11.0: version "2.11.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" @@ -3835,17 +3828,12 @@ form-data@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== - -form-data@2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099" - integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk= dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@~2.3.1, form-data@~2.3.2: +form-data@~2.3.1: version "2.3.3" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== From 758a936e8f72936a09f0c337874a1b478d07b709 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Tue, 28 Jan 2020 15:09:57 -0800 Subject: [PATCH 17/17] include IV in server response --- js/modules/loki_app_dot_net_api.js | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/js/modules/loki_app_dot_net_api.js b/js/modules/loki_app_dot_net_api.js index c08b6a112..c885f42a5 100644 --- a/js/modules/loki_app_dot_net_api.js +++ b/js/modules/loki_app_dot_net_api.js @@ -19,9 +19,6 @@ const MESSAGE_ATTACHMENT_TYPE = 'net.app.core.oembed'; const LOKI_ATTACHMENT_TYPE = 'attachment'; const LOKI_PREVIEW_TYPE = 'preview'; -// for onion routing -const IV_LENGTH = 16; - // the core ADN class that handles all communication with a specific server class LokiAppDotNetServerAPI { constructor(ourKey, url) { @@ -338,20 +335,7 @@ class LokiAppDotNetServerAPI { ephemeralKey.privKey // our privkey ); - // some randomness - const iv = libsignal.crypto.getRandomBytes(IV_LENGTH); - - // encrypt payloadData with symmetric Key using iv - const cipherBody = await libsignal.crypto.encrypt(symKey, payloadData, iv); - - // make final buffer for cipherText - const ivAndCiphertext = new Uint8Array( - iv.byteLength + cipherBody.byteLength - ); - // add iv - ivAndCiphertext.set(new Uint8Array(iv)); - // add ciphertext after iv position - ivAndCiphertext.set(new Uint8Array(cipherBody), iv.byteLength); + const ivAndCiphertext = await libloki.crypto.DHEncrypt(symKey, payloadData); // convert final buffer to base64 const cipherText64 = dcodeIO.ByteBuffer.wrap(ivAndCiphertext).toString( @@ -384,13 +368,17 @@ class LokiAppDotNetServerAPI { let response = JSON.parse(txtResponse); if (response.meta && response.meta.code === 200) { - const cipherBuffer = dcodeIO.ByteBuffer.wrap( + // convert base64 in response to binary + const ivAndCiphertextResponse = dcodeIO.ByteBuffer.wrap( response.data, 'base64' ).toArrayBuffer(); - const decryped = await libsignal.crypto.decrypt(symKey, cipherBuffer, iv); + const decrypted = await libloki.crypto.DHDecrypt( + symKey, + ivAndCiphertextResponse + ); const textDecoder = new TextDecoder(); - const json = textDecoder.decode(decryped); + const json = textDecoder.decode(decrypted); // replace response response = JSON.parse(json); } else {