From eea2b8e6a7dc81aaaa3572fdc3750b9d119f4b37 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Tue, 4 Jun 2019 14:01:51 +1000 Subject: [PATCH 1/6] Do port null check inside rpc call --- config/default.json | 1 - js/modules/loki_message_api.js | 61 ++++++++++++++++------------------ js/modules/loki_rpc.js | 3 +- js/modules/loki_snode_api.js | 6 ++-- main.js | 1 - preload.js | 6 +--- 6 files changed, 35 insertions(+), 43 deletions(-) diff --git a/config/default.json b/config/default.json index c2b0f0e2f..a9ec560cc 100644 --- a/config/default.json +++ b/config/default.json @@ -4,7 +4,6 @@ "cdnUrl": "random.snode", "contentProxyUrl": "random.snode", "localServerPort": "8081", - "snodeServerPort": "8080", "defaultPoWDifficulty": "100", "disableAutoUpdate": false, "updatesUrl": "https://updates2.signal.org/desktop", diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index 75de95d4b..f58bb9b04 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -43,9 +43,7 @@ const trySendP2p = async (pubKey, data64, isPing, messageEventData) => { return false; } try { - const port = p2pDetails.port ? `:${p2pDetails.port}` : ''; - - await rpc(p2pDetails.address, port, 'store', { + await rpc(p2pDetails.address, p2pDetails.port, 'store', { data: data64, }); lokiP2pAPI.setContactOnline(pubKey); @@ -68,9 +66,30 @@ const trySendP2p = async (pubKey, data64, isPing, messageEventData) => { } }; +const retrieveNextMessages = async (nodeUrl, nodeData, ourKey) => { + const params = { + pubKey: ourKey, + lastHash: nodeData.lastHash || '', + }; + const options = { + timeout: 40000, + headers: { + [LOKI_LONGPOLL_HEADER]: true, + }, + }; + + const result = await rpc( + `https://${nodeUrl}`, + nodeData.port, + 'retrieve', + params, + options + ); + return result.messages || []; +} + class LokiMessageAPI { - constructor({ snodeServerPort }) { - this.snodeServerPort = snodeServerPort ? `:${snodeServerPort}` : ''; + constructor() { this.jobQueue = new window.JobQueue(); this.sendingSwarmNodes = {}; } @@ -208,46 +227,24 @@ class LokiMessageAPI { return false; } - async retrieveNextMessages(nodeUrl, nodeData, ourKey) { - const params = { - pubKey: ourKey, - lastHash: nodeData.lastHash || '', - }; - const options = { - timeout: 40000, - headers: { - [LOKI_LONGPOLL_HEADER]: true, - }, - }; - - const result = await rpc( - `https://${nodeUrl}`, - this.snodeServerPort, - 'retrieve', - params, - options - ); - return result.messages || []; - } - async openConnection(callback) { const ourKey = window.textsecure.storage.user.getNumber(); while (!_.isEmpty(this.ourSwarmNodes)) { - const url = Object.keys(this.ourSwarmNodes)[0]; - const nodeData = this.ourSwarmNodes[url]; - delete this.ourSwarmNodes[url]; + const address = Object.keys(this.ourSwarmNodes)[0]; + const nodeData = this.ourSwarmNodes[address]; + delete this.ourSwarmNodes[address]; let successiveFailures = 0; while (successiveFailures < 3) { await sleepFor(successiveFailures * 1000); try { - let messages = await this.retrieveNextMessages(url, nodeData, ourKey); + let messages = await retrieveNextMessages(address, nodeData, ourKey); successiveFailures = 0; if (messages.length) { const lastMessage = _.last(messages); nodeData.lashHash = lastMessage.hash; lokiSnodeAPI.updateLastHash( - url, + address, lastMessage.hash, lastMessage.expiration ); diff --git a/js/modules/loki_rpc.js b/js/modules/loki_rpc.js index 1e0f52975..c33f368e9 100644 --- a/js/modules/loki_rpc.js +++ b/js/modules/loki_rpc.js @@ -103,7 +103,8 @@ const fetch = async (url, options = {}) => { // Wrapper for a JSON RPC request const rpc = (address, port, method, params, options = {}) => { const headers = options.headers || {}; - const url = `${address}${port}${endpointBase}`; + const portString = port ? `:${port}` : ''; + const url = `${address}${portString}${endpointBase}`; const body = { method, params, diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 1ae318961..4f97afd7a 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -32,13 +32,13 @@ const resolveCname = url => }); class LokiSnodeAPI { - constructor({ serverUrl, localUrl, snodeServerPort }) { + constructor({ serverUrl, localUrl }) { if (!is.string(serverUrl)) { throw new Error('WebAPI.initialize: Invalid server url'); } this.serverUrl = serverUrl; this.localUrl = localUrl; - this.snodeServerPort = snodeServerPort ? `:${snodeServerPort}` : ''; + this.randomSnodePool = []; this.swarmsPendingReplenish = {}; this.ourSwarmNodes = {}; this.contactSwarmNodes = {}; @@ -60,7 +60,7 @@ class LokiSnodeAPI { } } - async getMyLokiAddress() { + getMyLokiAddress() { /* resolve our local loki address */ return resolveCname(this.localUrl); } diff --git a/main.js b/main.js index eaad6252c..455299a98 100644 --- a/main.js +++ b/main.js @@ -154,7 +154,6 @@ function prepareURL(pathSegments, moreKeys) { serverUrl: config.get('serverUrl'), localUrl: config.get('localUrl'), cdnUrl: config.get('cdnUrl'), - snodeServerPort: config.get('snodeServerPort'), localServerPort: config.get('localServerPort'), defaultPoWDifficulty: config.get('defaultPoWDifficulty'), certificateAuthority: config.get('certificateAuthority'), diff --git a/preload.js b/preload.js index 5cad3915d..55dfb9772 100644 --- a/preload.js +++ b/preload.js @@ -293,17 +293,13 @@ const LokiSnodeAPI = require('./js/modules/loki_snode_api'); window.lokiSnodeAPI = new LokiSnodeAPI({ serverUrl: config.serverUrl, localUrl: config.localUrl, - snodeServerPort: config.snodeServerPort, }); window.LokiP2pAPI = require('./js/modules/loki_p2p_api'); const LokiMessageAPI = require('./js/modules/loki_message_api'); -window.lokiMessageAPI = new LokiMessageAPI({ - url: config.serverUrl, - snodeServerPort: config.snodeServerPort, -}); +window.lokiMessageAPI = new LokiMessageAPI(); const LocalLokiServer = require('./libloki/modules/local_loki_server'); From 6b2b9ce9260ce6ff4b71c0ea57cf3c32f6161530 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Tue, 4 Jun 2019 14:02:11 +1000 Subject: [PATCH 2/6] Remove default port and use port from storage server responses --- config/default.json | 2 ++ js/modules/loki_message_api.js | 14 ++++++------- js/modules/loki_rpc.js | 10 +++++++-- js/modules/loki_snode_api.js | 37 ++++++++++++++++++++++++++-------- main.js | 2 ++ preload.js | 2 ++ 6 files changed, 50 insertions(+), 17 deletions(-) diff --git a/config/default.json b/config/default.json index a9ec560cc..9a98c6aa5 100644 --- a/config/default.json +++ b/config/default.json @@ -5,6 +5,8 @@ "contentProxyUrl": "random.snode", "localServerPort": "8081", "defaultPoWDifficulty": "100", + "seedNodeUrl": "13.238.53.205", + "seedNodePort": "22023", "disableAutoUpdate": false, "updatesUrl": "https://updates2.signal.org/desktop", "updatesPublicKey": diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index f58bb9b04..f1bbddce3 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -170,8 +170,8 @@ class LokiMessageAPI { async openSendConnection(params) { while (!_.isEmpty(this.sendingSwarmNodes[params.timestamp])) { - const url = this.sendingSwarmNodes[params.timestamp].shift(); - const successfulSend = await this.sendToNode(url, params); + const snode = this.sendingSwarmNodes[params.timestamp].shift(); + const successfulSend = await this.sendToNode(snode.address, snode.port, params); if (successfulSend) { return true; } @@ -179,14 +179,14 @@ class LokiMessageAPI { return false; } - async sendToNode(url, params) { + async sendToNode(address, port, params) { let successiveFailures = 0; while (successiveFailures < 3) { await sleepFor(successiveFailures * 500); try { const result = await rpc( - `https://${url}`, - this.snodeServerPort, + `https://${address}`, + port, 'store', params ); @@ -222,8 +222,8 @@ class LokiMessageAPI { } } } - log.error(`Failed to send to node: ${url}`); - await lokiSnodeAPI.unreachableNode(params.pubKey, url); + log.error(`Failed to send to node: ${address}`); + await lokiSnodeAPI.unreachableNode(params.pubKey, address); return false; } diff --git a/js/modules/loki_rpc.js b/js/modules/loki_rpc.js index c33f368e9..111f65411 100644 --- a/js/modules/loki_rpc.js +++ b/js/modules/loki_rpc.js @@ -5,6 +5,7 @@ const { parse } = require('url'); const LOKI_EPHEMKEY_HEADER = 'X-Loki-EphemKey'; const endpointBase = '/v1/storage_rpc'; +const seedEndpointBase = '/json_rpc'; const decryptResponse = async (response, address) => { try { @@ -101,11 +102,15 @@ const fetch = async (url, options = {}) => { }; // Wrapper for a JSON RPC request -const rpc = (address, port, method, params, options = {}) => { +const rpc = (address, port, method, params, options = {}, seedRequest = false) => { const headers = options.headers || {}; const portString = port ? `:${port}` : ''; - const url = `${address}${portString}${endpointBase}`; + const endpoint = seedRequest ? seedEndpointBase : endpointBase; + const url = `${address}${portString}${endpoint}`; + // TODO: The jsonrpc and body field will be ignored on storage server const body = { + jsonrpc: '2.0', + id: '0', method, params, }; @@ -123,6 +128,7 @@ const rpc = (address, port, method, params, options = {}) => { return fetch(url, fetchOptions); }; + module.exports = { rpc, }; diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 4f97afd7a..e7e276687 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -65,9 +65,29 @@ class LokiSnodeAPI { return resolveCname(this.localUrl); } - getRandomSnodeAddress() { + async getRandomSnodeAddress() { /* resolve random snode */ - return resolveCname(this.serverUrl); + if (this.randomSnodePool.length === 0) { + await this.initialiseRandomPool(); + } + return this.randomSnodePool[Math.floor(Math.random() * this.randomSnodePool.length)]; + } + + async initialiseRandomPool() { + const result = await rpc( + `http://${window.seedNodeUrl}`, + window.seedNodePort, + 'get_service_nodes', + {}, // Params + {}, // Options + true // Seed request + ); + const snodes = result.result.service_node_states; + this.randomSnodePool = snodes.map(snode => ({ + address: snode.public_ip, + port: snode.storage_port, + }) + ); } async unreachableNode(pubKey, nodeUrl) { @@ -121,10 +141,11 @@ class LokiSnodeAPI { async updateOurSwarmNodes(newNodes) { this.ourSwarmNodes = {}; - const ps = newNodes.map(async url => { - const lastHash = await window.Signal.Data.getLastHashBySnode(url); - this.ourSwarmNodes[url] = { + const ps = newNodes.map(async snode => { + const lastHash = await window.Signal.Data.getLastHashBySnode(snode.address); + this.ourSwarmNodes[snode.address] = { lastHash, + port: snode.port, }; }); await Promise.all(ps); @@ -167,11 +188,11 @@ class LokiSnodeAPI { async getSwarmNodes(pubKey) { // TODO: Hit multiple random nodes and merge lists? - const nodeUrl = await this.getRandomSnodeAddress(); + const { address, port } = await this.getRandomSnodeAddress(); const result = await rpc( - `https://${nodeUrl}`, - this.snodeServerPort, + `https://${address}`, + port, 'get_snodes_for_pubkey', { pubKey, diff --git a/main.js b/main.js index 455299a98..b3bd6bbd0 100644 --- a/main.js +++ b/main.js @@ -156,6 +156,8 @@ function prepareURL(pathSegments, moreKeys) { cdnUrl: config.get('cdnUrl'), localServerPort: config.get('localServerPort'), defaultPoWDifficulty: config.get('defaultPoWDifficulty'), + seedNodeUrl: config.get('seedNodeUrl'), + seedNodePort: config.get('seedNodePort'), certificateAuthority: config.get('certificateAuthority'), environment: config.environment, node_version: process.versions.node, diff --git a/preload.js b/preload.js index 55dfb9772..f163079ad 100644 --- a/preload.js +++ b/preload.js @@ -288,6 +288,8 @@ window.WebAPI = initializeWebAPI({ proxyUrl: config.proxyUrl, }); +window.seedNodeUrl = config.seedNodeUrl; +window.seedNodePort = config.seedNodePort; const LokiSnodeAPI = require('./js/modules/loki_snode_api'); window.lokiSnodeAPI = new LokiSnodeAPI({ From 1f74088586abee83bdeee88c6df13c944f210a12 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Thu, 6 Jun 2019 16:33:28 +1000 Subject: [PATCH 3/6] Send to IP --- config/default.json | 2 +- js/modules/loki_message_api.js | 3 ++- js/modules/loki_snode_api.js | 14 ++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/config/default.json b/config/default.json index 9a98c6aa5..b0b5e0987 100644 --- a/config/default.json +++ b/config/default.json @@ -5,7 +5,7 @@ "contentProxyUrl": "random.snode", "localServerPort": "8081", "defaultPoWDifficulty": "100", - "seedNodeUrl": "13.238.53.205", + "seedNodeUrl": "3.104.19.14", "seedNodePort": "22023", "disableAutoUpdate": false, "updatesUrl": "https://updates2.signal.org/desktop", diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index f1bbddce3..ef86e215d 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -171,7 +171,8 @@ class LokiMessageAPI { async openSendConnection(params) { while (!_.isEmpty(this.sendingSwarmNodes[params.timestamp])) { const snode = this.sendingSwarmNodes[params.timestamp].shift(); - const successfulSend = await this.sendToNode(snode.address, snode.port, params); + // TODO: Revert back to using snode address instead of IP + const successfulSend = await this.sendToNode(snode.ip, snode.port, params); if (successfulSend) { return true; } diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index e7e276687..17d33753c 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -84,9 +84,9 @@ class LokiSnodeAPI { ); const snodes = result.result.service_node_states; this.randomSnodePool = snodes.map(snode => ({ - address: snode.public_ip, - port: snode.storage_port, - }) + address: snode.public_ip, + port: snode.storage_port, + }) ); } @@ -108,12 +108,9 @@ class LokiSnodeAPI { async updateLastHash(nodeUrl, lastHash, expiresAt) { await window.Signal.Data.updateLastHash({ nodeUrl, lastHash, expiresAt }); if (!this.ourSwarmNodes[nodeUrl]) { - this.ourSwarmNodes[nodeUrl] = { - lastHash, - }; - } else { - this.ourSwarmNodes[nodeUrl].lastHash = lastHash; + return; } + this.ourSwarmNodes[nodeUrl].lastHash = lastHash; } getSwarmNodesForPubKey(pubKey) { @@ -146,6 +143,7 @@ class LokiSnodeAPI { this.ourSwarmNodes[snode.address] = { lastHash, port: snode.port, + ip: snode.ip, }; }); await Promise.all(ps); From b6b0248e9db5e3712355336a562fb4d3dc704747 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Fri, 7 Jun 2019 11:23:33 +1000 Subject: [PATCH 4/6] Retrieve from seed --- js/modules/loki_message_api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index ef86e215d..773dbcfa9 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -239,7 +239,8 @@ class LokiMessageAPI { await sleepFor(successiveFailures * 1000); try { - let messages = await retrieveNextMessages(address, nodeData, ourKey); + // TODO: Revert back to using snode address instead of IP + let messages = await retrieveNextMessages(nodeData.ip, nodeData, ourKey); successiveFailures = 0; if (messages.length) { const lastMessage = _.last(messages); From d5546b947390e20aa478c8766c889792244ab919 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Fri, 7 Jun 2019 12:34:54 +1000 Subject: [PATCH 5/6] Lint --- js/modules/loki_message_api.js | 21 ++++++++++++--------- js/modules/loki_rpc.js | 10 ++++++++-- js/modules/loki_snode_api.js | 11 +++++++---- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index 773dbcfa9..fabbdceb2 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -86,7 +86,7 @@ const retrieveNextMessages = async (nodeUrl, nodeData, ourKey) => { options ); return result.messages || []; -} +}; class LokiMessageAPI { constructor() { @@ -172,7 +172,11 @@ class LokiMessageAPI { while (!_.isEmpty(this.sendingSwarmNodes[params.timestamp])) { const snode = this.sendingSwarmNodes[params.timestamp].shift(); // TODO: Revert back to using snode address instead of IP - const successfulSend = await this.sendToNode(snode.ip, snode.port, params); + const successfulSend = await this.sendToNode( + snode.ip, + snode.port, + params + ); if (successfulSend) { return true; } @@ -185,12 +189,7 @@ class LokiMessageAPI { while (successiveFailures < 3) { await sleepFor(successiveFailures * 500); try { - const result = await rpc( - `https://${address}`, - port, - 'store', - params - ); + const result = await rpc(`https://${address}`, port, 'store', params); // Make sure we aren't doing too much PoW const currentDifficulty = window.storage.get('PoWDifficulty', null); @@ -240,7 +239,11 @@ class LokiMessageAPI { try { // TODO: Revert back to using snode address instead of IP - let messages = await retrieveNextMessages(nodeData.ip, nodeData, ourKey); + let messages = await retrieveNextMessages( + nodeData.ip, + nodeData, + ourKey + ); successiveFailures = 0; if (messages.length) { const lastMessage = _.last(messages); diff --git a/js/modules/loki_rpc.js b/js/modules/loki_rpc.js index 111f65411..fda6d65c0 100644 --- a/js/modules/loki_rpc.js +++ b/js/modules/loki_rpc.js @@ -102,7 +102,14 @@ const fetch = async (url, options = {}) => { }; // Wrapper for a JSON RPC request -const rpc = (address, port, method, params, options = {}, seedRequest = false) => { +const rpc = ( + address, + port, + method, + params, + options = {}, + seedRequest = false +) => { const headers = options.headers || {}; const portString = port ? `:${port}` : ''; const endpoint = seedRequest ? seedEndpointBase : endpointBase; @@ -128,7 +135,6 @@ const rpc = (address, port, method, params, options = {}, seedRequest = false) = return fetch(url, fetchOptions); }; - module.exports = { rpc, }; diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 17d33753c..a10931763 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -70,7 +70,9 @@ class LokiSnodeAPI { if (this.randomSnodePool.length === 0) { await this.initialiseRandomPool(); } - return this.randomSnodePool[Math.floor(Math.random() * this.randomSnodePool.length)]; + return this.randomSnodePool[ + Math.floor(Math.random() * this.randomSnodePool.length) + ]; } async initialiseRandomPool() { @@ -86,8 +88,7 @@ class LokiSnodeAPI { this.randomSnodePool = snodes.map(snode => ({ address: snode.public_ip, port: snode.storage_port, - }) - ); + })); } async unreachableNode(pubKey, nodeUrl) { @@ -139,7 +140,9 @@ class LokiSnodeAPI { async updateOurSwarmNodes(newNodes) { this.ourSwarmNodes = {}; const ps = newNodes.map(async snode => { - const lastHash = await window.Signal.Data.getLastHashBySnode(snode.address); + const lastHash = await window.Signal.Data.getLastHashBySnode( + snode.address + ); this.ourSwarmNodes[snode.address] = { lastHash, port: snode.port, From 0948a344d29660518348d5e1187397cafea7f2b0 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Tue, 11 Jun 2019 17:27:27 +1000 Subject: [PATCH 6/6] Review refactor --- js/modules/loki_rpc.js | 4 +--- js/modules/loki_snode_api.js | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/js/modules/loki_rpc.js b/js/modules/loki_rpc.js index fda6d65c0..4560156fb 100644 --- a/js/modules/loki_rpc.js +++ b/js/modules/loki_rpc.js @@ -5,7 +5,6 @@ const { parse } = require('url'); const LOKI_EPHEMKEY_HEADER = 'X-Loki-EphemKey'; const endpointBase = '/v1/storage_rpc'; -const seedEndpointBase = '/json_rpc'; const decryptResponse = async (response, address) => { try { @@ -108,11 +107,10 @@ const rpc = ( method, params, options = {}, - seedRequest = false + endpoint = endpointBase ) => { const headers = options.headers || {}; const portString = port ? `:${port}` : ''; - const endpoint = seedRequest ? seedEndpointBase : endpointBase; const url = `${address}${portString}${endpoint}`; // TODO: The jsonrpc and body field will be ignored on storage server const body = { diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index a10931763..fc79cd77f 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -82,7 +82,7 @@ class LokiSnodeAPI { 'get_service_nodes', {}, // Params {}, // Options - true // Seed request + '/json_rpc' // Seed request endpoint ); const snodes = result.result.service_node_states; this.randomSnodePool = snodes.map(snode => ({