From 3a746109ea89c414017dc82f733d33aeac930c1c Mon Sep 17 00:00:00 2001 From: Maxim Shishmarev Date: Mon, 23 Mar 2020 15:00:51 +1100 Subject: [PATCH] Path building for onion requests --- app/sql.js | 94 ++++++++++++++ js/background.js | 3 + js/modules/data.js | 12 ++ js/modules/loki_rpc.js | 236 ++++++++++++++++++++++++++++++++++- js/modules/loki_snode_api.js | 200 ++++++++++++++++++++++++++++- libloki/storage.js | 10 ++ preload.js | 4 +- 7 files changed, 556 insertions(+), 3 deletions(-) diff --git a/app/sql.js b/app/sql.js index 478ec44d3..6b9874510 100644 --- a/app/sql.js +++ b/app/sql.js @@ -98,6 +98,8 @@ module.exports = { getAllSessions, getSwarmNodesByPubkey, + getGuardNodes, + updateGuardNodes, getConversationCount, saveConversation, @@ -807,6 +809,7 @@ async function updateSchema(instance) { const LOKI_SCHEMA_VERSIONS = [ updateToLokiSchemaVersion1, updateToLokiSchemaVersion2, + updateToLokiSchemaVersion3, ]; async function updateToLokiSchemaVersion1(currentVersion, instance) { @@ -975,6 +978,34 @@ async function updateToLokiSchemaVersion2(currentVersion, instance) { console.log('updateToLokiSchemaVersion2: success!'); } +async function updateToLokiSchemaVersion3(currentVersion, instance) { + if (currentVersion >= 3) { + return; + } + + await instance.run( + `CREATE TABLE ${GUARD_NODE_TABLE}( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + ed25519PubKey VARCHAR(64) + );` + ); + + console.log('updateToLokiSchemaVersion3: starting...'); + await instance.run('BEGIN TRANSACTION;'); + + + await instance.run( + `INSERT INTO loki_schema ( + version + ) values ( + 3 + );` + ); + + await instance.run('COMMIT TRANSACTION;'); + console.log('updateToLokiSchemaVersion3: success!'); +} + async function updateLokiSchema(instance) { const result = await instance.get( "SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';" @@ -1400,6 +1431,9 @@ async function removeAllSignedPreKeys() { } const PAIRING_AUTHORISATIONS_TABLE = 'pairingAuthorisations'; + +const GUARD_NODE_TABLE = 'guardNodes'; + async function getAuthorisationForSecondaryPubKey(pubKey, options) { const granted = options && options.granted; let filter = ''; @@ -1470,6 +1504,66 @@ async function getSecondaryDevicesFor(primaryDevicePubKey) { return map(authorisations, row => row.secondaryDevicePubKey); } +async function getGuardNodes() { + + const nodes = await db.all(`SELECT ed25519PubKey FROM ${GUARD_NODE_TABLE};`); + + if (!nodes) { + return null; + } + + return nodes; + +} + +async function createOrUpdatePreKey(data) { + const { id, recipient } = data; + if (!id) { + throw new Error('createOrUpdate: Provided data did not have a truthy id'); + } + + await db.run( + `INSERT OR REPLACE INTO ${PRE_KEYS_TABLE} ( + id, + recipient, + json + ) values ( + $id, + $recipient, + $json + )`, + { + $id: id, + $recipient: recipient || '', + $json: objectToJSON(data), + } + ); +} + +async function updateGuardNodes(nodes) { + + await db.run('BEGIN TRANSACTION;'); + + await db.run(`DELETE FROM ${GUARD_NODE_TABLE}`); + + await Promise.all(nodes.map(edkey => + + db.run( + `INSERT INTO ${GUARD_NODE_TABLE} ( + ed25519PubKey + ) values ($ed25519PubKey)`, + { + $ed25519PubKey: edkey, + } + ) + + )); + + await db.run('END TRANSACTION;'); + + +} + async function getPrimaryDeviceFor(secondaryDevicePubKey) { const row = await db.get( `SELECT primaryDevicePubKey FROM ${PAIRING_AUTHORISATIONS_TABLE} WHERE secondaryDevicePubKey = $secondaryDevicePubKey AND isGranted = 1;`, diff --git a/js/background.js b/js/background.js index 6d52f87c3..36db9df31 100644 --- a/js/background.js +++ b/js/background.js @@ -1428,6 +1428,9 @@ async function connect(firstRun) { window.log.info('connect'); + // Initialize paths for onion requests + await window.lokiSnodeAPI.buildNewOnionPaths(); + // Bootstrap our online/offline detection, only the first time we connect if (connectCount === 0 && navigator.onLine) { window.addEventListener('offline', onOffline); diff --git a/js/modules/data.js b/js/modules/data.js index 729f8c60e..5d97acd9c 100644 --- a/js/modules/data.js +++ b/js/modules/data.js @@ -101,6 +101,9 @@ module.exports = { getPrimaryDeviceFor, getPairedDevicesFor, + getGuardNodes, + updateGuardNodes, + createOrUpdateItem, getItemById, getAllItems, @@ -117,6 +120,7 @@ module.exports = { removeAllSessions, getAllSessions, + // Doesn't look like this is used at all getSwarmNodesByPubkey, getConversationCount, @@ -647,6 +651,14 @@ function getSecondaryDevicesFor(primaryDevicePubKey) { return channels.getSecondaryDevicesFor(primaryDevicePubKey); } +function getGuardNodes() { + return channels.getGuardNodes(); +} + +function updateGuardNodes(nodes) { + return channels.updateGuardNodes(nodes); +} + function getPrimaryDeviceFor(secondaryDevicePubKey) { return channels.getPrimaryDeviceFor(secondaryDevicePubKey); } diff --git a/js/modules/loki_rpc.js b/js/modules/loki_rpc.js index cb49b975b..13f661b2c 100644 --- a/js/modules/loki_rpc.js +++ b/js/modules/loki_rpc.js @@ -12,6 +12,10 @@ const snodeHttpsAgent = new https.Agent({ const LOKI_EPHEMKEY_HEADER = 'X-Loki-EphemKey'; const endpointBase = '/storage_rpc/v1'; + +// Request index for debugging +let onion_req_idx = 0; + const decryptResponse = async (response, address) => { let plaintext = false; try { @@ -31,8 +35,209 @@ const decryptResponse = async (response, address) => { const timeoutDelay = ms => new Promise(resolve => setTimeout(resolve, ms)); +const encryptForNode = async (node, payload) => { + + const textEncoder = new TextEncoder(); + const plaintext = textEncoder.encode(payload); + + const ephemeral = libloki.crypto.generateEphemeralKeyPair(); + + const snPubkey = StringView.hexToArrayBuffer(node.pubkey_x25519); + + const ephemeralSecret = libsignal.Curve.calculateAgreement( + snPubkey, + ephemeral.privKey + ); + + const salt = window.Signal.Crypto.bytesFromString("LOKI"); + + let key = await crypto.subtle.importKey('raw', salt, {name: 'HMAC', hash: {name: 'SHA-256'}}, false, ['sign']); + let symmetricKey = await crypto.subtle.sign( {name: 'HMAC', hash: 'SHA-256'}, key, ephemeralSecret); + + const ciphertext = await window.libloki.crypto.EncryptGCM( + symmetricKey, + plaintext + ); + + return {ciphertext, symmetricKey, "ephemeral_key": ephemeral.pubKey}; + +} + +// Returns the actual ciphertext, symmetric key that will be used +// for decryption, and an ephemeral_key to send to the next hop +const encryptForDestination = async (node, payload) => { + // Do we still need "headers"? + const req_str = JSON.stringify({"body": payload, "headers": ""}); + + return await encryptForNode(node, req_str); +} + +// `ctx` holds info used by `node` to relay further +const encryptForRelay = async (node, next_node, ctx) => { + + const payload = ctx.ciphertext; + + const req_json = { + "ciphertext": dcodeIO.ByteBuffer.wrap(payload).toString('base64'), + "ephemeral_key": StringView.arrayBufferToHex(ctx.ephemeral_key), + "destination": next_node.pubkey_ed25519, + } + + const req_str = JSON.stringify(req_json); + + return await encryptForNode(node, req_str); + +} + +const BAD_PATH = "bad_path"; + +// May return false BAD_PATH, indicating that we should try a new +const sendOnionRequest = async (req_idx, nodePath, targetNode, plaintext) => { + + log.info("Sending an onion request"); + + let ctx_1 = await encryptForDestination(targetNode, plaintext); + let ctx_2 = await encryptForRelay(nodePath[2], targetNode, ctx_1); + let ctx_3 = await encryptForRelay(nodePath[1], nodePath[2], ctx_2); + let ctx_4 = await encryptForRelay(nodePath[0], nodePath[1], ctx_3); + + const ciphertext_base64 = dcodeIO.ByteBuffer.wrap(ctx_4.ciphertext).toString('base64'); + + const payload = { + "ciphertext": ciphertext_base64, + "ephemeral_key": StringView.arrayBufferToHex(ctx_4.ephemeral_key), + } + + const fetchOptions = { + method: 'POST', + body: JSON.stringify(payload), + }; + + const url = `https://${nodePath[0].ip}:${nodePath[0].port}/onion_req`; + + // we only proxy to snodes... + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; + const response = await nodeFetch(url, fetchOptions); + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1; + + return await processOnionResponse(req_idx, response, ctx_1.symmetricKey, true); +} + +// Process a response as it arrives from `nodeFetch`, handling +// http errors and attempting to decrypt the body with `shared_key` +const processOnionResponse = async (req_idx, response, shared_key, use_aes_gcm) => { + + console.log(`(${req_idx}) [path] processing onion response`); + + // detect SNode is not ready (not in swarm; not done syncing) + if (response.status === 503) { + console.warn("Got 503: snode not ready"); + + return BAD_PATH; + } + + if (response.status == 504) { + log.warn('Got 504: Gateway timeout'); + return BAD_PATH; + } + + if (response.status == 404) { + // Why would we get this error on testnet? + log.warn('Got 404: Gateway timeout'); + return BAD_PATH; + } + + if (response.status !== 200) { + log.warn('lokiRpc sendToProxy fetch unhandled error code:', response.status); + return; + } + + const ciphertext = await response.text(); + if (!ciphertext) { + log.warn("[path]: Target node return empty ciphertext"); + return; + } + + let plaintext; + let ciphertextBuffer; + try { + ciphertextBuffer = dcodeIO.ByteBuffer.wrap( + ciphertext, + 'base64' + ).toArrayBuffer(); + + const decrypt_fn = use_aes_gcm ? window.libloki.crypto.DecryptGCM : window.libloki.crypto.DHDecrypt; + + const plaintextBuffer = await decrypt_fn( + shared_key, + ciphertextBuffer + ); + + const textDecoder = new TextDecoder(); + plaintext = textDecoder.decode(plaintextBuffer); + } catch(e) { + log.error( + 'lokiRpc sendToProxy decode error', + e.code, + e.message, + `from ${randSnode.ip}:${randSnode.port} ciphertext:`, + ciphertext + ); + if (ciphertextBuffer) { + log.error('ciphertextBuffer', ciphertextBuffer); + } + return; + } + + try { + const jsonRes = JSON.parse(plaintext); + // emulate nodeFetch response... + jsonRes.json = () => { + try { + let res = JSON.parse(jsonRes.body); + return res; + } catch (e) { + log.error( + 'lokiRpc sendToProxy parse error', + e.code, + e.message, + `from ${randSnode.ip}:${randSnode.port} json:`, + jsonRes.body + ); + } + return false; + }; + return jsonRes; + } catch (e) { + log.error( + 'lokiRpc sendToProxy parse error', + e.code, + e.message, + `from ${randSnode.ip}:${randSnode.port} json:`, + plaintext + ); + + return; + } + +} + const sendToProxy = async (options = {}, targetNode, retryNumber = 0) => { - const randSnode = await lokiSnodeAPI.getRandomSnodeAddress(); + + let snodePool = await lokiSnodeAPI.getRandomSnodePool(); + + if (snodePool.length < 2) { + console.error("Not enough service nodes for a proxy request, only have: ", snodePool.length); + return; + } + + // Making sure the proxy node is not the same as the target node: + const snodePoolSafe = _.without( + snodePool, + _.find(snodePool, { pubkey_ed25519: targetNode.pubkey_ed25519 }) + ); + + const randSnode = window.Lodash.sample(snodePoolSafe); // Don't allow arbitrary URLs, only snodes and loki servers const url = `https://${randSnode.ip}:${randSnode.port}/proxy`; @@ -262,6 +467,34 @@ const lokiFetch = async (url, options = {}, targetNode = null) => { }; try { + + // Absence of targetNode indicates that we want a direct connection + // (e.g. to connect to a seed node for the first time) + if (window.lokiFeatureFlags.useOnionRequests && targetNode) { + + // Loop until the result is not BAD_PATH + while (true) { + + // Get a path excluding `targetNode`: + const path = await lokiSnodeAPI.getOnionPath(targetNode); + const this_idx = onion_req_idx++; + + log.info(`(${this_idx}) using path ${path[0].ip}:${path[0].port} -> ${path[1].ip}:${path[1].port} -> ${path[2].ip}:${path[2].port} => ${targetNode.ip}:${targetNode.port}`); + + const result = await sendOnionRequest(this_idx, path, targetNode, fetchOptions.body); + + if (result == BAD_PATH) { + log.error("[path] Error on the path"); + lokiSnodeAPI.markPathAsBad(path); + } else { + return result ? result.json() : false; + } + + } + + } + + if (window.lokiFeatureFlags.useSnodeProxy && targetNode) { const result = await sendToProxy(fetchOptions, targetNode); // if not result, maybe we should throw?? @@ -332,6 +565,7 @@ const lokiFetch = async (url, options = {}, targetNode = null) => { }; // Wrapper for a JSON RPC request +// Annoyngly, this is used for Lokid requests too const lokiRpc = ( address, port, diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 46b18b9ee..4b40dedf9 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -1,8 +1,9 @@ /* eslint-disable class-methods-use-this */ -/* global window, ConversationController, _, log, clearTimeout */ +/* global window, textsecure, ConversationController, _, log, clearTimeout */ const is = require('@sindresorhus/is'); const { lokiRpc } = require('./loki_rpc'); +const nodeFetch = require('node-fetch'); const RANDOM_SNODES_TO_USE_FOR_PUBKEY_SWARM = 3; const RANDOM_SNODES_POOL_SIZE = 1024; @@ -18,6 +19,203 @@ class LokiSnodeAPI { this.randomSnodePool = []; this.swarmsPendingReplenish = {}; this.refreshRandomPoolPromise = false; + + this.onionPaths = []; + this.guardNodes = []; + } + + async getRandomSnodePool() { + + if (this.randomSnodePool.length === 0) { + await this.refreshRandomPool(); + } + return this.randomSnodePool; + } + + async test_guard_node(snode) { + + log.info("Testing a candidate guard node ", snode); + + // Send a post request and make sure it is OK + const endpoint = "/storage_rpc/v1"; + + const url = `https://${snode.ip}:${snode.port}${endpoint}`; + + const our_pk = textsecure.storage.user.getNumber(); + const pubKey = window.getStoragePubKey(our_pk); // truncate if testnet + + const method = 'get_snodes_for_pubkey'; + const params = { pubKey } + const body = { + jsonrpc: '2.0', + id: '0', + method, + params, + }; + + const fetchOptions = { + method: 'POST', + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + timeout: 1000 // 1s, we want a small timeout for testing + }; + + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; + const response = await nodeFetch(url, fetchOptions); + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1; + + if (!response.ok) { + log.log(`Node ${snode} failed the guard test`); + } + + return response.ok; + } + + async selectGuardNodes() { + + const _ = window.Lodash; + + let node_pool = await this.getRandomSnodePool(); + + if (node_pool.length === 0) { + log.error(`Could not select guarn nodes: node pool is empty`) + return []; + } + + let shuffled = _.shuffle(node_pool); + + let guard_nodes = []; + + const DESIRED_GUARD_COUNT = 3; + + while (guard_nodes.length < 3) { + + if (shuffled.length < DESIRED_GUARD_COUNT) { + log.error(`Not enought nodes in the pool`); + break; + } + + const candidate_nodes = shuffled.splice(0, DESIRED_GUARD_COUNT); + + // Test all three nodes at once + const idx_ok = await Promise.all(candidate_nodes.map(n => this.test_guard_node(n))); + + const good_nodes = _.zip(idx_ok, candidate_nodes).filter(x => x[0]).map(x => x[1]); + + guard_nodes = _.concat(guard_nodes, good_nodes); + } + + if (guard_nodes.length < DESIRED_GUARD_COUNT) { + log.error(`COULD NOT get enough guard nodes, only have: ${guard_nodes.length}`); + debugger; + } + + console.log("new guard nodes: ", guard_nodes); + + const edKeys = guard_nodes.map(n => n.pubkey_ed25519); + + await window.libloki.storage.updateGuardNodes(edKeys); + + return guard_nodes; + } + + async getOnionPath(toExclude = null) { + + const _ = window.Lodash; + + const good_paths = this.onionPaths.filter(x => !x.bad); + + if (good_paths.length < 2) { + log.error(`Must have at least 2 good onion paths, actual: ${good_paths.length}`); + await this.buildNewOnionPaths(); + } + + const paths = _.shuffle(good_paths); + + if (!toExclude) { + return paths[0]; + } + + // Select a path that doesn't contain `toExclude` + const other_paths = paths.filter(path => !_.some(path, node => node.pubkey_ed25519 == toExclude.pubkey_ed25519)); + + if (other_paths.length === 0) { + // This should never happen! + log.error("No onion paths available after filtering"); + } + + return other_paths[0].path; + } + + async markPathAsBad(path) { + this.onionPaths.forEach(p => { + if (p.path == path) { + p.bad = true; + } + }) + } + + async buildNewOnionPaths() { + + // Note: this function may be called concurrently, so + // might consider blocking the other calls + + const _ = window.Lodash; + + log.info("building new onion paths"); + + const all_nodes = await this.getRandomSnodePool(); + + if (this.guardNodes.length == 0) { + + // Not cached, load from DB + let nodes = await window.libloki.storage.getGuardNodes(); + + if (nodes.length == 0) { + log.warn("no guard nodes in DB. Will be selecting new guards nodes..."); + } else { + // We only store the nodes' keys, need to find full entries: + let ed_keys = nodes.map(x => x.ed25519PubKey); + this.guardNodes = all_nodes.filter(x => ed_keys.indexOf(x.pubkey_ed25519) !== -1); + + if (this.guardNodes.length < ed_keys.length) { + log.warn(`could not find some guard nodes: ${this.guardNodes.length}/${ed_keys.length}`); + } + + } + + // If guard nodes is still empty (the old nodes are now invalid), select new ones: + if (this.guardNodes.length == 0 || true) { + this.guardNodes = await this.selectGuardNodes(); + } + + } + + // TODO: select one guard node and 2 other nodes randomly + let other_nodes = _.difference(all_nodes, this.guardNodes); + + if (other_nodes.length < 2) { + log.error("Too few nodes to build an onion path!"); + return; + } + + other_nodes = _.shuffle(other_nodes); + const guards = _.shuffle(this.guardNodes); + + // Create path for every guard node: + + // Each path needs 2 nodes in addition to the guard node: + const max_path = Math.floor(Math.min(guards.length, other_nodes.length / 2)); + + // TODO: might want to keep some of the existing paths + this.onionPaths = []; + + for (let i = 0; i < max_path; i++) { + const path = [guards[i], other_nodes[i * 2], other_nodes[i * 2 + 1]]; + this.onionPaths.push({path, bad: false}); + } + + log.info("Built onion paths: ", this.onionPaths); } async getRandomSnodeAddress() { diff --git a/libloki/storage.js b/libloki/storage.js index ff52e52c5..6c5666a14 100644 --- a/libloki/storage.js +++ b/libloki/storage.js @@ -240,6 +240,14 @@ return window.Signal.Data.getSecondaryDevicesFor(primaryDevicePubKey); } + function getGuardNodes() { + return window.Signal.Data.getGuardNodes(); + } + + function updateGuardNodes(nodes) { + return window.Signal.Data.updateGuardNodes(nodes); + } + async function getAllDevicePubKeysForPrimaryPubKey(primaryDevicePubKey) { await saveAllPairingAuthorisationsFor(primaryDevicePubKey); const secondaryPubKeys = @@ -265,6 +273,8 @@ getAllDevicePubKeysForPrimaryPubKey, getSecondaryDevicesFor, getPrimaryDeviceMapping, + getGuardNodes, + updateGuardNodes }; // Libloki protocol store diff --git a/preload.js b/preload.js index 94fa47338..ebe87e1e3 100644 --- a/preload.js +++ b/preload.js @@ -411,6 +411,7 @@ window.lokiFeatureFlags = { privateGroupChats: true, useSnodeProxy: true, useSealedSender: true, + useOnionRequests: true, }; // eslint-disable-next-line no-extend-native,func-names @@ -419,7 +420,7 @@ Promise.prototype.ignore = function() { this.then(() => {}); }; -if (config.environment.includes('test')) { +if (config.environment.includes('test') && !config.environment == "swarm-testing1" && !config.environment == "swarm-testing2") { const isWindows = process.platform === 'win32'; /* eslint-disable global-require, import/no-extraneous-dependencies */ window.test = { @@ -439,5 +440,6 @@ if (config.environment.includes('test')) { updateSwarmNodes: () => {}, updateLastHash: () => {}, getSwarmNodesForPubKey: () => [], + buildNewOnionPaths: () => [], }; }