Path building for onion requests

app/sql.js

@@ -98,6 +98,8 @@ module.exports = {
   getAllSessions,
 
   getSwarmNodesByPubkey,
+  getGuardNodes,
+  updateGuardNodes,
 
   getConversationCount,
   saveConversation,
@@ -807,6 +809,7 @@ async function updateSchema(instance) {
 const LOKI_SCHEMA_VERSIONS = [
   updateToLokiSchemaVersion1,
   updateToLokiSchemaVersion2,
+  updateToLokiSchemaVersion3,
 ];
 
 async function updateToLokiSchemaVersion1(currentVersion, instance) {
@@ -975,6 +978,34 @@ async function updateToLokiSchemaVersion2(currentVersion, instance) {
   console.log('updateToLokiSchemaVersion2: success!');
 }
 
+async function updateToLokiSchemaVersion3(currentVersion, instance) {
+  if (currentVersion >= 3) {
+    return;
+  }
+
+  await instance.run(
+    `CREATE TABLE ${GUARD_NODE_TABLE}(
+      id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+      ed25519PubKey VARCHAR(64)
+    );`
+  );
+
+  console.log('updateToLokiSchemaVersion3: starting...');
+  await instance.run('BEGIN TRANSACTION;');
+
+
+  await instance.run(
+    `INSERT INTO loki_schema (
+        version
+      ) values (
+        3
+      );`
+  );
+
+  await instance.run('COMMIT TRANSACTION;');
+  console.log('updateToLokiSchemaVersion3: success!');
+}
+
 async function updateLokiSchema(instance) {
   const result = await instance.get(
     "SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';"
@@ -1400,6 +1431,9 @@ async function removeAllSignedPreKeys() {
 }
 
 const PAIRING_AUTHORISATIONS_TABLE = 'pairingAuthorisations';
+
+const GUARD_NODE_TABLE = 'guardNodes';
+
 async function getAuthorisationForSecondaryPubKey(pubKey, options) {
   const granted = options && options.granted;
   let filter = '';
@@ -1470,6 +1504,66 @@ async function getSecondaryDevicesFor(primaryDevicePubKey) {
   return map(authorisations, row => row.secondaryDevicePubKey);
 }
 
+async function getGuardNodes() {
+
+  const nodes = await db.all(`SELECT ed25519PubKey FROM ${GUARD_NODE_TABLE};`);
+
+  if (!nodes) {
+    return null;
+  }
+
+  return nodes;
+
+}
+
+async function createOrUpdatePreKey(data) {
+  const { id, recipient } = data;
+  if (!id) {
+    throw new Error('createOrUpdate: Provided data did not have a truthy id');
+  }
+
+  await db.run(
+    `INSERT OR REPLACE INTO ${PRE_KEYS_TABLE} (
+      id,
+      recipient,
+      json
+    ) values (
+      $id,
+      $recipient,
+      $json
+    )`,
+    {
+      $id: id,
+      $recipient: recipient || '',
+      $json: objectToJSON(data),
+    }
+  );
+}
+
+async function updateGuardNodes(nodes) {
+
+  await db.run('BEGIN TRANSACTION;');
+
+  await db.run(`DELETE FROM ${GUARD_NODE_TABLE}`);
+
+  await Promise.all(nodes.map(edkey => 
+
+    db.run(
+      `INSERT INTO ${GUARD_NODE_TABLE} (
+        ed25519PubKey
+      ) values ($ed25519PubKey)`,
+      {
+        $ed25519PubKey: edkey,
+      }
+    )
+
+  ));
+
+  await db.run('END TRANSACTION;');
+
+
+}
+
 async function getPrimaryDeviceFor(secondaryDevicePubKey) {
   const row = await db.get(
     `SELECT primaryDevicePubKey FROM ${PAIRING_AUTHORISATIONS_TABLE} WHERE secondaryDevicePubKey = $secondaryDevicePubKey AND isGranted = 1;`,

js/background.js

@@ -1428,6 +1428,9 @@
   async function connect(firstRun) {
     window.log.info('connect');
 
+    // Initialize paths for onion requests
+    await window.lokiSnodeAPI.buildNewOnionPaths();
+
     // Bootstrap our online/offline detection, only the first time we connect
     if (connectCount === 0 && navigator.onLine) {
       window.addEventListener('offline', onOffline);

js/modules/data.js

@@ -101,6 +101,9 @@ module.exports = {
   getPrimaryDeviceFor,
   getPairedDevicesFor,
 
+  getGuardNodes,
+  updateGuardNodes,
+
   createOrUpdateItem,
   getItemById,
   getAllItems,
@@ -117,6 +120,7 @@ module.exports = {
   removeAllSessions,
   getAllSessions,
 
+  // Doesn't look like this is used at all
   getSwarmNodesByPubkey,
 
   getConversationCount,
@@ -647,6 +651,14 @@ function getSecondaryDevicesFor(primaryDevicePubKey) {
   return channels.getSecondaryDevicesFor(primaryDevicePubKey);
 }
 
+function getGuardNodes() {
+  return channels.getGuardNodes();
+}
+
+function updateGuardNodes(nodes) {
+  return channels.updateGuardNodes(nodes);
+}
+
 function getPrimaryDeviceFor(secondaryDevicePubKey) {
   return channels.getPrimaryDeviceFor(secondaryDevicePubKey);
 }

js/modules/loki_rpc.js

@@ -12,6 +12,10 @@ const snodeHttpsAgent = new https.Agent({
 const LOKI_EPHEMKEY_HEADER = 'X-Loki-EphemKey';
 const endpointBase = '/storage_rpc/v1';
 
+
+// Request index for debugging
+let onion_req_idx = 0;
+
 const decryptResponse = async (response, address) => {
   let plaintext = false;
   try {
@@ -31,8 +35,209 @@ const decryptResponse = async (response, address) => {
 
 const timeoutDelay = ms => new Promise(resolve => setTimeout(resolve, ms));
 
+const encryptForNode = async (node, payload) => {
+
+  const textEncoder = new TextEncoder();
+  const plaintext = textEncoder.encode(payload);
+
+  const ephemeral = libloki.crypto.generateEphemeralKeyPair();
+
+  const snPubkey = StringView.hexToArrayBuffer(node.pubkey_x25519);
+
+  const ephemeralSecret = libsignal.Curve.calculateAgreement(
+    snPubkey,
+    ephemeral.privKey
+  );
+
+  const salt = window.Signal.Crypto.bytesFromString("LOKI");
+
+  let key = await crypto.subtle.importKey('raw', salt, {name: 'HMAC', hash: {name: 'SHA-256'}}, false, ['sign']);
+  let symmetricKey = await crypto.subtle.sign( {name: 'HMAC', hash: 'SHA-256'}, key, ephemeralSecret);
+
+  const ciphertext = await window.libloki.crypto.EncryptGCM(
+    symmetricKey,
+    plaintext
+  );
+
+  return {ciphertext, symmetricKey, "ephemeral_key": ephemeral.pubKey};
+
+}
+
+// Returns the actual ciphertext, symmetric key that will be used
+// for decryption, and an ephemeral_key to send to the next hop
+const encryptForDestination = async (node, payload) => {
+  // Do we still need "headers"?
+  const req_str = JSON.stringify({"body": payload, "headers": ""});
+
+  return await encryptForNode(node, req_str);
+}
+
+// `ctx` holds info used by `node` to relay further
+const encryptForRelay = async (node, next_node, ctx) => {
+
+  const payload = ctx.ciphertext;
+
+  const req_json = {
+    "ciphertext": dcodeIO.ByteBuffer.wrap(payload).toString('base64'),
+    "ephemeral_key": StringView.arrayBufferToHex(ctx.ephemeral_key),
+    "destination": next_node.pubkey_ed25519,
+  }
+
+  const req_str = JSON.stringify(req_json);
+
+  return await encryptForNode(node, req_str);
+
+}
+
+const BAD_PATH = "bad_path";
+
+// May return false BAD_PATH, indicating that we should try a new 
+const sendOnionRequest = async (req_idx, nodePath, targetNode, plaintext) => {
+
+  log.info("Sending an onion request");
+
+  let ctx_1 = await encryptForDestination(targetNode, plaintext);
+  let ctx_2 = await encryptForRelay(nodePath[2], targetNode, ctx_1);
+  let ctx_3 = await encryptForRelay(nodePath[1], nodePath[2], ctx_2);
+  let ctx_4 = await encryptForRelay(nodePath[0], nodePath[1], ctx_3);
+
+  const ciphertext_base64 = dcodeIO.ByteBuffer.wrap(ctx_4.ciphertext).toString('base64');
+
+  const payload = {
+    "ciphertext": ciphertext_base64,
+    "ephemeral_key": StringView.arrayBufferToHex(ctx_4.ephemeral_key),
+  }
+
+  const fetchOptions = {
+    method: 'POST',
+    body: JSON.stringify(payload),
+  };
+
+  const url = `https://${nodePath[0].ip}:${nodePath[0].port}/onion_req`;
+
+  // we only proxy to snodes...
+  process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
+  const response = await nodeFetch(url, fetchOptions);
+  process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1;
+
+  return await processOnionResponse(req_idx, response, ctx_1.symmetricKey, true);
+}
+
+// Process a response as it arrives from `nodeFetch`, handling
+// http errors and attempting to decrypt the body with `shared_key`
+const processOnionResponse = async (req_idx, response, shared_key, use_aes_gcm) => {
+
+  console.log(`(${req_idx}) [path] processing onion response`);
+
+  // detect SNode is not ready (not in swarm; not done syncing)
+  if (response.status === 503) {
+    console.warn("Got 503: snode not ready");
+
+    return BAD_PATH;
+  }
+
+  if (response.status == 504) {
+    log.warn('Got 504: Gateway timeout');
+    return BAD_PATH;
+  }
+
+  if (response.status == 404) {
+    // Why would we get this error on testnet?
+    log.warn('Got 404: Gateway timeout');
+    return BAD_PATH;
+  }
+
+  if (response.status !== 200) {
+    log.warn('lokiRpc sendToProxy fetch unhandled error code:', response.status);
+    return;
+  }
+  
+  const ciphertext = await response.text();
+  if (!ciphertext) {
+    log.warn("[path]: Target node return empty ciphertext");
+    return;
+  }
+
+  let plaintext;
+  let ciphertextBuffer;
+  try {
+    ciphertextBuffer = dcodeIO.ByteBuffer.wrap(
+      ciphertext,
+      'base64'
+    ).toArrayBuffer();
+
+    const decrypt_fn = use_aes_gcm ? window.libloki.crypto.DecryptGCM : window.libloki.crypto.DHDecrypt;
+
+    const plaintextBuffer = await decrypt_fn(
+      shared_key,
+      ciphertextBuffer
+    );
+
+    const textDecoder = new TextDecoder();
+    plaintext = textDecoder.decode(plaintextBuffer);
+  } catch(e) {
+    log.error(
+      'lokiRpc sendToProxy decode error',
+      e.code,
+      e.message,
+      `from ${randSnode.ip}:${randSnode.port} ciphertext:`,
+      ciphertext
+    );
+    if (ciphertextBuffer) {
+      log.error('ciphertextBuffer', ciphertextBuffer);
+    }
+    return;
+  }
+
+  try {
+    const jsonRes = JSON.parse(plaintext);
+    // emulate nodeFetch response...
+    jsonRes.json = () => {
+      try {
+        let res = JSON.parse(jsonRes.body);
+        return res;
+      } catch (e) {
+        log.error(
+          'lokiRpc sendToProxy parse error',
+          e.code,
+          e.message,
+          `from ${randSnode.ip}:${randSnode.port} json:`,
+          jsonRes.body
+        );
+      }
+      return false;
+    };
+    return jsonRes;
+  } catch (e) {
+    log.error(
+      'lokiRpc sendToProxy parse error',
+      e.code,
+      e.message,
+      `from ${randSnode.ip}:${randSnode.port} json:`,
+      plaintext
+    );
+
+    return;
+  }
+
+}
+
 const sendToProxy = async (options = {}, targetNode, retryNumber = 0) => {
-  const randSnode = await lokiSnodeAPI.getRandomSnodeAddress();
+
+  let snodePool = await lokiSnodeAPI.getRandomSnodePool();
+
+  if (snodePool.length < 2) {
+    console.error("Not enough service nodes for a proxy request, only have: ", snodePool.length);
+    return;
+  }
+
+  // Making sure the proxy node is not the same as the target node:
+  const snodePoolSafe = _.without(
+    snodePool,
+    _.find(snodePool, { pubkey_ed25519: targetNode.pubkey_ed25519 })
+  );
+
+  const randSnode = window.Lodash.sample(snodePoolSafe);
 
   // Don't allow arbitrary URLs, only snodes and loki servers
   const url = `https://${randSnode.ip}:${randSnode.port}/proxy`;
@@ -262,6 +467,34 @@ const lokiFetch = async (url, options = {}, targetNode = null) => {
   };
 
   try {
+
+    // Absence of targetNode indicates that we want a direct connection
+    // (e.g. to connect to a seed node for the first time)
+    if (window.lokiFeatureFlags.useOnionRequests && targetNode) {
+
+      // Loop until the result is not BAD_PATH
+      while (true) {
+
+        // Get a path excluding `targetNode`:
+        const path = await lokiSnodeAPI.getOnionPath(targetNode);
+        const this_idx = onion_req_idx++;
+  
+        log.info(`(${this_idx}) using path ${path[0].ip}:${path[0].port} -> ${path[1].ip}:${path[1].port} -> ${path[2].ip}:${path[2].port} => ${targetNode.ip}:${targetNode.port}`);
+  
+        const result = await sendOnionRequest(this_idx, path, targetNode, fetchOptions.body);
+  
+        if (result == BAD_PATH) {
+          log.error("[path] Error on the path");
+          lokiSnodeAPI.markPathAsBad(path);
+        } else {
+          return result ? result.json() : false;
+        }
+
+      }
+
+    }
+
+
     if (window.lokiFeatureFlags.useSnodeProxy && targetNode) {
       const result = await sendToProxy(fetchOptions, targetNode);
       // if not result, maybe we should throw??
@@ -332,6 +565,7 @@ const lokiFetch = async (url, options = {}, targetNode = null) => {
 };
 
 // Wrapper for a JSON RPC request
+// Annoyngly, this is used for Lokid requests too
 const lokiRpc = (
   address,
   port,

js/modules/loki_snode_api.js

@@ -1,8 +1,9 @@
 /* eslint-disable class-methods-use-this */
-/* global window, ConversationController, _, log, clearTimeout */
+/* global window, textsecure, ConversationController, _, log, clearTimeout */
 
 const is = require('@sindresorhus/is');
 const { lokiRpc } = require('./loki_rpc');
+const nodeFetch = require('node-fetch');
 
 const RANDOM_SNODES_TO_USE_FOR_PUBKEY_SWARM = 3;
 const RANDOM_SNODES_POOL_SIZE = 1024;
@@ -18,6 +19,203 @@ class LokiSnodeAPI {
     this.randomSnodePool = [];
     this.swarmsPendingReplenish = {};
     this.refreshRandomPoolPromise = false;
+
+    this.onionPaths = [];
+    this.guardNodes = [];
+  }
+
+  async getRandomSnodePool() {
+
+    if (this.randomSnodePool.length === 0) {
+      await this.refreshRandomPool();
+    }
+    return this.randomSnodePool;
+  }
+
+  async test_guard_node(snode) {
+
+    log.info("Testing a candidate guard node ", snode);
+
+    // Send a post request and make sure it is OK
+    const endpoint = "/storage_rpc/v1";
+
+    const url = `https://${snode.ip}:${snode.port}${endpoint}`;
+
+    const our_pk = textsecure.storage.user.getNumber();
+    const pubKey = window.getStoragePubKey(our_pk); // truncate if testnet
+
+    const method = 'get_snodes_for_pubkey';
+    const params = { pubKey }
+    const body = {
+      jsonrpc: '2.0',
+      id: '0',
+      method,
+      params,
+    };
+
+    const fetchOptions = {
+      method: 'POST',
+      body:    JSON.stringify(body),
+      headers: { 'Content-Type': 'application/json' },
+      timeout: 1000 // 1s, we want a small timeout for testing
+    };
+
+    process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
+    const response = await nodeFetch(url, fetchOptions);
+    process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1;
+
+    if (!response.ok) {
+      log.log(`Node ${snode} failed the guard test`);
+    }
+
+    return response.ok;
+  }
+
+  async selectGuardNodes() {
+
+    const _ = window.Lodash;
+
+    let node_pool = await this.getRandomSnodePool();
+
+    if (node_pool.length === 0) {
+      log.error(`Could not select guarn nodes: node pool is empty`)
+      return [];
+    }
+
+    let shuffled = _.shuffle(node_pool);
+
+    let guard_nodes = [];
+
+    const DESIRED_GUARD_COUNT = 3;
+
+    while (guard_nodes.length < 3) {
+
+      if (shuffled.length < DESIRED_GUARD_COUNT) {
+        log.error(`Not enought nodes in the pool`);
+        break;
+      }
+
+      const candidate_nodes = shuffled.splice(0, DESIRED_GUARD_COUNT);
+
+      // Test all three nodes at once
+      const idx_ok = await Promise.all(candidate_nodes.map(n => this.test_guard_node(n)));
+
+      const good_nodes = _.zip(idx_ok, candidate_nodes).filter(x => x[0]).map(x => x[1]);
+
+      guard_nodes = _.concat(guard_nodes, good_nodes);
+    }
+
+    if (guard_nodes.length < DESIRED_GUARD_COUNT) {
+      log.error(`COULD NOT get enough guard nodes, only have: ${guard_nodes.length}`);
+      debugger;
+    }
+
+    console.log("new guard nodes: ", guard_nodes);
+
+    const edKeys = guard_nodes.map(n => n.pubkey_ed25519);
+
+    await window.libloki.storage.updateGuardNodes(edKeys);
+
+    return guard_nodes;
+  }
+
+  async getOnionPath(toExclude = null) {
+
+    const _ = window.Lodash;
+
+    const good_paths = this.onionPaths.filter(x => !x.bad);
+
+    if (good_paths.length < 2) {
+      log.error(`Must have at least 2 good onion paths, actual: ${good_paths.length}`);
+      await this.buildNewOnionPaths();
+    }
+
+    const paths = _.shuffle(good_paths);
+
+    if (!toExclude) {
+      return paths[0];
+    }
+
+    // Select a path that doesn't contain `toExclude`
+    const other_paths = paths.filter(path => !_.some(path, node => node.pubkey_ed25519 == toExclude.pubkey_ed25519));
+
+    if (other_paths.length === 0) {
+      // This should never happen!
+      log.error("No onion paths available after filtering");
+    }
+
+    return other_paths[0].path;
+  }
+
+  async markPathAsBad(path) {
+    this.onionPaths.forEach(p => {
+      if (p.path == path) {
+        p.bad = true;
+      }
+    })
+  }
+  
+  async buildNewOnionPaths() {
+
+    // Note: this function may be called concurrently, so
+    // might consider blocking the other calls
+    
+    const _ = window.Lodash;
+
+    log.info("building new onion paths");
+
+    const all_nodes = await this.getRandomSnodePool();
+
+    if (this.guardNodes.length == 0) {
+
+      // Not cached, load from DB
+      let nodes = await window.libloki.storage.getGuardNodes();
+
+      if (nodes.length == 0) {
+        log.warn("no guard nodes in DB. Will be selecting new guards nodes...");
+      } else {
+        // We only store the nodes' keys, need to find full entries:
+        let ed_keys = nodes.map(x => x.ed25519PubKey);
+        this.guardNodes = all_nodes.filter(x => ed_keys.indexOf(x.pubkey_ed25519) !== -1);
+
+        if (this.guardNodes.length < ed_keys.length) {
+          log.warn(`could not find some guard nodes: ${this.guardNodes.length}/${ed_keys.length}`);
+        }
+
+      }
+
+      // If guard nodes is still empty (the old nodes are now invalid), select new ones:
+      if (this.guardNodes.length == 0 || true) {
+        this.guardNodes = await this.selectGuardNodes();
+      }
+
+    }
+
+    // TODO: select one guard node and 2 other nodes randomly
+    let other_nodes = _.difference(all_nodes, this.guardNodes);
+
+    if (other_nodes.length < 2) {
+      log.error("Too few nodes to build an onion path!");
+      return;
+    }
+
+    other_nodes = _.shuffle(other_nodes);
+    const guards = _.shuffle(this.guardNodes);
+
+    // Create path for every guard node:
+
+    // Each path needs 2 nodes in addition to the guard node:
+    const max_path = Math.floor(Math.min(guards.length, other_nodes.length / 2));
+
+    // TODO: might want to keep some of the existing paths
+    this.onionPaths = [];
+
+    for (let i = 0; i < max_path; i++) {
+      const path = [guards[i], other_nodes[i * 2], other_nodes[i * 2 + 1]];
+      this.onionPaths.push({path, bad: false});
+    }
+
+    log.info("Built onion paths: ", this.onionPaths);
   }
 
   async getRandomSnodeAddress() {

libloki/storage.js

@@ -240,6 +240,14 @@
     return window.Signal.Data.getSecondaryDevicesFor(primaryDevicePubKey);
   }
 
+  function getGuardNodes() {
+    return window.Signal.Data.getGuardNodes();
+  }
+
+  function updateGuardNodes(nodes) {
+    return window.Signal.Data.updateGuardNodes(nodes);
+  }
+
   async function getAllDevicePubKeysForPrimaryPubKey(primaryDevicePubKey) {
     await saveAllPairingAuthorisationsFor(primaryDevicePubKey);
     const secondaryPubKeys =
@@ -265,6 +273,8 @@
     getAllDevicePubKeysForPrimaryPubKey,
     getSecondaryDevicesFor,
     getPrimaryDeviceMapping,
+    getGuardNodes,
+    updateGuardNodes
   };
 
   // Libloki protocol store

preload.js

@@ -411,6 +411,7 @@ window.lokiFeatureFlags = {
   privateGroupChats: true,
   useSnodeProxy: true,
   useSealedSender: true,
+  useOnionRequests: true,
 };
 
 // eslint-disable-next-line no-extend-native,func-names
@@ -419,7 +420,7 @@ Promise.prototype.ignore = function() {
   this.then(() => {});
 };
 
-if (config.environment.includes('test')) {
+if (config.environment.includes('test') && !config.environment == "swarm-testing1" && !config.environment == "swarm-testing2") {
   const isWindows = process.platform === 'win32';
   /* eslint-disable global-require, import/no-extraneous-dependencies */
   window.test = {
@@ -439,5 +440,6 @@ if (config.environment.includes('test')) {
     updateSwarmNodes: () => {},
     updateLastHash: () => {},
     getSwarmNodesForPubKey: () => [],
+    buildNewOnionPaths: () => [],
   };
 }