Merge pull request #296 from BeaudanBrown/clearnet-seeds

Clearnet seeds

config/default.json

@@ -4,8 +4,9 @@
   "cdnUrl": "random.snode",
   "contentProxyUrl": "random.snode",
   "localServerPort": "8081",
-  "snodeServerPort": "8080",
   "defaultPoWDifficulty": "100",
+  "seedNodeUrl": "3.104.19.14",
+  "seedNodePort": "22023",
   "disableAutoUpdate": false,
   "updatesUrl": "https://updates2.signal.org/desktop",
   "updatesPublicKey":

js/modules/loki_message_api.js

@@ -43,9 +43,7 @@ const trySendP2p = async (pubKey, data64, isPing, messageEventData) => {
     return false;
   }
   try {
-    const port = p2pDetails.port ? `:${p2pDetails.port}` : '';
-
-    await rpc(p2pDetails.address, port, 'store', {
+    await rpc(p2pDetails.address, p2pDetails.port, 'store', {
       data: data64,
     });
     lokiP2pAPI.setContactOnline(pubKey);
@@ -68,9 +66,30 @@ const trySendP2p = async (pubKey, data64, isPing, messageEventData) => {
   }
 };
 
+const retrieveNextMessages = async (nodeUrl, nodeData, ourKey) => {
+  const params = {
+    pubKey: ourKey,
+    lastHash: nodeData.lastHash || '',
+  };
+  const options = {
+    timeout: 40000,
+    headers: {
+      [LOKI_LONGPOLL_HEADER]: true,
+    },
+  };
+
+  const result = await rpc(
+    `https://${nodeUrl}`,
+    nodeData.port,
+    'retrieve',
+    params,
+    options
+  );
+  return result.messages || [];
+};
+
 class LokiMessageAPI {
-  constructor({ snodeServerPort }) {
-    this.snodeServerPort = snodeServerPort ? `:${snodeServerPort}` : '';
+  constructor() {
     this.jobQueue = new window.JobQueue();
     this.sendingSwarmNodes = {};
   }
@@ -151,8 +170,13 @@ class LokiMessageAPI {
 
   async openSendConnection(params) {
     while (!_.isEmpty(this.sendingSwarmNodes[params.timestamp])) {
-      const url = this.sendingSwarmNodes[params.timestamp].shift();
-      const successfulSend = await this.sendToNode(url, params);
+      const snode = this.sendingSwarmNodes[params.timestamp].shift();
+      // TODO: Revert back to using snode address instead of IP
+      const successfulSend = await this.sendToNode(
+        snode.ip,
+        snode.port,
+        params
+      );
       if (successfulSend) {
         return true;
       }
@@ -160,17 +184,12 @@ class LokiMessageAPI {
     return false;
   }
 
-  async sendToNode(url, params) {
+  async sendToNode(address, port, params) {
     let successiveFailures = 0;
     while (successiveFailures < 3) {
       await sleepFor(successiveFailures * 500);
       try {
-        const result = await rpc(
-          `https://${url}`,
-          this.snodeServerPort,
-          'store',
-          params
-        );
+        const result = await rpc(`https://${address}`, port, 'store', params);
 
         // Make sure we aren't doing too much PoW
         const currentDifficulty = window.storage.get('PoWDifficulty', null);
@@ -203,51 +222,34 @@ class LokiMessageAPI {
         }
       }
     }
-    log.error(`Failed to send to node: ${url}`);
-    await lokiSnodeAPI.unreachableNode(params.pubKey, url);
+    log.error(`Failed to send to node: ${address}`);
+    await lokiSnodeAPI.unreachableNode(params.pubKey, address);
     return false;
   }
 
-  async retrieveNextMessages(nodeUrl, nodeData, ourKey) {
-    const params = {
-      pubKey: ourKey,
-      lastHash: nodeData.lastHash || '',
-    };
-    const options = {
-      timeout: 40000,
-      headers: {
-        [LOKI_LONGPOLL_HEADER]: true,
-      },
-    };
-
-    const result = await rpc(
-      `https://${nodeUrl}`,
-      this.snodeServerPort,
-      'retrieve',
-      params,
-      options
-    );
-    return result.messages || [];
-  }
-
   async openConnection(callback) {
     const ourKey = window.textsecure.storage.user.getNumber();
     while (!_.isEmpty(this.ourSwarmNodes)) {
-      const url = Object.keys(this.ourSwarmNodes)[0];
-      const nodeData = this.ourSwarmNodes[url];
-      delete this.ourSwarmNodes[url];
+      const address = Object.keys(this.ourSwarmNodes)[0];
+      const nodeData = this.ourSwarmNodes[address];
+      delete this.ourSwarmNodes[address];
       let successiveFailures = 0;
       while (successiveFailures < 3) {
         await sleepFor(successiveFailures * 1000);
 
         try {
-          let messages = await this.retrieveNextMessages(url, nodeData, ourKey);
+          // TODO: Revert back to using snode address instead of IP
+          let messages = await retrieveNextMessages(
+            nodeData.ip,
+            nodeData,
+            ourKey
+          );
           successiveFailures = 0;
           if (messages.length) {
             const lastMessage = _.last(messages);
             nodeData.lashHash = lastMessage.hash;
             lokiSnodeAPI.updateLastHash(
-              url,
+              address,
               lastMessage.hash,
               lastMessage.expiration
             );

js/modules/loki_rpc.js

@@ -101,10 +101,21 @@ const fetch = async (url, options = {}) => {
 };
 
 // Wrapper for a JSON RPC request
-const rpc = (address, port, method, params, options = {}) => {
+const rpc = (
+  address,
+  port,
+  method,
+  params,
+  options = {},
+  endpoint = endpointBase
+) => {
   const headers = options.headers || {};
-  const url = `${address}${port}${endpointBase}`;
+  const portString = port ? `:${port}` : '';
+  const url = `${address}${portString}${endpoint}`;
+  // TODO: The jsonrpc and body field will be ignored on storage server
   const body = {
+    jsonrpc: '2.0',
+    id: '0',
     method,
     params,
   };

js/modules/loki_snode_api.js

@@ -32,13 +32,13 @@ const resolveCname = url =>
   });
 
 class LokiSnodeAPI {
-  constructor({ serverUrl, localUrl, snodeServerPort }) {
+  constructor({ serverUrl, localUrl }) {
     if (!is.string(serverUrl)) {
       throw new Error('WebAPI.initialize: Invalid server url');
     }
     this.serverUrl = serverUrl;
     this.localUrl = localUrl;
-    this.snodeServerPort = snodeServerPort ? `:${snodeServerPort}` : '';
+    this.randomSnodePool = [];
     this.swarmsPendingReplenish = {};
     this.ourSwarmNodes = {};
     this.contactSwarmNodes = {};
@@ -60,14 +60,35 @@ class LokiSnodeAPI {
     }
   }
 
-  async getMyLokiAddress() {
+  getMyLokiAddress() {
     /* resolve our local loki address */
     return resolveCname(this.localUrl);
   }
 
-  getRandomSnodeAddress() {
+  async getRandomSnodeAddress() {
     /* resolve random snode */
-    return resolveCname(this.serverUrl);
+    if (this.randomSnodePool.length === 0) {
+      await this.initialiseRandomPool();
+    }
+    return this.randomSnodePool[
+      Math.floor(Math.random() * this.randomSnodePool.length)
+    ];
+  }
+
+  async initialiseRandomPool() {
+    const result = await rpc(
+      `http://${window.seedNodeUrl}`,
+      window.seedNodePort,
+      'get_service_nodes',
+      {}, // Params
+      {}, // Options
+      '/json_rpc' // Seed request endpoint
+    );
+    const snodes = result.result.service_node_states;
+    this.randomSnodePool = snodes.map(snode => ({
+      address: snode.public_ip,
+      port: snode.storage_port,
+    }));
   }
 
   async unreachableNode(pubKey, nodeUrl) {
@@ -88,12 +109,9 @@ class LokiSnodeAPI {
   async updateLastHash(nodeUrl, lastHash, expiresAt) {
     await window.Signal.Data.updateLastHash({ nodeUrl, lastHash, expiresAt });
     if (!this.ourSwarmNodes[nodeUrl]) {
-      this.ourSwarmNodes[nodeUrl] = {
-        lastHash,
-      };
-    } else {
-      this.ourSwarmNodes[nodeUrl].lastHash = lastHash;
+      return;
     }
+    this.ourSwarmNodes[nodeUrl].lastHash = lastHash;
   }
 
   getSwarmNodesForPubKey(pubKey) {
@@ -121,10 +139,14 @@ class LokiSnodeAPI {
 
   async updateOurSwarmNodes(newNodes) {
     this.ourSwarmNodes = {};
-    const ps = newNodes.map(async url => {
-      const lastHash = await window.Signal.Data.getLastHashBySnode(url);
-      this.ourSwarmNodes[url] = {
+    const ps = newNodes.map(async snode => {
+      const lastHash = await window.Signal.Data.getLastHashBySnode(
+        snode.address
+      );
+      this.ourSwarmNodes[snode.address] = {
         lastHash,
+        port: snode.port,
+        ip: snode.ip,
       };
     });
     await Promise.all(ps);
@@ -167,11 +189,11 @@ class LokiSnodeAPI {
 
   async getSwarmNodes(pubKey) {
     // TODO: Hit multiple random nodes and merge lists?
-    const nodeUrl = await this.getRandomSnodeAddress();
+    const { address, port } = await this.getRandomSnodeAddress();
 
     const result = await rpc(
-      `https://${nodeUrl}`,
-      this.snodeServerPort,
+      `https://${address}`,
+      port,
       'get_snodes_for_pubkey',
       {
         pubKey,

main.js

@@ -154,9 +154,10 @@ function prepareURL(pathSegments, moreKeys) {
       serverUrl: config.get('serverUrl'),
       localUrl: config.get('localUrl'),
       cdnUrl: config.get('cdnUrl'),
-      snodeServerPort: config.get('snodeServerPort'),
       localServerPort: config.get('localServerPort'),
       defaultPoWDifficulty: config.get('defaultPoWDifficulty'),
+      seedNodeUrl: config.get('seedNodeUrl'),
+      seedNodePort: config.get('seedNodePort'),
       certificateAuthority: config.get('certificateAuthority'),
       environment: config.environment,
       node_version: process.versions.node,

preload.js

@@ -291,22 +291,20 @@ window.WebAPI = initializeWebAPI({
   proxyUrl: config.proxyUrl,
 });
 
+window.seedNodeUrl = config.seedNodeUrl;
+window.seedNodePort = config.seedNodePort;
 const LokiSnodeAPI = require('./js/modules/loki_snode_api');
 
 window.lokiSnodeAPI = new LokiSnodeAPI({
   serverUrl: config.serverUrl,
   localUrl: config.localUrl,
-  snodeServerPort: config.snodeServerPort,
 });
 
 window.LokiP2pAPI = require('./js/modules/loki_p2p_api');
 
 const LokiMessageAPI = require('./js/modules/loki_message_api');
 
-window.lokiMessageAPI = new LokiMessageAPI({
-  url: config.serverUrl,
-  snodeServerPort: config.snodeServerPort,
-});
+window.lokiMessageAPI = new LokiMessageAPI();
 
 const LocalLokiServer = require('./libloki/modules/local_loki_server');