add tests to drop snode from path after 3 failure

package.json

@@ -18,7 +18,7 @@
     "start": "cross-env NODE_APP_INSTANCE=$MULTI electron .",
     "start-prod": "cross-env NODE_ENV=production NODE_APP_INSTANCE=devprod$MULTI electron .",
     "start-swarm-test": "cross-env NODE_ENV=swarm-testing NODE_APP_INSTANCE=$MULTI electron .",
-    "grunt": "yarn clean-transpile; grunt",
+    "grunt": "grunt",
     "grunt:dev": "yarn clean-transpile; yarn grunt dev --force",
     "icon-gen": "electron-icon-maker --input=images/session/session_icon_1024.png --output=./build",
     "generate": "yarn icon-gen && yarn grunt --force",

ts/session/onions/onionPath.ts

@@ -9,7 +9,7 @@ import { allowOnlyOneAtATime } from '../utils/Promise';
 const desiredGuardCount = 3;
 const minimumGuardCount = 2;
 
-type SnodePath = Array<SnodePool.Snode>;
+export type SnodePath = Array<SnodePool.Snode>;
 
 const onionRequestHops = 3;
 let onionPaths: Array<SnodePath> = [];
@@ -19,7 +19,8 @@ let onionPaths: Array<SnodePath> = [];
  * @returns a copy of the onion path currently used by the app.
  *
  */
-export const getTestOnionPath = () => {
+// tslint:disable-next-line: variable-name
+export const TEST_getTestOnionPath = () => {
   return _.cloneDeep(onionPaths);
 };
 
@@ -36,7 +37,12 @@ export const clearTestOnionPath = () => {
  * hold the failure count of the path starting with the snode ed25519 pubkey.
  * exported just for tests. do not interact with this directly
  */
-export const pathFailureCount: Record<string, number> = {};
+export let pathFailureCount: Record<string, number> = {};
+
+// tslint:disable-next-line: variable-name
+export const TEST_resetPathFailureCount = () => {
+  pathFailureCount = {};
+};
 
 // The number of times a path can fail before it's replaced.
 const pathFailureThreshold = 3;

ts/session/snode_api/onions.ts

@@ -14,7 +14,12 @@ import pRetry from 'p-retry';
 import { incrementBadPathCountOrDrop } from '../onions/onionPath';
 import _ from 'lodash';
 // hold the ed25519 key of a snode against the time it fails. Used to remove a snode only after a few failures (snodeFailureThreshold failures)
-const snodeFailureCount: Record<string, number> = {};
+let snodeFailureCount: Record<string, number> = {};
+
+// tslint:disable-next-line: variable-name
+export const TEST_resetSnodeFailureCount = () => {
+  snodeFailureCount = {};
+};
 
 // The number of times a snode can fail before it's replaced.
 const snodeFailureThreshold = 3;
@@ -32,7 +37,7 @@ export interface SnodeResponse {
   status: number;
 }
 
-const NEXT_NODE_NOT_FOUND_PREFIX = 'Next node not found: ';
+export const NEXT_NODE_NOT_FOUND_PREFIX = 'Next node not found: ';
 
 // Returns the actual ciphertext, symmetric key that will be used
 // for decryption, and an ephemeral_key to send to the next hop

ts/test/session/unit/onion/OnionErrors_test.ts

@@ -10,10 +10,15 @@ import * as SNodeAPI from '../../../../../ts/session/snode_api/';
 
 import chaiAsPromised from 'chai-as-promised';
 import { OnionPaths } from '../../../../session/onions/';
-import { OXEN_SERVER_ERROR, processOnionResponse } from '../../../../session/snode_api/onions';
+import {
+  NEXT_NODE_NOT_FOUND_PREFIX,
+  OXEN_SERVER_ERROR,
+  processOnionResponse,
+} from '../../../../session/snode_api/onions';
 import AbortController from 'abort-controller';
 import * as Data from '../../../../../ts/data/data';
 import { Snode } from '../../../../session/snode_api/snodePool';
+import { SnodePath } from '../../../../session/onions/onionPath';
 
 chai.use(chaiAsPromised as any);
 chai.should();
@@ -65,6 +70,8 @@ describe('OnionPathsErrors', () => {
     guardPubkeys = TestUtils.generateFakePubKeys(3).map(n => n.key);
     otherNodesPubkeys = TestUtils.generateFakePubKeys(9).map(n => n.key);
 
+    SNodeAPI.Onions.TEST_resetSnodeFailureCount();
+
     guard1ed = guardPubkeys[0];
     guard2ed = guardPubkeys[1];
     guard3ed = guardPubkeys[2];
@@ -427,4 +434,105 @@ describe('OnionPathsErrors', () => {
       expect(incrementBadSnodeCountOrDropSpy.callCount).to.eq(0);
     });
   });
+
+  /**
+   * processOnionResponse OXEN SERVER ERROR
+   */
+  describe('processOnionResponse - 502 - node not found', () => {
+    let oldOnionPaths: Array<SnodePath>;
+
+    beforeEach(async () => {
+      OnionPaths.TEST_resetPathFailureCount();
+
+      await OnionPaths.getOnionPath();
+
+      oldOnionPaths = OnionPaths.TEST_getTestOnionPath();
+    });
+
+    // open group server v2 only talkes onion routing request. So errors can only happen at destination
+    it('throws a retryable error on 502', async () => {
+      const targetNode = otherNodesPubkeys[0];
+      const failingSnode = oldOnionPaths[0][1];
+      try {
+        await processOnionResponse({
+          response: getFakeResponseOnPath(
+            502,
+            `${NEXT_NODE_NOT_FOUND_PREFIX}${failingSnode.pubkey_ed25519}`
+          ),
+          symmetricKey: new Uint8Array(),
+          guardNode: guardSnode1,
+          lsrpcEd25519Key: targetNode,
+          associatedWith,
+        });
+        throw new Error('Error expected');
+      } catch (e) {
+        expect(e.message).to.equal('Bad Path handled. Retry this request. Status: 502');
+        expect(e.name).to.not.equal('AbortError');
+      }
+      expect(updateSwarmSpy.callCount).to.eq(0);
+      // now we make sure that this bad snode was dropped from this pubkey's swarm
+      expect(dropSnodeFromSwarmSpy.callCount).to.eq(0);
+
+      // this specific node failed just once
+      expect(dropSnodeFromSnodePool.callCount).to.eq(0);
+      expect(dropSnodeFromPathSpy.callCount).to.eq(0);
+      expect(incrementBadPathCountOrDropSpy.callCount).to.eq(0);
+      expect(incrementBadSnodeCountOrDropSpy.callCount).to.eq(1);
+      expect(incrementBadSnodeCountOrDropSpy.firstCall.args[0]).to.deep.eq({
+        snodeEd25519: failingSnode.pubkey_ed25519,
+        associatedWith,
+      });
+    });
+
+    it('drop a snode from pool, swarm and path if it keep failing', async () => {
+      const targetNode = otherNodesPubkeys[0];
+      const failingSnode = oldOnionPaths[0][1];
+      for (let index = 0; index < 3; index++) {
+        try {
+          await processOnionResponse({
+            response: getFakeResponseOnPath(
+              502,
+              `${NEXT_NODE_NOT_FOUND_PREFIX}${failingSnode.pubkey_ed25519}`
+            ),
+            symmetricKey: new Uint8Array(),
+            guardNode: guardSnode1,
+            lsrpcEd25519Key: targetNode,
+            associatedWith,
+          });
+          throw new Error('Error expected');
+        } catch (e) {
+          expect(e.message).to.equal('Bad Path handled. Retry this request. Status: 502');
+          expect(e.name).to.not.equal('AbortError');
+        }
+      }
+
+      expect(updateSwarmSpy.callCount).to.eq(0);
+      // now we make sure that this bad snode was dropped from this pubkey's swarm
+      expect(dropSnodeFromSwarmSpy.callCount).to.eq(1);
+      expect(dropSnodeFromSwarmSpy.firstCall.args[0]).to.eq(associatedWith);
+      expect(dropSnodeFromSwarmSpy.firstCall.args[1]).to.eq(failingSnode.pubkey_ed25519);
+
+      // this specific node failed just once
+      expect(dropSnodeFromSnodePool.callCount).to.eq(1);
+      expect(dropSnodeFromSnodePool.firstCall.args[0]).to.eq(failingSnode.pubkey_ed25519);
+      expect(dropSnodeFromPathSpy.callCount).to.eq(1);
+      expect(dropSnodeFromPathSpy.firstCall.args[0]).to.eq(failingSnode.pubkey_ed25519);
+
+      // we expect incrementBadSnodeCountOrDropSpy to be called three times with the same failing snode as we know who it is
+      expect(incrementBadSnodeCountOrDropSpy.callCount).to.eq(3);
+      expect(incrementBadSnodeCountOrDropSpy.args[0][0]).to.deep.eq({
+        snodeEd25519: failingSnode.pubkey_ed25519,
+        associatedWith,
+      });
+      expect(incrementBadSnodeCountOrDropSpy.args[1][0]).to.deep.eq({
+        snodeEd25519: failingSnode.pubkey_ed25519,
+        associatedWith,
+      });
+      expect(incrementBadSnodeCountOrDropSpy.args[2][0]).to.deep.eq({
+        snodeEd25519: failingSnode.pubkey_ed25519,
+        associatedWith,
+      });
+      expect(incrementBadPathCountOrDropSpy.callCount).to.eq(0);
+    });
+  });
 });

ts/test/session/unit/onion/OnionPaths_test.ts

@@ -142,6 +142,8 @@ describe('OnionPaths', () => {
 
       beforeEach(() => {
         sandbox2.stub(SNodeAPI.SnodePool, 'refreshRandomPoolDetail').resolves(fakeSnodePool);
+        SNodeAPI.Onions.TEST_resetSnodeFailureCount();
+        OnionPaths.TEST_resetPathFailureCount();
         // this just triggers a build of the onionPaths
       });
       afterEach(() => {
@@ -152,10 +154,10 @@ describe('OnionPaths', () => {
         // get a copy of what old ones look like
         await OnionPaths.getOnionPath();
 
-        const oldOnionPath = OnionPaths.getTestOnionPath();
+        const oldOnionPath = OnionPaths.TEST_getTestOnionPath();
 
         await OnionPaths.dropSnodeFromPath(oldOnionPath[2][2].pubkey_ed25519);
-        const newOnionPath = OnionPaths.getTestOnionPath();
+        const newOnionPath = OnionPaths.TEST_getTestOnionPath();
 
         // only the last snode should have been updated
         expect(newOnionPath).to.be.not.deep.equal(oldOnionPath);
@@ -170,9 +172,9 @@ describe('OnionPaths', () => {
         // get a copy of what old ones look like
         await OnionPaths.getOnionPath();
 
-        const oldOnionPath = OnionPaths.getTestOnionPath();
+        const oldOnionPath = OnionPaths.TEST_getTestOnionPath();
         await OnionPaths.dropSnodeFromPath(oldOnionPath[2][1].pubkey_ed25519);
-        const newOnionPath = OnionPaths.getTestOnionPath();
+        const newOnionPath = OnionPaths.TEST_getTestOnionPath();
 
         const allEd25519Keys = _.flattenDeep(oldOnionPath).map(m => m.pubkey_ed25519);