From 05745d77260b7ba6c94669f73e27be7026230fee Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Tue, 25 May 2021 17:40:08 +1000 Subject: [PATCH] add tests to drop snode from path after 3 failure --- package.json | 2 +- ts/session/onions/onionPath.ts | 12 +- ts/session/snode_api/onions.ts | 9 +- .../session/unit/onion/OnionErrors_test.ts | 110 +++++++++++++++++- ts/test/session/unit/onion/OnionPaths_test.ts | 10 +- 5 files changed, 132 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 80168b2bb..3faef1cf1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "start": "cross-env NODE_APP_INSTANCE=$MULTI electron .", "start-prod": "cross-env NODE_ENV=production NODE_APP_INSTANCE=devprod$MULTI electron .", "start-swarm-test": "cross-env NODE_ENV=swarm-testing NODE_APP_INSTANCE=$MULTI electron .", - "grunt": "yarn clean-transpile; grunt", + "grunt": "grunt", "grunt:dev": "yarn clean-transpile; yarn grunt dev --force", "icon-gen": "electron-icon-maker --input=images/session/session_icon_1024.png --output=./build", "generate": "yarn icon-gen && yarn grunt --force", diff --git a/ts/session/onions/onionPath.ts b/ts/session/onions/onionPath.ts index 32f7800f3..343dea10a 100644 --- a/ts/session/onions/onionPath.ts +++ b/ts/session/onions/onionPath.ts @@ -9,7 +9,7 @@ import { allowOnlyOneAtATime } from '../utils/Promise'; const desiredGuardCount = 3; const minimumGuardCount = 2; -type SnodePath = Array; +export type SnodePath = Array; const onionRequestHops = 3; let onionPaths: Array = []; @@ -19,7 +19,8 @@ let onionPaths: Array = []; * @returns a copy of the onion path currently used by the app. * */ -export const getTestOnionPath = () => { +// tslint:disable-next-line: variable-name +export const TEST_getTestOnionPath = () => { return _.cloneDeep(onionPaths); }; @@ -36,7 +37,12 @@ export const clearTestOnionPath = () => { * hold the failure count of the path starting with the snode ed25519 pubkey. * exported just for tests. do not interact with this directly */ -export const pathFailureCount: Record = {}; +export let pathFailureCount: Record = {}; + +// tslint:disable-next-line: variable-name +export const TEST_resetPathFailureCount = () => { + pathFailureCount = {}; +}; // The number of times a path can fail before it's replaced. const pathFailureThreshold = 3; diff --git a/ts/session/snode_api/onions.ts b/ts/session/snode_api/onions.ts index fb7273e8e..dfa657cd2 100644 --- a/ts/session/snode_api/onions.ts +++ b/ts/session/snode_api/onions.ts @@ -14,7 +14,12 @@ import pRetry from 'p-retry'; import { incrementBadPathCountOrDrop } from '../onions/onionPath'; import _ from 'lodash'; // hold the ed25519 key of a snode against the time it fails. Used to remove a snode only after a few failures (snodeFailureThreshold failures) -const snodeFailureCount: Record = {}; +let snodeFailureCount: Record = {}; + +// tslint:disable-next-line: variable-name +export const TEST_resetSnodeFailureCount = () => { + snodeFailureCount = {}; +}; // The number of times a snode can fail before it's replaced. const snodeFailureThreshold = 3; @@ -32,7 +37,7 @@ export interface SnodeResponse { status: number; } -const NEXT_NODE_NOT_FOUND_PREFIX = 'Next node not found: '; +export const NEXT_NODE_NOT_FOUND_PREFIX = 'Next node not found: '; // Returns the actual ciphertext, symmetric key that will be used // for decryption, and an ephemeral_key to send to the next hop diff --git a/ts/test/session/unit/onion/OnionErrors_test.ts b/ts/test/session/unit/onion/OnionErrors_test.ts index 1b7e725c8..1b82fe5fb 100644 --- a/ts/test/session/unit/onion/OnionErrors_test.ts +++ b/ts/test/session/unit/onion/OnionErrors_test.ts @@ -10,10 +10,15 @@ import * as SNodeAPI from '../../../../../ts/session/snode_api/'; import chaiAsPromised from 'chai-as-promised'; import { OnionPaths } from '../../../../session/onions/'; -import { OXEN_SERVER_ERROR, processOnionResponse } from '../../../../session/snode_api/onions'; +import { + NEXT_NODE_NOT_FOUND_PREFIX, + OXEN_SERVER_ERROR, + processOnionResponse, +} from '../../../../session/snode_api/onions'; import AbortController from 'abort-controller'; import * as Data from '../../../../../ts/data/data'; import { Snode } from '../../../../session/snode_api/snodePool'; +import { SnodePath } from '../../../../session/onions/onionPath'; chai.use(chaiAsPromised as any); chai.should(); @@ -65,6 +70,8 @@ describe('OnionPathsErrors', () => { guardPubkeys = TestUtils.generateFakePubKeys(3).map(n => n.key); otherNodesPubkeys = TestUtils.generateFakePubKeys(9).map(n => n.key); + SNodeAPI.Onions.TEST_resetSnodeFailureCount(); + guard1ed = guardPubkeys[0]; guard2ed = guardPubkeys[1]; guard3ed = guardPubkeys[2]; @@ -427,4 +434,105 @@ describe('OnionPathsErrors', () => { expect(incrementBadSnodeCountOrDropSpy.callCount).to.eq(0); }); }); + + /** + * processOnionResponse OXEN SERVER ERROR + */ + describe('processOnionResponse - 502 - node not found', () => { + let oldOnionPaths: Array; + + beforeEach(async () => { + OnionPaths.TEST_resetPathFailureCount(); + + await OnionPaths.getOnionPath(); + + oldOnionPaths = OnionPaths.TEST_getTestOnionPath(); + }); + + // open group server v2 only talkes onion routing request. So errors can only happen at destination + it('throws a retryable error on 502', async () => { + const targetNode = otherNodesPubkeys[0]; + const failingSnode = oldOnionPaths[0][1]; + try { + await processOnionResponse({ + response: getFakeResponseOnPath( + 502, + `${NEXT_NODE_NOT_FOUND_PREFIX}${failingSnode.pubkey_ed25519}` + ), + symmetricKey: new Uint8Array(), + guardNode: guardSnode1, + lsrpcEd25519Key: targetNode, + associatedWith, + }); + throw new Error('Error expected'); + } catch (e) { + expect(e.message).to.equal('Bad Path handled. Retry this request. Status: 502'); + expect(e.name).to.not.equal('AbortError'); + } + expect(updateSwarmSpy.callCount).to.eq(0); + // now we make sure that this bad snode was dropped from this pubkey's swarm + expect(dropSnodeFromSwarmSpy.callCount).to.eq(0); + + // this specific node failed just once + expect(dropSnodeFromSnodePool.callCount).to.eq(0); + expect(dropSnodeFromPathSpy.callCount).to.eq(0); + expect(incrementBadPathCountOrDropSpy.callCount).to.eq(0); + expect(incrementBadSnodeCountOrDropSpy.callCount).to.eq(1); + expect(incrementBadSnodeCountOrDropSpy.firstCall.args[0]).to.deep.eq({ + snodeEd25519: failingSnode.pubkey_ed25519, + associatedWith, + }); + }); + + it('drop a snode from pool, swarm and path if it keep failing', async () => { + const targetNode = otherNodesPubkeys[0]; + const failingSnode = oldOnionPaths[0][1]; + for (let index = 0; index < 3; index++) { + try { + await processOnionResponse({ + response: getFakeResponseOnPath( + 502, + `${NEXT_NODE_NOT_FOUND_PREFIX}${failingSnode.pubkey_ed25519}` + ), + symmetricKey: new Uint8Array(), + guardNode: guardSnode1, + lsrpcEd25519Key: targetNode, + associatedWith, + }); + throw new Error('Error expected'); + } catch (e) { + expect(e.message).to.equal('Bad Path handled. Retry this request. Status: 502'); + expect(e.name).to.not.equal('AbortError'); + } + } + + expect(updateSwarmSpy.callCount).to.eq(0); + // now we make sure that this bad snode was dropped from this pubkey's swarm + expect(dropSnodeFromSwarmSpy.callCount).to.eq(1); + expect(dropSnodeFromSwarmSpy.firstCall.args[0]).to.eq(associatedWith); + expect(dropSnodeFromSwarmSpy.firstCall.args[1]).to.eq(failingSnode.pubkey_ed25519); + + // this specific node failed just once + expect(dropSnodeFromSnodePool.callCount).to.eq(1); + expect(dropSnodeFromSnodePool.firstCall.args[0]).to.eq(failingSnode.pubkey_ed25519); + expect(dropSnodeFromPathSpy.callCount).to.eq(1); + expect(dropSnodeFromPathSpy.firstCall.args[0]).to.eq(failingSnode.pubkey_ed25519); + + // we expect incrementBadSnodeCountOrDropSpy to be called three times with the same failing snode as we know who it is + expect(incrementBadSnodeCountOrDropSpy.callCount).to.eq(3); + expect(incrementBadSnodeCountOrDropSpy.args[0][0]).to.deep.eq({ + snodeEd25519: failingSnode.pubkey_ed25519, + associatedWith, + }); + expect(incrementBadSnodeCountOrDropSpy.args[1][0]).to.deep.eq({ + snodeEd25519: failingSnode.pubkey_ed25519, + associatedWith, + }); + expect(incrementBadSnodeCountOrDropSpy.args[2][0]).to.deep.eq({ + snodeEd25519: failingSnode.pubkey_ed25519, + associatedWith, + }); + expect(incrementBadPathCountOrDropSpy.callCount).to.eq(0); + }); + }); }); diff --git a/ts/test/session/unit/onion/OnionPaths_test.ts b/ts/test/session/unit/onion/OnionPaths_test.ts index a75c0005f..56694b2f4 100644 --- a/ts/test/session/unit/onion/OnionPaths_test.ts +++ b/ts/test/session/unit/onion/OnionPaths_test.ts @@ -142,6 +142,8 @@ describe('OnionPaths', () => { beforeEach(() => { sandbox2.stub(SNodeAPI.SnodePool, 'refreshRandomPoolDetail').resolves(fakeSnodePool); + SNodeAPI.Onions.TEST_resetSnodeFailureCount(); + OnionPaths.TEST_resetPathFailureCount(); // this just triggers a build of the onionPaths }); afterEach(() => { @@ -152,10 +154,10 @@ describe('OnionPaths', () => { // get a copy of what old ones look like await OnionPaths.getOnionPath(); - const oldOnionPath = OnionPaths.getTestOnionPath(); + const oldOnionPath = OnionPaths.TEST_getTestOnionPath(); await OnionPaths.dropSnodeFromPath(oldOnionPath[2][2].pubkey_ed25519); - const newOnionPath = OnionPaths.getTestOnionPath(); + const newOnionPath = OnionPaths.TEST_getTestOnionPath(); // only the last snode should have been updated expect(newOnionPath).to.be.not.deep.equal(oldOnionPath); @@ -170,9 +172,9 @@ describe('OnionPaths', () => { // get a copy of what old ones look like await OnionPaths.getOnionPath(); - const oldOnionPath = OnionPaths.getTestOnionPath(); + const oldOnionPath = OnionPaths.TEST_getTestOnionPath(); await OnionPaths.dropSnodeFromPath(oldOnionPath[2][1].pubkey_ed25519); - const newOnionPath = OnionPaths.getTestOnionPath(); + const newOnionPath = OnionPaths.TEST_getTestOnionPath(); const allEd25519Keys = _.flattenDeep(oldOnionPath).map(m => m.pubkey_ed25519);