From 2357368ddf0e5f6932fd8ca38c5b51dc55d1ef3b Mon Sep 17 00:00:00 2001
From: Audric Ackermann <audric@loki.network>
Date: Tue, 23 Jul 2024 11:29:28 +1000
Subject: [PATCH 1/3] feat: fetch snode storage version and filter for edge
snode
---
ts/data/data.ts | 1 +
ts/node/migration/sessionMigrations.ts | 23 ++-
ts/session/apis/seed_node_api/SeedNodeAPI.ts | 3 +
.../apis/snode_api/SnodeRequestTypes.ts | 1 +
.../apis/snode_api/getServiceNodesList.ts | 6 +-
ts/session/onions/onionPath.ts | 69 +++++++--
ts/test/session/unit/onion/OnionPaths_test.ts | 134 ++++++++++++++++++
.../session/unit/onion/SeedNodeAPI_test.ts | 1 +
ts/test/test-utils/utils/pubkey.ts | 10 +-
9 files changed, 232 insertions(+), 16 deletions(-)
diff --git a/ts/data/data.ts b/ts/data/data.ts
index ed585f7f3..1287bbe22 100644
--- a/ts/data/data.ts
+++ b/ts/data/data.ts
@@ -47,6 +47,7 @@ export interface Snode {
port: number;
pubkey_x25519: string;
pubkey_ed25519: string;
+ storage_server_version: Array<number>;
}
export type SwarmNode = Snode & {
diff --git a/ts/node/migration/sessionMigrations.ts b/ts/node/migration/sessionMigrations.ts
index e000ed1b8..f0ec7ad27 100644
--- a/ts/node/migration/sessionMigrations.ts
+++ b/ts/node/migration/sessionMigrations.ts
@@ -17,6 +17,7 @@ import {
CLOSED_GROUP_V2_KEY_PAIRS_TABLE,
CONVERSATIONS_TABLE,
GUARD_NODE_TABLE,
+ ITEMS_TABLE,
LAST_HASHES_TABLE,
MESSAGES_TABLE,
NODES_FOR_PUBKEY_TABLE,
@@ -26,7 +27,7 @@ import {
rebuildFtsTable,
} from '../database_utility';
-import { SettingsKey } from '../../data/settings-key';
+import { SettingsKey, SNODE_POOL_ITEM_ID } from '../../data/settings-key';
import { sleepFor } from '../../session/utils/Promise';
import { sqlNode } from '../sql';
import MIGRATION_HELPERS from './helpers';
@@ -105,6 +106,7 @@ const LOKI_SCHEMA_VERSIONS = [
updateToSessionSchemaVersion34,
updateToSessionSchemaVersion35,
updateToSessionSchemaVersion36,
+ updateToSessionSchemaVersion37,
];
function updateToSessionSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) {
@@ -1949,6 +1951,25 @@ function updateToSessionSchemaVersion36(currentVersion: number, db: BetterSqlite
console.log(`updateToSessionSchemaVersion${targetVersion}: success!`);
}
+function updateToSessionSchemaVersion37(currentVersion: number, db: BetterSqlite3.Database) {
+ const targetVersion = 37;
+ if (currentVersion >= targetVersion) {
+ return;
+ }
+
+ console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`);
+
+ db.transaction(() => {
+ console.info(`clearing ${SNODE_POOL_ITEM_ID} cache`);
+ db.prepare(`DELETE FROM ${ITEMS_TABLE} WHERE id = $snodePoolId;`).run({
+ snodePoolId: SNODE_POOL_ITEM_ID,
+ });
+ writeSessionSchemaVersion(targetVersion, db);
+ })();
+
+ console.log(`updateToSessionSchemaVersion${targetVersion}: success!`);
+}
+
export function printTableColumns(table: string, db: BetterSqlite3.Database) {
console.info(db.pragma(`table_info('${table}');`));
}
diff --git a/ts/session/apis/seed_node_api/SeedNodeAPI.ts b/ts/session/apis/seed_node_api/SeedNodeAPI.ts
index 899b26f4d..9a6d214ae 100644
--- a/ts/session/apis/seed_node_api/SeedNodeAPI.ts
+++ b/ts/session/apis/seed_node_api/SeedNodeAPI.ts
@@ -35,6 +35,7 @@ export async function fetchSnodePoolFromSeedNodeWithRetries(
port: snode.storage_port,
pubkey_x25519: snode.pubkey_x25519,
pubkey_ed25519: snode.pubkey_ed25519,
+ storage_server_version: snode.storage_server_version,
}));
window?.log?.info(
'SeedNodeAPI::fetchSnodePoolFromSeedNodeWithRetries - Refreshed random snode pool with',
@@ -140,6 +141,7 @@ export interface SnodeFromSeed {
storage_port: number;
pubkey_x25519: string;
pubkey_ed25519: string;
+ storage_server_version: Array<number>;
}
const getSnodeListFromSeednodeOneAtAtime = async (seedNodes: Array<string>) =>
@@ -241,6 +243,7 @@ async function getSnodesFromSeedUrl(urlObj: URL): Promise<Array<any>> {
storage_port: true,
pubkey_x25519: true,
pubkey_ed25519: true,
+ storage_server_version: true,
},
},
};
diff --git a/ts/session/apis/snode_api/SnodeRequestTypes.ts b/ts/session/apis/snode_api/SnodeRequestTypes.ts
index ee2a7dbb7..365f975db 100644
--- a/ts/session/apis/snode_api/SnodeRequestTypes.ts
+++ b/ts/session/apis/snode_api/SnodeRequestTypes.ts
@@ -79,6 +79,7 @@ type FetchSnodeListParams = {
storage_port: true;
pubkey_x25519: true;
pubkey_ed25519: true;
+ storage_server_version: true;
};
};
diff --git a/ts/session/apis/snode_api/getServiceNodesList.ts b/ts/session/apis/snode_api/getServiceNodesList.ts
index 8f35d8288..0bb256e2b 100644
--- a/ts/session/apis/snode_api/getServiceNodesList.ts
+++ b/ts/session/apis/snode_api/getServiceNodesList.ts
@@ -18,6 +18,7 @@ function buildSnodeListRequests(): Array<GetServiceNodesSubRequest> {
storage_port: true,
pubkey_x25519: true,
pubkey_ed25519: true,
+ storage_server_version: true,
},
},
},
@@ -49,14 +50,15 @@ async function getSnodePoolFromSnode(targetNode: Snode): Promise<Array<Snode>> {
}
// Filter 0.0.0.0 nodes which haven't submitted uptime proofs
- const snodes = json.result.service_node_states
+ const snodes: Array<Snode> = json.result.service_node_states
.filter((snode: any) => snode.public_ip !== '0.0.0.0')
.map((snode: any) => ({
ip: snode.public_ip,
port: snode.storage_port,
pubkey_x25519: snode.pubkey_x25519,
pubkey_ed25519: snode.pubkey_ed25519,
- })) as Array<Snode>;
+ storage_server_version: snode.storage_server_version,
+ }));
GetNetworkTime.handleTimestampOffsetFromNetwork('get_service_nodes', json.t);
// we the return list by the snode is already made of uniq snodes
diff --git a/ts/session/onions/onionPath.ts b/ts/session/onions/onionPath.ts
index 84b9d14a1..49f9bfe3f 100644
--- a/ts/session/onions/onionPath.ts
+++ b/ts/session/onions/onionPath.ts
@@ -1,10 +1,11 @@
/* eslint-disable import/no-mutable-exports */
/* eslint-disable no-await-in-loop */
-import _, { compact } from 'lodash';
+import _, { compact, isFinite, isNumber, sample } from 'lodash';
import pRetry from 'p-retry';
// eslint-disable-next-line import/no-named-default
import { default as insecureNodeFetch } from 'node-fetch';
+import semver from 'semver';
import { Data, Snode } from '../../data/data';
import * as SnodePool from '../apis/snode_api/snodePool';
import { UserUtils } from '../utils';
@@ -15,11 +16,16 @@ import { ERROR_CODE_NO_CONNECT } from '../apis/snode_api/SNodeAPI';
import { OnionPaths } from '.';
import { APPLICATION_JSON } from '../../types/MIME';
import { ed25519Str } from '../utils/String';
+import { DURATION } from '../constants';
const desiredGuardCount = 3;
const minimumGuardCount = 2;
const ONION_REQUEST_HOPS = 3;
+export function getOnionPathMinTimeout() {
+ return DURATION.SECONDS;
+}
+
export let onionPaths: Array<Array<Snode>> = [];
/**
@@ -498,16 +504,27 @@ async function buildNewOnionPathsWorker() {
for (let i = 0; i < maxPath; i += 1) {
const path = [guards[i]];
- for (let j = 0; j < nodesNeededPerPaths; j += 1) {
- const randomWinner = _.sample(otherNodes);
- if (!randomWinner) {
- throw new Error('randomWinner unset during path building task');
+
+ do {
+ // selection of the last snode (edge snode) needs at least v2.8.0
+ if (path.length === nodesNeededPerPaths) {
+ const randomEdgeSnode = getRandomEdgeSnode(otherNodes);
+ otherNodes = otherNodes.filter(n => {
+ return n.pubkey_ed25519 !== randomEdgeSnode?.pubkey_ed25519;
+ });
+ path.push(randomEdgeSnode);
+ } else {
+ const snode = sample(otherNodes);
+ if (!snode) {
+ throw new Error('no more snode found for path building');
+ }
+ otherNodes = otherNodes.filter(n => {
+ return n.pubkey_ed25519 !== snode?.pubkey_ed25519;
+ });
+
+ path.push(snode);
}
- otherNodes = otherNodes.filter(n => {
- return n.pubkey_ed25519 !== randomWinner?.pubkey_ed25519;
- });
- path.push(randomWinner);
- }
+ } while (path.length <= nodesNeededPerPaths);
onionPaths.push(path);
}
@@ -516,7 +533,7 @@ async function buildNewOnionPathsWorker() {
{
retries: 3, // 4 total
factor: 1,
- minTimeout: 1000,
+ minTimeout: OnionPaths.getOnionPathMinTimeout(),
onFailedAttempt: e => {
window?.log?.warn(
`buildNewOnionPathsWorker attempt #${e.attemptNumber} failed. ${e.retriesLeft} retries left... Error: ${e.message}`
@@ -525,3 +542,33 @@ async function buildNewOnionPathsWorker() {
}
);
}
+
+export function getRandomEdgeSnode(snodes: Array<Snode>) {
+ const allSnodesWithv280 = snodes.filter(snode => {
+ const snodeStorageVersion = snode.storage_server_version;
+
+ if (
+ !snodeStorageVersion ||
+ !Array.isArray(snodeStorageVersion) ||
+ snodeStorageVersion.length !== 3 ||
+ snodeStorageVersion.some(m => !isNumber(m) || !isFinite(m))
+ ) {
+ return false;
+ }
+ const storageVersionAsString = `${snodeStorageVersion[0]}.${snodeStorageVersion[1]}.${snodeStorageVersion[2]}`;
+ const verifiedStorageVersion = semver.valid(storageVersionAsString);
+ if (!verifiedStorageVersion) {
+ return false;
+ }
+ if (semver.lt(verifiedStorageVersion, '2.8.0')) {
+ return false;
+ }
+ return true;
+ });
+
+ const randomEdgeSnode = sample(allSnodesWithv280);
+ if (!randomEdgeSnode) {
+ throw new Error('did not find a single snode which can be the edge');
+ }
+ return randomEdgeSnode;
+}
diff --git a/ts/test/session/unit/onion/OnionPaths_test.ts b/ts/test/session/unit/onion/OnionPaths_test.ts
index c30b8517e..932ef4f29 100644
--- a/ts/test/session/unit/onion/OnionPaths_test.ts
+++ b/ts/test/session/unit/onion/OnionPaths_test.ts
@@ -16,6 +16,7 @@ import {
} from '../../../test-utils/utils';
import { SeedNodeAPI } from '../../../../session/apis/seed_node_api';
import { ServiceNodesList } from '../../../../session/apis/snode_api/getServiceNodesList';
+import { TEST_resetState } from '../../../../session/apis/snode_api/snodePool';
chai.use(chaiAsPromised as any);
chai.should();
@@ -110,4 +111,137 @@ describe('OnionPaths', () => {
});
});
});
+
+ describe('getRandomEdgeSnode', () => {
+ it('find single valid snode in poll of many non valid snodes', () => {
+ const originalSnodePool = generateFakeSnodes(20);
+ const firstValidSnodePool = originalSnodePool.map((m, i) => {
+ if (i > 0) {
+ return {
+ ...m,
+ storage_server_version: [2, 7, 0],
+ };
+ }
+ return m;
+ });
+
+ expect(OnionPaths.getRandomEdgeSnode(firstValidSnodePool)).to.be.deep.eq(
+ originalSnodePool[0]
+ );
+
+ const lastValidSnodePool = originalSnodePool.map((m, i) => {
+ if (i !== originalSnodePool.length - 1) {
+ return {
+ ...m,
+ storage_server_version: [2, 7, 0],
+ };
+ }
+ return m;
+ });
+
+ expect(OnionPaths.getRandomEdgeSnode(lastValidSnodePool)).to.be.deep.eq(
+ originalSnodePool[originalSnodePool.length - 1]
+ );
+ });
+
+ it('random if multiple matches', () => {
+ const originalSnodePool = generateFakeSnodes(5);
+ const multipleMatchesSnodePool = originalSnodePool.map((m, i) => {
+ if (i % 5 === 0) {
+ return {
+ ...m,
+ storage_server_version: [2, 7, 0],
+ };
+ }
+ return m;
+ });
+ const filtered = originalSnodePool.filter((_m, i) => i % 5 !== 0);
+ const winner = OnionPaths.getRandomEdgeSnode(multipleMatchesSnodePool);
+ expect(filtered).to.deep.include(winner);
+ });
+
+ it('throws if we run out of snodes with valid version', () => {
+ const originalSnodePool = generateFakeSnodes(5);
+ const multipleMatchesSnodePool = originalSnodePool.map(m => {
+ return {
+ ...m,
+ storage_server_version: [2, 7, 0],
+ };
+ });
+ expect(() => {
+ OnionPaths.getRandomEdgeSnode(multipleMatchesSnodePool);
+ }).to.throw();
+ });
+ });
+
+ describe('pick edge snode with at least storage server v2.8.0', () => {
+ let fetchSnodePoolFromSeedNodeWithRetries: Sinon.SinonStub;
+ beforeEach(async () => {
+ // Utils Stubs
+ Sinon.stub(OnionPaths, 'selectGuardNodes').resolves(fakeGuardNodes);
+ Sinon.stub(ServiceNodesList, 'getSnodePoolFromSnode').resolves(fakeGuardNodes);
+ // we can consider that nothing is in the DB for those tests
+ stubData('getSnodePoolFromDb').resolves([]);
+
+ TestUtils.stubData('getGuardNodes').resolves(fakeGuardNodesFromDB);
+ TestUtils.stubData('createOrUpdateItem').resolves();
+ TestUtils.stubWindow('getSeedNodeList', () => ['seednode1']);
+
+ TestUtils.stubWindowLog();
+ TEST_resetState();
+
+ fetchSnodePoolFromSeedNodeWithRetries = Sinon.stub(
+ SeedNodeAPI,
+ 'fetchSnodePoolFromSeedNodeWithRetries'
+ );
+ SNodeAPI.Onions.resetSnodeFailureCount();
+ OnionPaths.resetPathFailureCount();
+ OnionPaths.clearTestOnionPath();
+ Sinon.stub(OnionPaths, 'getOnionPathMinTimeout').returns(10);
+ });
+
+ afterEach(() => {
+ Sinon.restore();
+ });
+
+ it('builds a path correctly if no issues with input', async () => {
+ fetchSnodePoolFromSeedNodeWithRetries.resolves(generateFakeSnodes(20));
+ const newOnionPath = await OnionPaths.getOnionPath({});
+ expect(newOnionPath.length).to.eq(3);
+ });
+
+ it('throws if we cannot find a valid edge snode', async () => {
+ const badPool = generateFakeSnodes(20).map(m => {
+ return { ...m, storage_server_version: [2, 1, 1] };
+ });
+ fetchSnodePoolFromSeedNodeWithRetries.reset();
+ fetchSnodePoolFromSeedNodeWithRetries.resolves(badPool);
+
+ if (OnionPaths.TEST_getTestOnionPath().length) {
+ throw new Error('expected this to be empty');
+ }
+
+ try {
+ await OnionPaths.getOnionPath({});
+
+ throw new Error('fake error');
+ } catch (e) {
+ expect(e.message).to.not.be.eq('fake error');
+ }
+ });
+
+ it('rebuild a bunch of paths and check that last snode is always >=2.8.0', async () => {
+ for (let index = 0; index < 1000; index++) {
+ // build 20 times a path and make sure that the edge snode is always with at least version 2.8.0, when half of the snodes are not upgraded
+ const pool = generateFakeSnodes(20).map((m, i) => {
+ return i % 2 === 0 ? { ...m, storage_server_version: [2, 1, 1] } : m;
+ });
+ fetchSnodePoolFromSeedNodeWithRetries.resolves(pool);
+ const newOnionPath = await OnionPaths.getOnionPath({});
+ expect(newOnionPath.length).to.eq(3);
+
+ expect(newOnionPath[2].storage_server_version).to.deep.eq([2, 8, 0]);
+ }
+ });
+ });
});
diff --git a/ts/test/session/unit/onion/SeedNodeAPI_test.ts b/ts/test/session/unit/onion/SeedNodeAPI_test.ts
index 4d2d335c8..1cd86d450 100644
--- a/ts/test/session/unit/onion/SeedNodeAPI_test.ts
+++ b/ts/test/session/unit/onion/SeedNodeAPI_test.ts
@@ -35,6 +35,7 @@ const fakeSnodePoolFromSeedNode: Array<SnodeFromSeed> = fakeSnodePool.map(m => {
storage_port: m.port,
pubkey_x25519: m.pubkey_x25519,
pubkey_ed25519: m.pubkey_ed25519,
+ storage_server_version: m.storage_server_version,
};
});
diff --git a/ts/test/test-utils/utils/pubkey.ts b/ts/test/test-utils/utils/pubkey.ts
index 617473ce2..52e522bc8 100644
--- a/ts/test/test-utils/utils/pubkey.ts
+++ b/ts/test/test-utils/utils/pubkey.ts
@@ -45,19 +45,25 @@ export function generateFakePubKeys(amount: number): Array<PubKey> {
export function generateFakeSnode(): Snode {
return {
- ip: `136.243.${Math.random() * 255}.${Math.random() * 255}`,
+ ip: `${ipv4Section()}.${ipv4Section()}.${ipv4Section()}.${ipv4Section()}`,
port: 22116,
pubkey_x25519: generateFakePubKeyStr(),
pubkey_ed25519: generateFakePubKeyStr(),
+ storage_server_version: [2, 8, 0],
};
}
+function ipv4Section() {
+ return Math.floor(Math.random() * 255);
+}
+
export function generateFakeSnodeWithEdKey(ed25519Pubkey: string): Snode {
return {
- ip: `136.243.${Math.random() * 255}.${Math.random() * 255}`,
+ ip: `${ipv4Section()}.${ipv4Section()}.${ipv4Section()}.${ipv4Section()}`,
port: 22116,
pubkey_x25519: generateFakePubKeyStr(),
pubkey_ed25519: ed25519Pubkey,
+ storage_server_version: [2, 8, 0],
};
}
From 420bd3cb8fb6c136e130cd44edcb6b57823a5ba6 Mon Sep 17 00:00:00 2001
From: Audric Ackermann <audric@loki.network>
Date: Tue, 23 Jul 2024 13:30:55 +1000
Subject: [PATCH 2/3] chore: bump version
---
package.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/package.json b/package.json
index ea9848393..d9a439abd 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
"name": "session-desktop",
"productName": "Session",
"description": "Private messaging from your desktop",
- "version": "1.12.4",
+ "version": "1.12.5",
"license": "GPL-3.0",
"author": {
"name": "Oxen Labs",
From 3133f8a51b476fe5f923e102ea56974c1fc1638a Mon Sep 17 00:00:00 2001
From: Audric Ackermann <audric@loki.network>
Date: Tue, 23 Jul 2024 13:45:24 +1000
Subject: [PATCH 3/3] chore: bump macos runner to macos-12
---
.github/workflows/build-binaries.yml | 2 +-
.github/workflows/pull-request.yml | 2 +-
.github/workflows/release.yml | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml
index de0de1338..cd6d7af06 100644
--- a/.github/workflows/build-binaries.yml
+++ b/.github/workflows/build-binaries.yml
@@ -20,7 +20,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [windows-2022, macos-11, ubuntu-20.04]
+ os: [windows-2022, macos-12, ubuntu-20.04]
env:
SIGNAL_ENV: production
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 4109fb7a5..3d2334bd7 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -18,7 +18,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [windows-2022, macos-11, ubuntu-20.04]
+ os: [windows-2022, macos-12, ubuntu-20.04]
env:
SIGNAL_ENV: production
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 56d993f0b..acb114a50 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -12,7 +12,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [windows-2022, macos-11, ubuntu-20.04]
+ os: [windows-2022, macos-12, ubuntu-20.04]
env:
SIGNAL_ENV: production
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}