From 0a208c0d1534bfb76e0f361b44afdb47683ff1e2 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Tue, 8 Jun 2021 14:35:30 +1000 Subject: [PATCH] store snodes list on db, use them if possible on app start if not enough snodes or no snodes at all, a request to the seed node is made instead --- ts/data/data.ts | 28 ++++++- ts/receiver/attachments.ts | 1 + ts/session/onions/onionPath.ts | 18 ++-- ts/session/onions/onionSend.ts | 2 +- ts/session/snode_api/SNodeAPI.ts | 3 +- ts/session/snode_api/lokiRpc.ts | 3 +- ts/session/snode_api/onions.ts | 11 +-- ts/session/snode_api/snodePool.ts | 84 +++++++++++-------- ts/session/snode_api/swarmPolling.ts | 3 +- ts/session/utils/Promise.ts | 3 +- .../session/unit/onion/OnionErrors_test.ts | 22 ++--- 11 files changed, 106 insertions(+), 72 deletions(-) diff --git a/ts/data/data.ts b/ts/data/data.ts index 6f806e3d6..3f2c07253 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -46,12 +46,15 @@ export type GuardNode = { ed25519PubKey: string; }; -export type SwarmNode = { - address: string; +export interface Snode { ip: string; - port: string; - pubkey_ed25519: string; + port: number; pubkey_x25519: string; + pubkey_ed25519: string; +} + +export type SwarmNode = Snode & { + address: string; }; export type ServerToken = { @@ -976,3 +979,20 @@ export async function getMessagesWithFileAttachments( limit: options?.limit, }); } + +export const SNODE_POOL_ITEM_ID = 'SNODE_POOL_ITEM_ID'; + +export async function getSnodePoolFromDb(): Promise | null> { + // this is currently all stored as a big string as we don't really need to do anything with them (no filtering or anything) + // everything is made in memory and written to disk + const snodesJson = await exports.getItemById(SNODE_POOL_ITEM_ID); + if (!snodesJson || !snodesJson.value) { + return null; + } + + return JSON.parse(snodesJson.value); +} + +export async function updateSnodePoolOnDb(snodesAsJsonString: string): Promise { + await exports.createOrUpdateItem({ id: SNODE_POOL_ITEM_ID, value: snodesAsJsonString }); +} diff --git a/ts/receiver/attachments.ts b/ts/receiver/attachments.ts index 46858777e..1822c6361 100644 --- a/ts/receiver/attachments.ts +++ b/ts/receiver/attachments.ts @@ -296,6 +296,7 @@ export async function queueAttachmentDownloads( count += await processQuoteAttachments(message, conversation); + // I don 't think we rely on this for anything if (await processGroupAvatar(message, conversation)) { count += 1; } diff --git a/ts/session/onions/onionPath.ts b/ts/session/onions/onionPath.ts index 49cc8f89d..676d08dde 100644 --- a/ts/session/onions/onionPath.ts +++ b/ts/session/onions/onionPath.ts @@ -1,4 +1,4 @@ -import { getGuardNodes, updateGuardNodes } from '../../../ts/data/data'; +import { getGuardNodes, Snode, updateGuardNodes } from '../../../ts/data/data'; import * as SnodePool from '../snode_api/snodePool'; import _ from 'lodash'; import { default as insecureNodeFetch } from 'node-fetch'; @@ -9,7 +9,7 @@ import { allowOnlyOneAtATime } from '../utils/Promise'; const desiredGuardCount = 3; const minimumGuardCount = 2; -export type SnodePath = Array; +export type SnodePath = Array; const onionRequestHops = 3; let onionPaths: Array = []; @@ -56,7 +56,7 @@ const pathFailureThreshold = 3; // This array is meant to store nodes will full info, // so using GuardNode would not be correct (there is // some naming issue here it seems) -let guardNodes: Array = []; +let guardNodes: Array = []; export const ed25519Str = (ed25519Key: string) => `(...${ed25519Key.substr(58)})`; @@ -119,7 +119,7 @@ export async function dropSnodeFromPath(snodeEd25519: string) { onionPaths[pathWithSnodeIndex] = pathtoPatchUp; } -export async function getOnionPath(toExclude?: SnodePool.Snode): Promise> { +export async function getOnionPath(toExclude?: Snode): Promise> { let attemptNumber = 0; while (onionPaths.length < minimumGuardCount) { window?.log?.error( @@ -215,14 +215,14 @@ async function dropPathStartingWithGuardNode(guardNodeEd25519: string) { guardNodes = guardNodes.filter(g => g.pubkey_ed25519 !== guardNodeEd25519); pathFailureCount[guardNodeEd25519] = 0; - SnodePool.dropSnodeFromSnodePool(guardNodeEd25519); + await SnodePool.dropSnodeFromSnodePool(guardNodeEd25519); // write the updates guard nodes to the db. // the next call to getOnionPath will trigger a rebuild of the path await updateGuardNodes(edKeys); } -async function testGuardNode(snode: SnodePool.Snode) { +async function testGuardNode(snode: Snode) { window?.log?.info(`Testing a candidate guard node ${ed25519Str(snode.pubkey_ed25519)}`); // Send a post request and make sure it is OK @@ -276,7 +276,7 @@ async function testGuardNode(snode: SnodePool.Snode) { /** * Only exported for testing purpose. DO NOT use this directly */ -export async function selectGuardNodes(): Promise> { +export async function selectGuardNodes(): Promise> { // `getRandomSnodePool` is expected to refresh itself on low nodes const nodePool = await SnodePool.getRandomSnodePool(); if (nodePool.length < desiredGuardCount) { @@ -289,7 +289,7 @@ export async function selectGuardNodes(): Promise> { const shuffled = _.shuffle(nodePool); - let selectedGuardNodes: Array = []; + let selectedGuardNodes: Array = []; // The use of await inside while is intentional: // we only want to repeat if the await fails @@ -308,7 +308,7 @@ export async function selectGuardNodes(): Promise> { const goodNodes = _.zip(idxOk, candidateNodes) .filter(x => x[0]) - .map(x => x[1]) as Array; + .map(x => x[1]) as Array; selectedGuardNodes = _.concat(selectedGuardNodes, goodNodes); } diff --git a/ts/session/onions/onionSend.ts b/ts/session/onions/onionSend.ts index c6e3cddde..650a5b07d 100644 --- a/ts/session/onions/onionSend.ts +++ b/ts/session/onions/onionSend.ts @@ -7,12 +7,12 @@ import { snodeHttpsAgent, SnodeResponse, } from '../snode_api/onions'; -import { Snode } from '../snode_api/snodePool'; import _, { toNumber } from 'lodash'; import { default as insecureNodeFetch } from 'node-fetch'; import { PROTOCOLS } from '../constants'; import { toHex } from '../utils/String'; import pRetry from 'p-retry'; +import { Snode } from '../../data/data'; // FIXME audric we should soon be able to get rid of that const FILESERVER_HOSTS = [ diff --git a/ts/session/snode_api/SNodeAPI.ts b/ts/session/snode_api/SNodeAPI.ts index 9ab954fa3..3906d614e 100644 --- a/ts/session/snode_api/SNodeAPI.ts +++ b/ts/session/snode_api/SNodeAPI.ts @@ -11,7 +11,7 @@ const { remote } = Electron; import { snodeRpc } from './lokiRpc'; -import { getRandomSnode, getRandomSnodePool, requiredSnodesForAgreement, Snode } from './snodePool'; +import { getRandomSnode, getRandomSnodePool, requiredSnodesForAgreement } from './snodePool'; import { Constants } from '..'; import { getSodium, sha256 } from '../crypto'; import _, { range } from 'lodash'; @@ -23,6 +23,7 @@ import { stringToUint8Array, toHex, } from '../utils/String'; +import { Snode } from '../../data/data'; // ONS name can have [a-zA-Z0-9_-] except that - is not allowed as start or end // do not define a regex but rather create it on the fly to avoid https://stackoverflow.com/questions/3891641/regex-test-only-works-every-other-time diff --git a/ts/session/snode_api/lokiRpc.ts b/ts/session/snode_api/lokiRpc.ts index 77a26b715..53d3088f5 100644 --- a/ts/session/snode_api/lokiRpc.ts +++ b/ts/session/snode_api/lokiRpc.ts @@ -1,6 +1,5 @@ import { default as insecureNodeFetch } from 'node-fetch'; - -import { Snode } from './snodePool'; +import { Snode } from '../../data/data'; import { lokiOnionFetch, snodeHttpsAgent, SnodeResponse } from './onions'; diff --git a/ts/session/snode_api/onions.ts b/ts/session/snode_api/onions.ts index 21759ba63..ea7634af9 100644 --- a/ts/session/snode_api/onions.ts +++ b/ts/session/snode_api/onions.ts @@ -1,12 +1,7 @@ import { default as insecureNodeFetch } from 'node-fetch'; import https from 'https'; -import { - dropSnodeFromSnodePool, - dropSnodeFromSwarmIfNeeded, - Snode, - updateSwarmFor, -} from './snodePool'; +import { dropSnodeFromSnodePool, dropSnodeFromSwarmIfNeeded, updateSwarmFor } from './snodePool'; import ByteBuffer from 'bytebuffer'; import { OnionPaths } from '../onions'; import { fromBase64ToArrayBuffer, toHex } from '../utils/String'; @@ -17,6 +12,8 @@ import { hrefPnServerDev, hrefPnServerProd } from '../../pushnotification/PnServ // hold the ed25519 key of a snode against the time it fails. Used to remove a snode only after a few failures (snodeFailureThreshold failures) let snodeFailureCount: Record = {}; +import { Snode } from '../../data/data'; + // tslint:disable-next-line: variable-name export const TEST_resetSnodeFailureCount = () => { snodeFailureCount = {}; @@ -590,7 +587,7 @@ export async function incrementBadSnodeCountOrDrop({ } window?.log?.info(`Dropping ${snodeEd25519} from snodepool`); - dropSnodeFromSnodePool(snodeEd25519); + await dropSnodeFromSnodePool(snodeEd25519); // the snode was ejected from the pool so it won't be used again. // in case of snode pool refresh, we need to be able to try to contact this node again so reset its failure count to 0. snodeFailureCount[snodeEd25519] = 0; diff --git a/ts/session/snode_api/snodePool.ts b/ts/session/snode_api/snodePool.ts index 911c3493f..f99d11fe8 100644 --- a/ts/session/snode_api/snodePool.ts +++ b/ts/session/snode_api/snodePool.ts @@ -28,16 +28,8 @@ const minSnodePoolCount = 12; */ export const requiredSnodesForAgreement = 24; -export interface Snode { - ip: string; - port: number; - pubkey_x25519: string; - pubkey_ed25519: string; - version: string; -} - // This should be renamed to `allNodes` or something -let randomSnodePool: Array = []; +let randomSnodePool: Array = []; // We only store nodes' identifiers here, const swarmCache: Map> = new Map(); @@ -48,7 +40,9 @@ export type SeedNode = { }; // just get the filtered list -async function tryGetSnodeListFromLokidSeednode(seedNodes: Array): Promise> { +async function tryGetSnodeListFromLokidSeednode( + seedNodes: Array +): Promise> { window?.log?.info('tryGetSnodeListFromLokidSeednode starting...'); if (!seedNodes.length) { @@ -111,10 +105,11 @@ async function tryGetSnodeListFromLokidSeednode(seedNodes: Array): Pro * Use `dropSnodeFromSwarmIfNeeded` for that * @param snodeEd25519 the snode ed25519 to drop from the snode pool */ -export function dropSnodeFromSnodePool(snodeEd25519: string) { +export async function dropSnodeFromSnodePool(snodeEd25519: string) { const exists = _.some(randomSnodePool, x => x.pubkey_ed25519 === snodeEd25519); if (exists) { _.remove(randomSnodePool, x => x.pubkey_ed25519 === snodeEd25519); + await Data.updateSnodePoolOnDb(JSON.stringify(randomSnodePool)); window?.log?.warn( `Marking ${ed25519Str(snodeEd25519)} as unreachable, ${ @@ -128,7 +123,7 @@ export function dropSnodeFromSnodePool(snodeEd25519: string) { * * @param excluding can be used to exclude some nodes from the random list. Useful to rebuild a path excluding existing node already in a path */ -export async function getRandomSnode(excludingEd25519Snode?: Array): Promise { +export async function getRandomSnode(excludingEd25519Snode?: Array): Promise { // resolve random snode if (randomSnodePool.length === 0) { // Should not this be saved to the database? @@ -140,7 +135,7 @@ export async function getRandomSnode(excludingEd25519Snode?: Array): Pro } // We know the pool can't be empty at this point if (!excludingEd25519Snode) { - return _.sample(randomSnodePool) as Snode; + return _.sample(randomSnodePool) as Data.Snode; } // we have to double check even after removing the nodes to exclude we still have some nodes in the list @@ -157,42 +152,37 @@ export async function getRandomSnode(excludingEd25519Snode?: Array): Pro // used for tests throw new Error('SeedNodeError'); } - return _.sample(snodePoolExcluding) as Snode; + return _.sample(snodePoolExcluding) as Data.Snode; } /** * This function force the snode poll to be refreshed from a random seed node again. * This should be called once in a day or so for when the app it kept on. */ -export async function forceRefreshRandomSnodePool(): Promise> { - await refreshRandomPool(); +export async function forceRefreshRandomSnodePool(): Promise> { + await refreshRandomPool(true); return randomSnodePool; } -export async function getRandomSnodePool(): Promise> { +export async function getRandomSnodePool(): Promise> { if (randomSnodePool.length === 0) { await refreshRandomPool(); } return randomSnodePool; } -// not cacheable because we write to this.randomSnodePool elsewhere -export function getNodesMinVersion(minVersion: string): Array { - return randomSnodePool.filter((node: any) => node.version && semver.gt(node.version, minVersion)); -} - async function getSnodeListFromLokidSeednode( seedNodes: Array, retries = 0 -): Promise> { +): Promise> { const SEED_NODE_RETRIES = 3; window?.log?.info('getSnodeListFromLokidSeednode starting...'); if (!seedNodes.length) { window?.log?.info('loki_snode_api::getSnodeListFromLokidSeednode - seedNodes are empty'); return []; } - let snodes: Array = []; + let snodes: Array = []; try { snodes = await tryGetSnodeListFromLokidSeednode(seedNodes); } catch (e) { @@ -225,7 +215,9 @@ async function getSnodeListFromLokidSeednode( * Exported only for tests. This is not to be used by the app directly * @param seedNodes the seednodes to use to fetch snodes details */ -export async function refreshRandomPoolDetail(seedNodes: Array): Promise> { +export async function refreshRandomPoolDetail( + seedNodes: Array +): Promise> { let snodes = []; try { window?.log?.info(`refreshRandomPoolDetail with seedNodes.length ${seedNodes.length}`); @@ -265,8 +257,10 @@ export async function refreshRandomPoolDetail(seedNodes: Array): Promi * This function runs only once at a time, and fetches the snode pool from a random seed node, * or if we have enough snodes, fetches the snode pool from one of the snode. */ -export async function refreshRandomPool(): Promise { - if (!window.getSeedNodeList() || !window.getSeedNodeList()?.length) { +export async function refreshRandomPool(forceRefresh = false): Promise { + const seedNodes = window.getSeedNodeList(); + + if (!seedNodes || !seedNodes.length) { window?.log?.error( 'LokiSnodeAPI:::refreshRandomPool - getSeedNodeList has not been loaded yet' ); @@ -274,12 +268,30 @@ export async function refreshRandomPool(): Promise { return; } // tslint:disable-next-line:no-parameter-reassignment - const seedNodes = window.getSeedNodeList(); window?.log?.info("right before allowOnlyOneAtATime 'refreshRandomPool'"); return allowOnlyOneAtATime('refreshRandomPool', async () => { window?.log?.info("running allowOnlyOneAtATime 'refreshRandomPool'"); + // if we have forceRefresh set, we want to request snodes from snodes or from the seed server. + if (randomSnodePool.length === 0 && !forceRefresh) { + const fetchedFromDb = await Data.getSnodePoolFromDb(); + // write to memory only if it is valid. + // if the size is not enough. we will contact a seed node. + if (fetchedFromDb?.length) { + window?.log?.info(`refreshRandomPool: fetched from db ${fetchedFromDb.length} snodes.`); + randomSnodePool = fetchedFromDb; + + if (randomSnodePool.length < minSnodePoolCount) { + window?.log?.warn('refreshRandomPool: not enough snodes in db, going to fetch from seed'); + } + + return; + } else { + window?.log?.warn('refreshRandomPool: did not find snodes in db.'); + } + } + // we don't have nodes to fetch the pool from them, so call the seed node instead. if (randomSnodePool.length < minSnodePoolCount) { window?.log?.info( @@ -287,7 +299,7 @@ export async function refreshRandomPool(): Promise { ); randomSnodePool = await exports.refreshRandomPoolDetail(seedNodes); - + await Data.updateSnodePoolOnDb(JSON.stringify(randomSnodePool)); return; } try { @@ -308,6 +320,7 @@ export async function refreshRandomPool(): Promise { } window?.log?.info('updating snode list with snode pool length:', commonNodes.length); randomSnodePool = commonNodes; + await Data.updateSnodePoolOnDb(JSON.stringify(randomSnodePool)); }, { retries: 3, @@ -328,6 +341,7 @@ export async function refreshRandomPool(): Promise { // fallback to a seed node fetch of the snode pool randomSnodePool = await exports.refreshRandomPoolDetail(seedNodes); + await Data.updateSnodePoolOnDb(JSON.stringify(randomSnodePool)); } }); } @@ -352,8 +366,8 @@ export async function dropSnodeFromSwarmIfNeeded( await internalUpdateSwarmFor(pubkey, updatedSwarm); } -export async function updateSwarmFor(pubkey: string, snodes: Array): Promise { - const edkeys = snodes.map((sn: Snode) => sn.pubkey_ed25519); +export async function updateSwarmFor(pubkey: string, snodes: Array): Promise { + const edkeys = snodes.map((sn: Data.Snode) => sn.pubkey_ed25519); await internalUpdateSwarmFor(pubkey, edkeys); } @@ -382,12 +396,14 @@ export async function getSwarmFromCacheOrDb(pubkey: string): Promise> { +export async function getSwarmFor(pubkey: string): Promise> { const nodes = await getSwarmFromCacheOrDb(pubkey); // See how many are actually still reachable // the nodes still reachable are the one still present in the snode pool - const goodNodes = randomSnodePool.filter((n: Snode) => nodes.indexOf(n.pubkey_ed25519) !== -1); + const goodNodes = randomSnodePool.filter( + (n: Data.Snode) => nodes.indexOf(n.pubkey_ed25519) !== -1 + ); if (goodNodes.length >= minSwarmSnodeCount) { return goodNodes; @@ -396,7 +412,7 @@ export async function getSwarmFor(pubkey: string): Promise> { // Request new node list from the network const freshNodes = _.shuffle(await requestSnodesForPubkey(pubkey)); - const edkeys = freshNodes.map((n: Snode) => n.pubkey_ed25519); + const edkeys = freshNodes.map((n: Data.Snode) => n.pubkey_ed25519); await internalUpdateSwarmFor(pubkey, edkeys); return freshNodes; diff --git a/ts/session/snode_api/swarmPolling.ts b/ts/session/snode_api/swarmPolling.ts index 0f5ff2d43..839d8cd22 100644 --- a/ts/session/snode_api/swarmPolling.ts +++ b/ts/session/snode_api/swarmPolling.ts @@ -1,5 +1,5 @@ import { PubKey } from '../types'; -import { getSwarmFor, Snode } from './snodePool'; +import { getSwarmFor } from './snodePool'; import { retrieveNextMessages } from './SNodeAPI'; import { SignalService } from '../../protobuf'; import * as Receiver from '../../receiver/receiver'; @@ -8,6 +8,7 @@ import { getLastHashBySnode, getSeenMessagesByHashList, saveSeenMessageHashes, + Snode, updateLastHash, } from '../../../ts/data/data'; diff --git a/ts/session/utils/Promise.ts b/ts/session/utils/Promise.ts index 1642ceda6..8d1ec73b9 100644 --- a/ts/session/utils/Promise.ts +++ b/ts/session/utils/Promise.ts @@ -1,5 +1,4 @@ -import { Snode } from '../snode_api/snodePool'; - +import { Snode } from '../../data/data'; type SimpleFunction = (arg: T) => void; type Return = Promise | T; diff --git a/ts/test/session/unit/onion/OnionErrors_test.ts b/ts/test/session/unit/onion/OnionErrors_test.ts index 4b53809f4..1227e4441 100644 --- a/ts/test/session/unit/onion/OnionErrors_test.ts +++ b/ts/test/session/unit/onion/OnionErrors_test.ts @@ -17,12 +17,7 @@ import { } from '../../../../session/snode_api/onions'; import AbortController from 'abort-controller'; import * as Data from '../../../../../ts/data/data'; -import { Snode } from '../../../../session/snode_api/snodePool'; -import { - pathFailureCount, - SnodePath, - TEST_getTestguardNodes, -} from '../../../../session/onions/onionPath'; +import { pathFailureCount, SnodePath } from '../../../../session/onions/onionPath'; chai.use(chaiAsPromised as any); chai.should(); @@ -60,10 +55,10 @@ describe('OnionPathsErrors', () => { // tslint:disable-next-line: one-variable-per-declaration let guardPubkeys: Array, otherNodesPubkeys: Array, - guardNodesArray: Array, - guardSnode1: Snode, - otherNodesArray: Array, - fakeSnodePool: Array, + guardNodesArray: Array, + guardSnode1: Data.Snode, + otherNodesArray: Array, + fakeSnodePool: Array, associatedWith: string, fakeSwarmForAssociatedWith: Array; @@ -119,6 +114,11 @@ describe('OnionPathsErrors', () => { // those are still doing what they do, but we spy on their executation updateSwarmSpy = sandbox.stub(Data, 'updateSwarmNodesForPubkey').resolves(); + sandbox + .stub(Data, 'getItemById') + .withArgs(Data.SNODE_POOL_ITEM_ID) + .resolves({ id: Data.SNODE_POOL_ITEM_ID, value: '' }); + sandbox.stub(Data, 'createOrUpdateItem').resolves(); dropSnodeFromSnodePool = sandbox.spy(SNodeAPI.SnodePool, 'dropSnodeFromSnodePool'); dropSnodeFromSwarmIfNeededSpy = sandbox.spy(SNodeAPI.SnodePool, 'dropSnodeFromSwarmIfNeeded'); dropSnodeFromPathSpy = sandbox.spy(OnionPaths, 'dropSnodeFromPath'); @@ -296,7 +296,7 @@ describe('OnionPathsErrors', () => { it('throws a non-retryable error we get a 421 status code with a new swarm', async () => { const targetNode = otherNodesPubkeys[0]; - const resultExpected: Array = [ + const resultExpected: Array = [ otherNodesArray[4], otherNodesArray[5], otherNodesArray[6],