|
|
|
|
@ -3,7 +3,7 @@ import pRetry from 'p-retry';
|
|
|
|
|
|
|
|
|
|
import { Data, Snode } from '../../../data/data';
|
|
|
|
|
|
|
|
|
|
import { Onions, SnodePool } from '.';
|
|
|
|
|
import { Onions } from '.';
|
|
|
|
|
import { OnionPaths } from '../../onions';
|
|
|
|
|
import { ed25519Str } from '../../onions/onionPath';
|
|
|
|
|
import { SeedNodeAPI } from '../seed_node_api';
|
|
|
|
|
@ -19,12 +19,12 @@ const minSwarmSnodeCount = 3;
|
|
|
|
|
* If we get less than minSnodePoolCount we consider that we need to fetch the new snode pool from a seed node
|
|
|
|
|
* and not from those snodes.
|
|
|
|
|
*/
|
|
|
|
|
export const minSnodePoolCount = 12;
|
|
|
|
|
const minSnodePoolCount = 12;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* If we get less than this amount of snodes (24), lets try to get an updated list from those while we can
|
|
|
|
|
*/
|
|
|
|
|
export const minSnodePoolCountBeforeRefreshFromSnodes = minSnodePoolCount * 2;
|
|
|
|
|
const minSnodePoolCountBeforeRefreshFromSnodes = minSnodePoolCount * 2;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* If we do a request to fetch nodes from snodes and they don't return at least
|
|
|
|
|
@ -32,12 +32,12 @@ export const minSnodePoolCountBeforeRefreshFromSnodes = minSnodePoolCount * 2;
|
|
|
|
|
*
|
|
|
|
|
* Too many nodes are not shared for this call to be trustworthy
|
|
|
|
|
*/
|
|
|
|
|
export const requiredSnodesForAgreement = 24;
|
|
|
|
|
const requiredSnodesForAgreement = 24;
|
|
|
|
|
|
|
|
|
|
let randomSnodePool: Array<Snode> = [];
|
|
|
|
|
|
|
|
|
|
export function TEST_resetState() {
|
|
|
|
|
randomSnodePool = [];
|
|
|
|
|
function TEST_resetState(snodePoolForTest: Array<Snode> = []) {
|
|
|
|
|
randomSnodePool = snodePoolForTest;
|
|
|
|
|
swarmCache.clear();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -49,7 +49,7 @@ const swarmCache: Map<string, Array<string>> = new Map();
|
|
|
|
|
* Use `dropSnodeFromSwarmIfNeeded` for that
|
|
|
|
|
* @param snodeEd25519 the snode ed25519 to drop from the snode pool
|
|
|
|
|
*/
|
|
|
|
|
export async function dropSnodeFromSnodePool(snodeEd25519: string) {
|
|
|
|
|
async function dropSnodeFromSnodePool(snodeEd25519: string) {
|
|
|
|
|
const exists = _.some(randomSnodePool, x => x.pubkey_ed25519 === snodeEd25519);
|
|
|
|
|
if (exists) {
|
|
|
|
|
_.remove(randomSnodePool, x => x.pubkey_ed25519 === snodeEd25519);
|
|
|
|
|
@ -67,11 +67,11 @@ export async function dropSnodeFromSnodePool(snodeEd25519: string) {
|
|
|
|
|
* excludingEd25519Snode can be used to exclude some nodes from the random list.
|
|
|
|
|
* Useful to rebuild a path excluding existing node already in a path
|
|
|
|
|
*/
|
|
|
|
|
export async function getRandomSnode(excludingEd25519Snode?: Array<string>): Promise<Snode> {
|
|
|
|
|
async function getRandomSnode(excludingEd25519Snode?: Array<string>): Promise<Snode> {
|
|
|
|
|
// make sure we have a few snodes in the pool excluding the one passed as args
|
|
|
|
|
const requiredCount = minSnodePoolCount + (excludingEd25519Snode?.length || 0);
|
|
|
|
|
const requiredCount = SnodePool.minSnodePoolCount + (excludingEd25519Snode?.length || 0);
|
|
|
|
|
if (randomSnodePool.length < requiredCount) {
|
|
|
|
|
await getSnodePoolFromDBOrFetchFromSeed(excludingEd25519Snode?.length);
|
|
|
|
|
await SnodePool.getSnodePoolFromDBOrFetchFromSeed(excludingEd25519Snode?.length);
|
|
|
|
|
|
|
|
|
|
if (randomSnodePool.length < requiredCount) {
|
|
|
|
|
window?.log?.warn(
|
|
|
|
|
@ -103,9 +103,9 @@ export async function getRandomSnode(excludingEd25519Snode?: Array<string>): Pro
|
|
|
|
|
* This function force the snode poll to be refreshed from a random seed node or snodes if we have enough of them.
|
|
|
|
|
* This should be called once in a day or so for when the app it kept on.
|
|
|
|
|
*/
|
|
|
|
|
export async function forceRefreshRandomSnodePool(): Promise<Array<Snode>> {
|
|
|
|
|
async function forceRefreshRandomSnodePool(): Promise<Array<Snode>> {
|
|
|
|
|
try {
|
|
|
|
|
await getSnodePoolFromDBOrFetchFromSeed();
|
|
|
|
|
await SnodePool.getSnodePoolFromDBOrFetchFromSeed();
|
|
|
|
|
|
|
|
|
|
window?.log?.info(
|
|
|
|
|
`forceRefreshRandomSnodePool: enough snodes to fetch from them, so we try using them ${randomSnodePool.length}`
|
|
|
|
|
@ -113,7 +113,7 @@ export async function forceRefreshRandomSnodePool(): Promise<Array<Snode>> {
|
|
|
|
|
|
|
|
|
|
// this function throws if it does not have enough snodes to do it
|
|
|
|
|
await tryToGetConsensusWithSnodesWithRetries();
|
|
|
|
|
if (randomSnodePool.length < minSnodePoolCountBeforeRefreshFromSnodes) {
|
|
|
|
|
if (randomSnodePool.length < SnodePool.minSnodePoolCountBeforeRefreshFromSnodes) {
|
|
|
|
|
throw new Error('forceRefreshRandomSnodePool still too small after refetching from snodes');
|
|
|
|
|
}
|
|
|
|
|
} catch (e) {
|
|
|
|
|
@ -140,15 +140,21 @@ export async function forceRefreshRandomSnodePool(): Promise<Array<Snode>> {
|
|
|
|
|
* Fetches from DB if snode pool is not cached, and returns it if the length is >= 12.
|
|
|
|
|
* If length is < 12, fetches from seed an updated list of snodes
|
|
|
|
|
*/
|
|
|
|
|
export async function getSnodePoolFromDBOrFetchFromSeed(
|
|
|
|
|
async function getSnodePoolFromDBOrFetchFromSeed(
|
|
|
|
|
countToAddToRequirement = 0
|
|
|
|
|
): Promise<Array<Snode>> {
|
|
|
|
|
if (randomSnodePool && randomSnodePool.length > minSnodePoolCount + countToAddToRequirement) {
|
|
|
|
|
if (
|
|
|
|
|
randomSnodePool &&
|
|
|
|
|
randomSnodePool.length > SnodePool.minSnodePoolCount + countToAddToRequirement
|
|
|
|
|
) {
|
|
|
|
|
return randomSnodePool;
|
|
|
|
|
}
|
|
|
|
|
const fetchedFromDb = await Data.getSnodePoolFromDb();
|
|
|
|
|
|
|
|
|
|
if (!fetchedFromDb || fetchedFromDb.length <= minSnodePoolCount + countToAddToRequirement) {
|
|
|
|
|
if (
|
|
|
|
|
!fetchedFromDb ||
|
|
|
|
|
fetchedFromDb.length <= SnodePool.minSnodePoolCount + countToAddToRequirement
|
|
|
|
|
) {
|
|
|
|
|
window?.log?.warn(
|
|
|
|
|
`getSnodePoolFromDBOrFetchFromSeed: not enough snodes in db (${fetchedFromDb?.length}), Fetching from seed node instead... `
|
|
|
|
|
);
|
|
|
|
|
@ -164,9 +170,9 @@ export async function getSnodePoolFromDBOrFetchFromSeed(
|
|
|
|
|
return randomSnodePool;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function getRandomSnodePool(): Promise<Array<Snode>> {
|
|
|
|
|
if (randomSnodePool.length <= minSnodePoolCount) {
|
|
|
|
|
await getSnodePoolFromDBOrFetchFromSeed();
|
|
|
|
|
async function getRandomSnodePool(): Promise<Array<Snode>> {
|
|
|
|
|
if (randomSnodePool.length <= SnodePool.minSnodePoolCount) {
|
|
|
|
|
await SnodePool.getSnodePoolFromDBOrFetchFromSeed();
|
|
|
|
|
}
|
|
|
|
|
return randomSnodePool;
|
|
|
|
|
}
|
|
|
|
|
@ -178,7 +184,7 @@ export async function getRandomSnodePool(): Promise<Array<Snode>> {
|
|
|
|
|
* This function does not throw.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
export async function TEST_fetchFromSeedWithRetriesAndWriteToDb() {
|
|
|
|
|
async function TEST_fetchFromSeedWithRetriesAndWriteToDb() {
|
|
|
|
|
const seedNodes = window.getSeedNodeList();
|
|
|
|
|
|
|
|
|
|
if (!seedNodes || !seedNodes.length) {
|
|
|
|
|
@ -217,7 +223,7 @@ async function tryToGetConsensusWithSnodesWithRetries() {
|
|
|
|
|
async () => {
|
|
|
|
|
const commonNodes = await ServiceNodesList.getSnodePoolFromSnodes();
|
|
|
|
|
|
|
|
|
|
if (!commonNodes || commonNodes.length < requiredSnodesForAgreement) {
|
|
|
|
|
if (!commonNodes || commonNodes.length < SnodePool.requiredSnodesForAgreement) {
|
|
|
|
|
// throwing makes trigger a retry if we have some left.
|
|
|
|
|
window?.log?.info(
|
|
|
|
|
`tryToGetConsensusWithSnodesWithRetries: Not enough common nodes ${commonNodes?.length}`
|
|
|
|
|
@ -252,7 +258,7 @@ async function tryToGetConsensusWithSnodesWithRetries() {
|
|
|
|
|
* @param pubkey the associatedWith publicKey
|
|
|
|
|
* @param snodeToDropEd25519 the snode pubkey to drop
|
|
|
|
|
*/
|
|
|
|
|
export async function dropSnodeFromSwarmIfNeeded(
|
|
|
|
|
async function dropSnodeFromSwarmIfNeeded(
|
|
|
|
|
pubkey: string,
|
|
|
|
|
snodeToDropEd25519: string
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
@ -261,7 +267,7 @@ export async function dropSnodeFromSwarmIfNeeded(
|
|
|
|
|
`Dropping ${ed25519Str(snodeToDropEd25519)} from swarm of ${ed25519Str(pubkey)}`
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const existingSwarm = await getSwarmFromCacheOrDb(pubkey);
|
|
|
|
|
const existingSwarm = await SnodePool.getSwarmFromCacheOrDb(pubkey);
|
|
|
|
|
|
|
|
|
|
if (!existingSwarm.includes(snodeToDropEd25519)) {
|
|
|
|
|
return;
|
|
|
|
|
@ -271,7 +277,7 @@ export async function dropSnodeFromSwarmIfNeeded(
|
|
|
|
|
await internalUpdateSwarmFor(pubkey, updatedSwarm);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function updateSwarmFor(pubkey: string, snodes: Array<Snode>): Promise<void> {
|
|
|
|
|
async function updateSwarmFor(pubkey: string, snodes: Array<Snode>): Promise<void> {
|
|
|
|
|
const edkeys = snodes.map((sn: Snode) => sn.pubkey_ed25519);
|
|
|
|
|
await internalUpdateSwarmFor(pubkey, edkeys);
|
|
|
|
|
}
|
|
|
|
|
@ -283,7 +289,7 @@ async function internalUpdateSwarmFor(pubkey: string, edkeys: Array<string>) {
|
|
|
|
|
await Data.updateSwarmNodesForPubkey(pubkey, edkeys);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function getSwarmFromCacheOrDb(pubkey: string): Promise<Array<string>> {
|
|
|
|
|
async function getSwarmFromCacheOrDb(pubkey: string): Promise<Array<string>> {
|
|
|
|
|
// NOTE: important that maybeNodes is not [] here
|
|
|
|
|
const existingCache = swarmCache.get(pubkey);
|
|
|
|
|
if (existingCache === undefined) {
|
|
|
|
|
@ -301,13 +307,12 @@ export async function getSwarmFromCacheOrDb(pubkey: string): Promise<Array<strin
|
|
|
|
|
* This call fetch from cache or db the swarm and extract only the one currently reachable.
|
|
|
|
|
* If not enough snodes valid are in the swarm, if fetches new snodes for this pubkey from the network.
|
|
|
|
|
*/
|
|
|
|
|
export async function getSwarmFor(pubkey: string): Promise<Array<Snode>> {
|
|
|
|
|
const nodes = await getSwarmFromCacheOrDb(pubkey);
|
|
|
|
|
async function getSwarmFor(pubkey: string): Promise<Array<Snode>> {
|
|
|
|
|
const nodes = await SnodePool.getSwarmFromCacheOrDb(pubkey);
|
|
|
|
|
|
|
|
|
|
// See how many are actually still reachable
|
|
|
|
|
// the nodes still reachable are the one still present in the snode pool
|
|
|
|
|
const goodNodes = randomSnodePool.filter((n: Snode) => nodes.indexOf(n.pubkey_ed25519) !== -1);
|
|
|
|
|
|
|
|
|
|
if (goodNodes.length >= minSwarmSnodeCount) {
|
|
|
|
|
return goodNodes;
|
|
|
|
|
}
|
|
|
|
|
@ -316,8 +321,8 @@ export async function getSwarmFor(pubkey: string): Promise<Array<Snode>> {
|
|
|
|
|
return getSwarmFromNetworkAndSave(pubkey);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function getNodeFromSwarmOrThrow(pubkey: string): Promise<Snode> {
|
|
|
|
|
const swarm = await getSwarmFor(pubkey);
|
|
|
|
|
async function getNodeFromSwarmOrThrow(pubkey: string): Promise<Snode> {
|
|
|
|
|
const swarm = await SnodePool.getSwarmFor(pubkey);
|
|
|
|
|
if (!isEmpty(swarm)) {
|
|
|
|
|
const node = sample(swarm);
|
|
|
|
|
if (node) {
|
|
|
|
|
@ -336,7 +341,7 @@ export async function getNodeFromSwarmOrThrow(pubkey: string): Promise<Snode> {
|
|
|
|
|
* @param pubkey the pubkey to request the swarm for
|
|
|
|
|
* @returns the fresh swarm, shuffled
|
|
|
|
|
*/
|
|
|
|
|
export async function getFreshSwarmFor(pubkey: string): Promise<Array<Snode>> {
|
|
|
|
|
async function getFreshSwarmFor(pubkey: string): Promise<Array<Snode>> {
|
|
|
|
|
return getSwarmFromNetworkAndSave(pubkey);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -350,3 +355,29 @@ async function getSwarmFromNetworkAndSave(pubkey: string) {
|
|
|
|
|
|
|
|
|
|
return shuffledSwarm;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export const SnodePool = {
|
|
|
|
|
// consts
|
|
|
|
|
minSnodePoolCount,
|
|
|
|
|
minSnodePoolCountBeforeRefreshFromSnodes,
|
|
|
|
|
requiredSnodesForAgreement,
|
|
|
|
|
|
|
|
|
|
// snode pool mgmt
|
|
|
|
|
dropSnodeFromSnodePool,
|
|
|
|
|
forceRefreshRandomSnodePool,
|
|
|
|
|
getRandomSnode,
|
|
|
|
|
getRandomSnodePool,
|
|
|
|
|
getSnodePoolFromDBOrFetchFromSeed,
|
|
|
|
|
|
|
|
|
|
// swarm mgmt
|
|
|
|
|
dropSnodeFromSwarmIfNeeded,
|
|
|
|
|
updateSwarmFor,
|
|
|
|
|
getSwarmFromCacheOrDb,
|
|
|
|
|
getSwarmFor,
|
|
|
|
|
getNodeFromSwarmOrThrow,
|
|
|
|
|
getFreshSwarmFor,
|
|
|
|
|
|
|
|
|
|
// tests
|
|
|
|
|
TEST_resetState,
|
|
|
|
|
TEST_fetchFromSeedWithRetriesAndWriteToDb,
|
|
|
|
|
};
|
|
|
|
|
|
Audric Ackermann
1 year ago