From 7f7f0fe26cf5f779ed0db62cf9a4969f75d13ae3 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Wed, 24 Apr 2024 13:57:47 +1000 Subject: [PATCH] fix: clear swarms from snodes not in pool on full fetch --- .gitignore | 2 ++ package.json | 12 ++++---- ts/components/calling/IncomingCallDialog.tsx | 2 +- ts/components/dialog/DeleteAccountModal.tsx | 3 +- ts/data/data.ts | 5 ++++ ts/data/dataInit.ts | 1 + .../conversations/unsendingInteractions.ts | 2 +- ts/models/conversation.ts | 3 +- ts/node/sql.ts | 29 +++++++++++++++++++ ts/session/apis/snode_api/SNodeAPI.ts | 3 +- ts/session/apis/snode_api/onions.ts | 4 +-- ts/session/apis/snode_api/retrieveRequest.ts | 2 +- ts/session/apis/snode_api/snodePool.ts | 15 +++++++++- ts/session/apis/snode_api/swarmPolling.ts | 2 +- ts/session/onions/onionPath.ts | 3 +- ts/session/sending/MessageSender.ts | 3 +- ts/session/utils/String.ts | 2 ++ ts/session/utils/calling/CallManager.ts | 8 +++-- 18 files changed, 77 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 27cf464db..46770cfbb 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ stylesheets/dist/ *.LICENSE.txt ts/webworker/workers/node/**/*.node +.yarn/**/*.mjs +.yarn/**/*.cjs diff --git a/package.json b/package.json index 7c6849700..acb4abb61 100644 --- a/package.json +++ b/package.json @@ -216,11 +216,13 @@ "afterSign": "build/notarize.js", "afterPack": "build/afterPackHook.js", "artifactName": "${name}-${os}-${arch}-${version}.${ext}", - "extraResources": [{ - "from": "./build/launcher-script.sh", - "to": "./launcher-script.sh" - }, - "mmdb/GeoLite2-Country.mmdb"], + "extraResources": [ + { + "from": "./build/launcher-script.sh", + "to": "./launcher-script.sh" + }, + "mmdb/GeoLite2-Country.mmdb" + ], "mac": { "category": "public.app-category.social-networking", "icon": "build/icon-mac.icns", diff --git a/ts/components/calling/IncomingCallDialog.tsx b/ts/components/calling/IncomingCallDialog.tsx index ccaba5c46..a3ec501d5 100644 --- a/ts/components/calling/IncomingCallDialog.tsx +++ b/ts/components/calling/IncomingCallDialog.tsx @@ -3,13 +3,13 @@ import { useSelector } from 'react-redux'; import styled from 'styled-components'; import { useConversationUsername } from '../../hooks/useParamSelector'; -import { ed25519Str } from '../../session/onions/onionPath'; import { CallManager } from '../../session/utils'; import { callTimeoutMs } from '../../session/utils/calling/CallManager'; import { getHasIncomingCall, getHasIncomingCallFrom } from '../../state/selectors/call'; import { Avatar, AvatarSize } from '../avatar/Avatar'; import { SessionButton, SessionButtonColor, SessionButtonType } from '../basic/SessionButton'; import { SessionWrapperModal } from '../SessionWrapperModal'; +import { ed25519Str } from '../../session/utils/String'; export const CallWindow = styled.div` position: absolute; diff --git a/ts/components/dialog/DeleteAccountModal.tsx b/ts/components/dialog/DeleteAccountModal.tsx index b11dd3516..3f394850c 100644 --- a/ts/components/dialog/DeleteAccountModal.tsx +++ b/ts/components/dialog/DeleteAccountModal.tsx @@ -1,7 +1,7 @@ import React, { useCallback, useState } from 'react'; import { useDispatch } from 'react-redux'; import { SnodeAPI } from '../../session/apis/snode_api/SNodeAPI'; -import { ed25519Str } from '../../session/onions/onionPath'; + import { forceSyncConfigurationNowIfNeeded } from '../../session/utils/sync/syncUtils'; import { updateConfirmModal, updateDeleteAccountModal } from '../../state/ducks/modalDialog'; import { SessionWrapperModal } from '../SessionWrapperModal'; @@ -14,6 +14,7 @@ import { deleteAllLogs } from '../../node/logs'; import { clearInbox } from '../../session/apis/open_group_api/sogsv3/sogsV3ClearInbox'; import { getAllValidOpenGroupV2ConversationRoomInfos } from '../../session/apis/open_group_api/utils/OpenGroupUtils'; import { SessionRadioGroup } from '../basic/SessionRadioGroup'; +import { ed25519Str } from '../../session/utils/String'; const deleteDbLocally = async () => { window?.log?.info('last message sent successfully. Deleting everything'); diff --git a/ts/data/data.ts b/ts/data/data.ts index e5190e997..ed585f7f3 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -109,6 +109,10 @@ async function updateSwarmNodesForPubkey( await channels.updateSwarmNodesForPubkey(pubkey, snodeEdKeys); } +async function clearOutAllSnodesNotInPool(edKeysOfSnodePool: Array): Promise { + await channels.clearOutAllSnodesNotInPool(edKeysOfSnodePool); +} + // Closed group /** @@ -802,6 +806,7 @@ export const Data = { generateAttachmentKeyIfEmpty, getSwarmNodesForPubkey, updateSwarmNodesForPubkey, + clearOutAllSnodesNotInPool, getAllEncryptionKeyPairsForGroup, getLatestClosedGroupEncryptionKeyPair, addClosedGroupEncryptionKeyPair, diff --git a/ts/data/dataInit.ts b/ts/data/dataInit.ts index abf1c3f43..3577c9f54 100644 --- a/ts/data/dataInit.ts +++ b/ts/data/dataInit.ts @@ -24,6 +24,7 @@ const channelsToMake = new Set([ 'removeItemById', 'getSwarmNodesForPubkey', 'updateSwarmNodesForPubkey', + 'clearOutAllSnodesNotInPool', 'saveConversation', 'fetchConvoMemoryDetails', 'getConversationById', diff --git a/ts/interactions/conversations/unsendingInteractions.ts b/ts/interactions/conversations/unsendingInteractions.ts index 828c05903..fae67ea3a 100644 --- a/ts/interactions/conversations/unsendingInteractions.ts +++ b/ts/interactions/conversations/unsendingInteractions.ts @@ -9,12 +9,12 @@ import { SnodeAPI } from '../../session/apis/snode_api/SNodeAPI'; import { SnodeNamespaces } from '../../session/apis/snode_api/namespaces'; import { getConversationController } from '../../session/conversations'; import { UnsendMessage } from '../../session/messages/outgoing/controlMessage/UnsendMessage'; -import { ed25519Str } from '../../session/onions/onionPath'; import { PubKey } from '../../session/types'; import { ToastUtils, UserUtils } from '../../session/utils'; import { closeRightPanel, resetSelectedMessageIds } from '../../state/ducks/conversations'; import { updateConfirmModal } from '../../state/ducks/modalDialog'; import { resetRightOverlayMode } from '../../state/ducks/section'; +import { ed25519Str } from '../../session/utils/String'; /** * Deletes messages for everyone in a 1-1 or everyone in a closed group conversation. diff --git a/ts/models/conversation.ts b/ts/models/conversation.ts index 096fd86d4..794a8c67e 100644 --- a/ts/models/conversation.ts +++ b/ts/models/conversation.ts @@ -42,7 +42,7 @@ import { VisibleMessageParams, } from '../session/messages/outgoing/visibleMessage/VisibleMessage'; import { perfEnd, perfStart } from '../session/utils/Performance'; -import { toHex } from '../session/utils/String'; +import { ed25519Str, toHex } from '../session/utils/String'; import { createTaskWithTimeout } from '../session/utils/TaskWithTimeout'; import { actions as conversationActions, @@ -74,7 +74,6 @@ import { MessageRequestResponse, MessageRequestResponseParams, } from '../session/messages/outgoing/controlMessage/MessageRequestResponse'; -import { ed25519Str } from '../session/onions/onionPath'; import { ConfigurationSync } from '../session/utils/job_runners/jobs/ConfigurationSyncJob'; import { SessionUtilContact } from '../session/utils/libsession/libsession_utils_contacts'; import { SessionUtilConvoInfoVolatile } from '../session/utils/libsession/libsession_utils_convo_info_volatile'; diff --git a/ts/node/sql.ts b/ts/node/sql.ts index 52323b03a..80fcecbfd 100644 --- a/ts/node/sql.ts +++ b/ts/node/sql.ts @@ -12,6 +12,7 @@ import { differenceBy, forEach, fromPairs, + intersection, isArray, isEmpty, isNumber, @@ -78,6 +79,7 @@ import { initDbInstanceWith, isInstanceInitialized, } from './sqlInstance'; +import { ed25519Str } from '../session/utils/String'; // eslint:disable: function-name non-literal-fs-path @@ -398,6 +400,32 @@ function updateSwarmNodesForPubkey(pubkey: string, snodeEdKeys: Array) { }); } +function clearOutAllSnodesNotInPool(edKeysOfSnodePool: Array) { + const allSwarms = assertGlobalInstance() + .prepare(`SELECT * FROM ${NODES_FOR_PUBKEY_TABLE};`) + .all(); + + allSwarms.forEach(swarm => { + try { + const json = JSON.parse(swarm.json); + if (isArray(json)) { + const intersect = intersection(json, edKeysOfSnodePool); + if (intersect.length !== json.length) { + updateSwarmNodesForPubkey(swarm.pubkey, intersect); + console.info( + `clearOutAllSnodesNotInPool: updating swarm of ${ed25519Str(swarm.pubkey)} to `, + intersect + ); + } + } + } catch (e) { + console.warn( + `Failed to parse swarm while iterating in clearOutAllSnodesNotInPool for pk: ${ed25519Str(swarm?.pubkey)}` + ); + } + }); +} + function getConversationCount() { const row = assertGlobalInstance().prepare(`SELECT count(*) from ${CONVERSATIONS_TABLE};`).get(); if (!row) { @@ -2449,6 +2477,7 @@ export const sqlNode = { getSwarmNodesForPubkey, updateSwarmNodesForPubkey, + clearOutAllSnodesNotInPool, getGuardNodes, updateGuardNodes, diff --git a/ts/session/apis/snode_api/SNodeAPI.ts b/ts/session/apis/snode_api/SNodeAPI.ts index 2eeefde01..91f214e7c 100644 --- a/ts/session/apis/snode_api/SNodeAPI.ts +++ b/ts/session/apis/snode_api/SNodeAPI.ts @@ -4,9 +4,8 @@ import { compact, sample } from 'lodash'; import pRetry from 'p-retry'; import { Snode } from '../../../data/data'; import { getSodiumRenderer } from '../../crypto'; -import { ed25519Str } from '../../onions/onionPath'; import { StringUtils, UserUtils } from '../../utils'; -import { fromBase64ToArray, fromHexToArray } from '../../utils/String'; +import { ed25519Str, fromBase64ToArray, fromHexToArray } from '../../utils/String'; import { doSnodeBatchRequest } from './batchRequest'; import { getSwarmFor } from './snodePool'; import { SnodeSignature } from './snodeSignatures'; diff --git a/ts/session/apis/snode_api/onions.ts b/ts/session/apis/snode_api/onions.ts index 1478bce85..4f2c1750a 100644 --- a/ts/session/apis/snode_api/onions.ts +++ b/ts/session/apis/snode_api/onions.ts @@ -11,8 +11,8 @@ import { AbortSignal as AbortSignalNode } from 'node-fetch/externals'; import { dropSnodeFromSnodePool, dropSnodeFromSwarmIfNeeded, updateSwarmFor } from './snodePool'; import { OnionPaths } from '../../onions'; -import { ed25519Str, incrementBadPathCountOrDrop } from '../../onions/onionPath'; -import { toHex } from '../../utils/String'; +import { incrementBadPathCountOrDrop } from '../../onions/onionPath'; +import { ed25519Str, toHex } from '../../utils/String'; import { Snode } from '../../../data/data'; import { callUtilsWorker } from '../../../webworker/workers/browser/util_worker_interface'; diff --git a/ts/session/apis/snode_api/retrieveRequest.ts b/ts/session/apis/snode_api/retrieveRequest.ts index e5ed42639..5dfcaf007 100644 --- a/ts/session/apis/snode_api/retrieveRequest.ts +++ b/ts/session/apis/snode_api/retrieveRequest.ts @@ -124,7 +124,7 @@ async function retrieveNextMessages( ); // let exceptions bubble up // no retry for this one as this a call we do every few seconds while polling for messages - const timeOutMs = 4 * 1000; + const timeOutMs = 10 * 1000; // yes this is a long timeout for just messages, but 4s timeout way to often... const timeoutPromise = async () => sleepFor(timeOutMs); const fetchPromise = async () => doSnodeBatchRequest(retrieveRequestsParams, targetNode, timeOutMs, associatedWith); diff --git a/ts/session/apis/snode_api/snodePool.ts b/ts/session/apis/snode_api/snodePool.ts index f13a27e21..20a029d2e 100644 --- a/ts/session/apis/snode_api/snodePool.ts +++ b/ts/session/apis/snode_api/snodePool.ts @@ -3,12 +3,12 @@ import pRetry from 'p-retry'; import { Data, Snode } from '../../../data/data'; -import { ed25519Str } from '../../onions/onionPath'; import { OnionPaths } from '../../onions'; import { Onions, SnodePool } from '.'; import { SeedNodeAPI } from '../seed_node_api'; import { requestSnodesForPubkeyFromNetwork } from './getSwarmFor'; import { ServiceNodesList } from './getServiceNodesList'; +import { ed25519Str } from '../../utils/String'; /** * If we get less than this snode in a swarm, we fetch new snodes for this pubkey @@ -204,6 +204,18 @@ export async function TEST_fetchFromSeedWithRetriesAndWriteToDb() { } } +async function clearOutAllSnodesNotInPool(snodePool: Array) { + if (snodePool.length <= 10) { + return; + } + const edKeysOfSnodePool = snodePool.map(m => m.pubkey_ed25519); + + await Data.clearOutAllSnodesNotInPool(edKeysOfSnodePool); + + // just remove all the cached entries, we will refetch them as needed from the DB + swarmCache.clear(); +} + /** * This function retries a few times to get a consensus between 3 snodes of at least 24 snodes in the snode pool. * @@ -230,6 +242,7 @@ async function tryToGetConsensusWithSnodesWithRetries() { ); randomSnodePool = commonNodes; await Data.updateSnodePoolOnDb(JSON.stringify(randomSnodePool)); + await clearOutAllSnodesNotInPool(randomSnodePool); OnionPaths.resetPathFailureCount(); Onions.resetSnodeFailureCount(); diff --git a/ts/session/apis/snode_api/swarmPolling.ts b/ts/session/apis/snode_api/swarmPolling.ts index 9b8f0ee7b..cedebc937 100644 --- a/ts/session/apis/snode_api/swarmPolling.ts +++ b/ts/session/apis/snode_api/swarmPolling.ts @@ -22,12 +22,12 @@ import { import { DURATION, SWARM_POLLING_TIMEOUT } from '../../constants'; import { getConversationController } from '../../conversations'; import { IncomingMessage } from '../../messages/incoming/IncomingMessage'; -import { ed25519Str } from '../../onions/onionPath'; import { StringUtils, UserUtils } from '../../utils'; import { LibSessionUtil } from '../../utils/libsession/libsession_utils'; import { SnodeNamespace, SnodeNamespaces } from './namespaces'; import { SnodeAPIRetrieve } from './retrieveRequest'; import { RetrieveMessageItem, RetrieveMessagesResultsBatched } from './types'; +import { ed25519Str } from '../../utils/String'; export function extractWebSocketContent( message: string, diff --git a/ts/session/onions/onionPath.ts b/ts/session/onions/onionPath.ts index d05229976..84b9d14a1 100644 --- a/ts/session/onions/onionPath.ts +++ b/ts/session/onions/onionPath.ts @@ -14,6 +14,7 @@ import { updateOnionPaths } from '../../state/ducks/onion'; import { ERROR_CODE_NO_CONNECT } from '../apis/snode_api/SNodeAPI'; import { OnionPaths } from '.'; import { APPLICATION_JSON } from '../../types/MIME'; +import { ed25519Str } from '../utils/String'; const desiredGuardCount = 3; const minimumGuardCount = 2; @@ -63,8 +64,6 @@ const pathFailureThreshold = 3; // some naming issue here it seems) export let guardNodes: Array = []; -export const ed25519Str = (ed25519Key: string) => `(...${ed25519Key.substr(58)})`; - export async function buildNewOnionPathsOneAtATime() { // this function may be called concurrently make sure we only have one inflight return allowOnlyOneAtATime('buildNewOnionPaths', async () => { diff --git a/ts/session/sending/MessageSender.ts b/ts/session/sending/MessageSender.ts index e2dcdf4ab..b7cbac601 100644 --- a/ts/session/sending/MessageSender.ts +++ b/ts/session/sending/MessageSender.ts @@ -32,11 +32,10 @@ import { SharedConfigMessage } from '../messages/outgoing/controlMessage/SharedC import { UnsendMessage } from '../messages/outgoing/controlMessage/UnsendMessage'; import { ClosedGroupNewMessage } from '../messages/outgoing/controlMessage/group/ClosedGroupNewMessage'; import { OpenGroupVisibleMessage } from '../messages/outgoing/visibleMessage/OpenGroupVisibleMessage'; -import { ed25519Str } from '../onions/onionPath'; import { PubKey } from '../types'; import { RawMessage } from '../types/RawMessage'; import { UserUtils } from '../utils'; -import { fromUInt8ArrayToBase64 } from '../utils/String'; +import { ed25519Str, fromUInt8ArrayToBase64 } from '../utils/String'; import { EmptySwarmError } from '../utils/errors'; // ================ SNODE STORE ================ diff --git a/ts/session/utils/String.ts b/ts/session/utils/String.ts index b12f09971..68713e5e5 100644 --- a/ts/session/utils/String.ts +++ b/ts/session/utils/String.ts @@ -71,3 +71,5 @@ export const sanitizeSessionUsername = (inputName: string) => { return validChars; }; + +export const ed25519Str = (ed25519Key: string) => `(...${ed25519Key.substr(58)})`; diff --git a/ts/session/utils/calling/CallManager.ts b/ts/session/utils/calling/CallManager.ts index cc411f0aa..b58406297 100644 --- a/ts/session/utils/calling/CallManager.ts +++ b/ts/session/utils/calling/CallManager.ts @@ -18,7 +18,6 @@ import { import { openConversationWithMessages } from '../../../state/ducks/conversations'; import { getConversationController } from '../../conversations'; import { CallMessage } from '../../messages/outgoing/controlMessage/CallMessage'; -import { ed25519Str } from '../../onions/onionPath'; import { PubKey } from '../../types'; import { getMessageQueue } from '../..'; @@ -35,6 +34,7 @@ import { ReadyToDisappearMsgUpdate } from '../../disappearing_messages/types'; import { MessageSender } from '../../sending'; import { getIsRinging } from '../RingingManager'; import { getBlackSilenceMediaStream } from './Silence'; +import { ed25519Str } from '../String'; export type InputItem = { deviceId: string; label: string }; @@ -415,8 +415,10 @@ async function createOfferAndSendIt(recipient: string, msgIdentifier: string | n if (offer && offer.sdp) { const lines = offer.sdp.split(/\r?\n/); const lineWithFtmpIndex = lines.findIndex(f => f.startsWith('a=fmtp:111')); - const partBeforeComma = lines[lineWithFtmpIndex].split(';'); - lines[lineWithFtmpIndex] = `${partBeforeComma[0]};cbr=1`; + if (lineWithFtmpIndex > -1) { + const partBeforeComma = lines[lineWithFtmpIndex].split(';'); + lines[lineWithFtmpIndex] = `${partBeforeComma[0]};cbr=1`; + } let overridenSdps = lines.join('\n'); overridenSdps = overridenSdps.replace( // eslint-disable-next-line prefer-regex-literals