fix: clear swarms from snodes not in pool on full fetch

.gitignore

@@ -53,3 +53,5 @@ stylesheets/dist/
 *.LICENSE.txt
 ts/webworker/workers/node/**/*.node
 
+.yarn/**/*.mjs
+.yarn/**/*.cjs

package.json

@@ -216,11 +216,13 @@
     "afterSign": "build/notarize.js",
     "afterPack": "build/afterPackHook.js",
     "artifactName": "${name}-${os}-${arch}-${version}.${ext}",
-    "extraResources": [{
+    "extraResources": [
+      {
         "from": "./build/launcher-script.sh",
         "to": "./launcher-script.sh"
       },
-    "mmdb/GeoLite2-Country.mmdb"],
+      "mmdb/GeoLite2-Country.mmdb"
+    ],
     "mac": {
       "category": "public.app-category.social-networking",
       "icon": "build/icon-mac.icns",

ts/components/calling/IncomingCallDialog.tsx

@@ -3,13 +3,13 @@ import { useSelector } from 'react-redux';
 
 import styled from 'styled-components';
 import { useConversationUsername } from '../../hooks/useParamSelector';
-import { ed25519Str } from '../../session/onions/onionPath';
 import { CallManager } from '../../session/utils';
 import { callTimeoutMs } from '../../session/utils/calling/CallManager';
 import { getHasIncomingCall, getHasIncomingCallFrom } from '../../state/selectors/call';
 import { Avatar, AvatarSize } from '../avatar/Avatar';
 import { SessionButton, SessionButtonColor, SessionButtonType } from '../basic/SessionButton';
 import { SessionWrapperModal } from '../SessionWrapperModal';
+import { ed25519Str } from '../../session/utils/String';
 
 export const CallWindow = styled.div`
   position: absolute;

ts/components/dialog/DeleteAccountModal.tsx

@@ -1,7 +1,7 @@
 import React, { useCallback, useState } from 'react';
 import { useDispatch } from 'react-redux';
 import { SnodeAPI } from '../../session/apis/snode_api/SNodeAPI';
-import { ed25519Str } from '../../session/onions/onionPath';
+
 import { forceSyncConfigurationNowIfNeeded } from '../../session/utils/sync/syncUtils';
 import { updateConfirmModal, updateDeleteAccountModal } from '../../state/ducks/modalDialog';
 import { SessionWrapperModal } from '../SessionWrapperModal';
@@ -14,6 +14,7 @@ import { deleteAllLogs } from '../../node/logs';
 import { clearInbox } from '../../session/apis/open_group_api/sogsv3/sogsV3ClearInbox';
 import { getAllValidOpenGroupV2ConversationRoomInfos } from '../../session/apis/open_group_api/utils/OpenGroupUtils';
 import { SessionRadioGroup } from '../basic/SessionRadioGroup';
+import { ed25519Str } from '../../session/utils/String';
 
 const deleteDbLocally = async () => {
   window?.log?.info('last message sent successfully. Deleting everything');

ts/data/data.ts

@@ -109,6 +109,10 @@ async function updateSwarmNodesForPubkey(
   await channels.updateSwarmNodesForPubkey(pubkey, snodeEdKeys);
 }
 
+async function clearOutAllSnodesNotInPool(edKeysOfSnodePool: Array<string>): Promise<void> {
+  await channels.clearOutAllSnodesNotInPool(edKeysOfSnodePool);
+}
+
 // Closed group
 
 /**
@@ -802,6 +806,7 @@ export const Data = {
   generateAttachmentKeyIfEmpty,
   getSwarmNodesForPubkey,
   updateSwarmNodesForPubkey,
+  clearOutAllSnodesNotInPool,
   getAllEncryptionKeyPairsForGroup,
   getLatestClosedGroupEncryptionKeyPair,
   addClosedGroupEncryptionKeyPair,

ts/data/dataInit.ts

@@ -24,6 +24,7 @@ const channelsToMake = new Set([
   'removeItemById',
   'getSwarmNodesForPubkey',
   'updateSwarmNodesForPubkey',
+  'clearOutAllSnodesNotInPool',
   'saveConversation',
   'fetchConvoMemoryDetails',
   'getConversationById',

ts/interactions/conversations/unsendingInteractions.ts

@@ -9,12 +9,12 @@ import { SnodeAPI } from '../../session/apis/snode_api/SNodeAPI';
 import { SnodeNamespaces } from '../../session/apis/snode_api/namespaces';
 import { getConversationController } from '../../session/conversations';
 import { UnsendMessage } from '../../session/messages/outgoing/controlMessage/UnsendMessage';
-import { ed25519Str } from '../../session/onions/onionPath';
 import { PubKey } from '../../session/types';
 import { ToastUtils, UserUtils } from '../../session/utils';
 import { closeRightPanel, resetSelectedMessageIds } from '../../state/ducks/conversations';
 import { updateConfirmModal } from '../../state/ducks/modalDialog';
 import { resetRightOverlayMode } from '../../state/ducks/section';
+import { ed25519Str } from '../../session/utils/String';
 
 /**
  * Deletes messages for everyone in a 1-1 or everyone in a closed group conversation.

ts/models/conversation.ts

@@ -42,7 +42,7 @@ import {
   VisibleMessageParams,
 } from '../session/messages/outgoing/visibleMessage/VisibleMessage';
 import { perfEnd, perfStart } from '../session/utils/Performance';
-import { toHex } from '../session/utils/String';
+import { ed25519Str, toHex } from '../session/utils/String';
 import { createTaskWithTimeout } from '../session/utils/TaskWithTimeout';
 import {
   actions as conversationActions,
@@ -74,7 +74,6 @@ import {
   MessageRequestResponse,
   MessageRequestResponseParams,
 } from '../session/messages/outgoing/controlMessage/MessageRequestResponse';
-import { ed25519Str } from '../session/onions/onionPath';
 import { ConfigurationSync } from '../session/utils/job_runners/jobs/ConfigurationSyncJob';
 import { SessionUtilContact } from '../session/utils/libsession/libsession_utils_contacts';
 import { SessionUtilConvoInfoVolatile } from '../session/utils/libsession/libsession_utils_convo_info_volatile';

ts/node/sql.ts

@@ -12,6 +12,7 @@ import {
   differenceBy,
   forEach,
   fromPairs,
+  intersection,
   isArray,
   isEmpty,
   isNumber,
@@ -78,6 +79,7 @@ import {
   initDbInstanceWith,
   isInstanceInitialized,
 } from './sqlInstance';
+import { ed25519Str } from '../session/utils/String';
 
 // eslint:disable: function-name non-literal-fs-path
 
@@ -398,6 +400,32 @@ function updateSwarmNodesForPubkey(pubkey: string, snodeEdKeys: Array<string>) {
     });
 }
 
+function clearOutAllSnodesNotInPool(edKeysOfSnodePool: Array<string>) {
+  const allSwarms = assertGlobalInstance()
+    .prepare(`SELECT * FROM ${NODES_FOR_PUBKEY_TABLE};`)
+    .all();
+
+  allSwarms.forEach(swarm => {
+    try {
+      const json = JSON.parse(swarm.json);
+      if (isArray(json)) {
+        const intersect = intersection(json, edKeysOfSnodePool);
+        if (intersect.length !== json.length) {
+          updateSwarmNodesForPubkey(swarm.pubkey, intersect);
+          console.info(
+            `clearOutAllSnodesNotInPool: updating swarm of ${ed25519Str(swarm.pubkey)} to `,
+            intersect
+          );
+        }
+      }
+    } catch (e) {
+      console.warn(
+        `Failed to parse swarm while iterating in clearOutAllSnodesNotInPool for pk: ${ed25519Str(swarm?.pubkey)}`
+      );
+    }
+  });
+}
+
 function getConversationCount() {
   const row = assertGlobalInstance().prepare(`SELECT count(*) from ${CONVERSATIONS_TABLE};`).get();
   if (!row) {
@@ -2449,6 +2477,7 @@ export const sqlNode = {
 
   getSwarmNodesForPubkey,
   updateSwarmNodesForPubkey,
+  clearOutAllSnodesNotInPool,
   getGuardNodes,
   updateGuardNodes,
 

ts/session/apis/snode_api/SNodeAPI.ts

@@ -4,9 +4,8 @@ import { compact, sample } from 'lodash';
 import pRetry from 'p-retry';
 import { Snode } from '../../../data/data';
 import { getSodiumRenderer } from '../../crypto';
-import { ed25519Str } from '../../onions/onionPath';
 import { StringUtils, UserUtils } from '../../utils';
-import { fromBase64ToArray, fromHexToArray } from '../../utils/String';
+import { ed25519Str, fromBase64ToArray, fromHexToArray } from '../../utils/String';
 import { doSnodeBatchRequest } from './batchRequest';
 import { getSwarmFor } from './snodePool';
 import { SnodeSignature } from './snodeSignatures';

ts/session/apis/snode_api/onions.ts

@@ -11,8 +11,8 @@ import { AbortSignal as AbortSignalNode } from 'node-fetch/externals';
 import { dropSnodeFromSnodePool, dropSnodeFromSwarmIfNeeded, updateSwarmFor } from './snodePool';
 
 import { OnionPaths } from '../../onions';
-import { ed25519Str, incrementBadPathCountOrDrop } from '../../onions/onionPath';
-import { toHex } from '../../utils/String';
+import {  incrementBadPathCountOrDrop } from '../../onions/onionPath';
+import { ed25519Str, toHex } from '../../utils/String';
 
 import { Snode } from '../../../data/data';
 import { callUtilsWorker } from '../../../webworker/workers/browser/util_worker_interface';

ts/session/apis/snode_api/retrieveRequest.ts

@@ -124,7 +124,7 @@ async function retrieveNextMessages(
   );
   // let exceptions bubble up
   // no retry for this one as this a call we do every few seconds while polling for messages
-  const timeOutMs = 4 * 1000;
+  const timeOutMs = 10 * 1000; // yes this is a long timeout for just messages, but 4s timeout way to often...
   const timeoutPromise = async () => sleepFor(timeOutMs);
   const fetchPromise = async () =>
     doSnodeBatchRequest(retrieveRequestsParams, targetNode, timeOutMs, associatedWith);

ts/session/apis/snode_api/snodePool.ts

@@ -3,12 +3,12 @@ import pRetry from 'p-retry';
 
 import { Data, Snode } from '../../../data/data';
 
-import { ed25519Str } from '../../onions/onionPath';
 import { OnionPaths } from '../../onions';
 import { Onions, SnodePool } from '.';
 import { SeedNodeAPI } from '../seed_node_api';
 import { requestSnodesForPubkeyFromNetwork } from './getSwarmFor';
 import { ServiceNodesList } from './getServiceNodesList';
+import { ed25519Str } from '../../utils/String';
 
 /**
  * If we get less than this snode in a swarm, we fetch new snodes for this pubkey
@@ -204,6 +204,18 @@ export async function TEST_fetchFromSeedWithRetriesAndWriteToDb() {
   }
 }
 
+async function clearOutAllSnodesNotInPool(snodePool: Array<Snode>) {
+  if (snodePool.length <= 10) {
+    return;
+  }
+  const edKeysOfSnodePool = snodePool.map(m => m.pubkey_ed25519);
+
+  await Data.clearOutAllSnodesNotInPool(edKeysOfSnodePool);
+
+  // just remove all the cached entries, we will refetch them as needed from the DB
+  swarmCache.clear();
+}
+
 /**
  * This function retries a few times to get a consensus between 3 snodes of at least 24 snodes in the snode pool.
  *
@@ -230,6 +242,7 @@ async function tryToGetConsensusWithSnodesWithRetries() {
       );
       randomSnodePool = commonNodes;
       await Data.updateSnodePoolOnDb(JSON.stringify(randomSnodePool));
+      await clearOutAllSnodesNotInPool(randomSnodePool);
 
       OnionPaths.resetPathFailureCount();
       Onions.resetSnodeFailureCount();

ts/session/apis/snode_api/swarmPolling.ts

@@ -22,12 +22,12 @@ import {
 import { DURATION, SWARM_POLLING_TIMEOUT } from '../../constants';
 import { getConversationController } from '../../conversations';
 import { IncomingMessage } from '../../messages/incoming/IncomingMessage';
-import { ed25519Str } from '../../onions/onionPath';
 import { StringUtils, UserUtils } from '../../utils';
 import { LibSessionUtil } from '../../utils/libsession/libsession_utils';
 import { SnodeNamespace, SnodeNamespaces } from './namespaces';
 import { SnodeAPIRetrieve } from './retrieveRequest';
 import { RetrieveMessageItem, RetrieveMessagesResultsBatched } from './types';
+import { ed25519Str } from '../../utils/String';
 
 export function extractWebSocketContent(
   message: string,

ts/session/onions/onionPath.ts

@@ -14,6 +14,7 @@ import { updateOnionPaths } from '../../state/ducks/onion';
 import { ERROR_CODE_NO_CONNECT } from '../apis/snode_api/SNodeAPI';
 import { OnionPaths } from '.';
 import { APPLICATION_JSON } from '../../types/MIME';
+import { ed25519Str } from '../utils/String';
 
 const desiredGuardCount = 3;
 const minimumGuardCount = 2;
@@ -63,8 +64,6 @@ const pathFailureThreshold = 3;
 // some naming issue here it seems)
 export let guardNodes: Array<Snode> = [];
 
-export const ed25519Str = (ed25519Key: string) => `(...${ed25519Key.substr(58)})`;
-
 export async function buildNewOnionPathsOneAtATime() {
   // this function may be called concurrently make sure we only have one inflight
   return allowOnlyOneAtATime('buildNewOnionPaths', async () => {

ts/session/sending/MessageSender.ts

@@ -32,11 +32,10 @@ import { SharedConfigMessage } from '../messages/outgoing/controlMessage/SharedC
 import { UnsendMessage } from '../messages/outgoing/controlMessage/UnsendMessage';
 import { ClosedGroupNewMessage } from '../messages/outgoing/controlMessage/group/ClosedGroupNewMessage';
 import { OpenGroupVisibleMessage } from '../messages/outgoing/visibleMessage/OpenGroupVisibleMessage';
-import { ed25519Str } from '../onions/onionPath';
 import { PubKey } from '../types';
 import { RawMessage } from '../types/RawMessage';
 import { UserUtils } from '../utils';
-import { fromUInt8ArrayToBase64 } from '../utils/String';
+import { ed25519Str, fromUInt8ArrayToBase64 } from '../utils/String';
 import { EmptySwarmError } from '../utils/errors';
 
 // ================ SNODE STORE ================

ts/session/utils/String.ts

@@ -71,3 +71,5 @@ export const sanitizeSessionUsername = (inputName: string) => {
 
   return validChars;
 };
+
+export const ed25519Str = (ed25519Key: string) => `(...${ed25519Key.substr(58)})`;

ts/session/utils/calling/CallManager.ts

@@ -18,7 +18,6 @@ import {
 import { openConversationWithMessages } from '../../../state/ducks/conversations';
 import { getConversationController } from '../../conversations';
 import { CallMessage } from '../../messages/outgoing/controlMessage/CallMessage';
-import { ed25519Str } from '../../onions/onionPath';
 import { PubKey } from '../../types';
 
 import { getMessageQueue } from '../..';
@@ -35,6 +34,7 @@ import { ReadyToDisappearMsgUpdate } from '../../disappearing_messages/types';
 import { MessageSender } from '../../sending';
 import { getIsRinging } from '../RingingManager';
 import { getBlackSilenceMediaStream } from './Silence';
+import { ed25519Str } from '../String';
 
 export type InputItem = { deviceId: string; label: string };
 
@@ -415,8 +415,10 @@ async function createOfferAndSendIt(recipient: string, msgIdentifier: string | n
     if (offer && offer.sdp) {
       const lines = offer.sdp.split(/\r?\n/);
       const lineWithFtmpIndex = lines.findIndex(f => f.startsWith('a=fmtp:111'));
+      if (lineWithFtmpIndex > -1) {
         const partBeforeComma = lines[lineWithFtmpIndex].split(';');
         lines[lineWithFtmpIndex] = `${partBeforeComma[0]};cbr=1`;
+      }
       let overridenSdps = lines.join('\n');
       overridenSdps = overridenSdps.replace(
         // eslint-disable-next-line prefer-regex-literals