fix slow app while removing v1 convoss

app/sql_channel.js

@@ -28,6 +28,7 @@ function initialize() {
       }
 
       const result = await fn(...args);
+
       event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result);
     } catch (error) {
       const errorForDisplay = error && error.stack ? error.stack : error;

js/background.js

@@ -280,15 +280,6 @@
     );
     window.log.info('Cleanup: complete');
 
-    const observer = new PerformanceObserver(list => {
-      console.warn('Long Task detected! 🚩️');
-      const entries = list.getEntries();
-      console.warn(entries);
-      // debugger;
-    });
-
-    observer.observe({ entryTypes: ['longtask'] });
-
     window.log.info('listening for registration events');
     Whisper.events.on('registration_done', async () => {
       window.log.info('handling registration event');

ts/components/session/ActionsPanel.tsx

@@ -14,6 +14,7 @@ import {
   getItemById,
   hasSyncedInitialConfigurationItem,
   lastAvatarUploadTimestamp,
+  removeConversation,
 } from '../../data/data';
 import { OnionPaths } from '../../session/onions';
 import { getMessageQueue } from '../../session/sending';
@@ -43,6 +44,8 @@ import { IMAGE_JPEG } from '../../types/MIME';
 import { FSv2 } from '../../fileserver';
 import { debounce } from 'lodash';
 import { DURATION } from '../../session/constants';
+import { actions as conversationActions } from '../../state/ducks/conversations';
+
 // tslint:disable-next-line: no-import-side-effect no-submodule-imports
 
 export enum SectionType {
@@ -160,14 +163,20 @@ const triggerSyncIfNeeded = async () => {
 
 const removeAllV1OpenGroups = async () => {
   const allV1Convos = (await getAllOpenGroupV1Conversations()).models || [];
-  // do this sequentially to avoid hurting the db
+  // do not remove messages of opengroupv1 for now. We have to find a way of doing it without making the whole app extremely slow
   // tslint:disable-next-line: prefer-for-of
   for (let index = 0; index < allV1Convos.length; index++) {
     const v1Convo = allV1Convos[index];
     try {
-      window.log.info('removing v1Convo: ', v1Convo.id);
-      // calling this here rather than in a migration because it takes care of removing the attachments, the messages, the profile images, etc.
-      await ConversationController.getInstance().deleteContact(v1Convo.id);
+      await removeConversation(v1Convo.id);
+      window.log.info(`deleting v1convo : ${v1Convo.id}`);
+      ConversationController.getInstance().unsafeDelete(v1Convo);
+      if (window.inboxStore) {
+        window.inboxStore?.dispatch(conversationActions.conversationRemoved(v1Convo.id));
+        window.inboxStore?.dispatch(
+          conversationActions.conversationChanged(v1Convo.id, v1Convo.getProps())
+        );
+      }
     } catch (e) {
       window.log.warn(`failed to delete opengroupv1 ${v1Convo.id}`, e);
     }

ts/data/data.ts

@@ -596,7 +596,6 @@ export async function getAllOpenGroupV1Conversations(): Promise<ConversationColl
 }
 
 export async function getPubkeysInPublicConversation(id: string): Promise<Array<string>> {
-  window?.log?.info(`getPubkeysInPublicConversation in '${id}'`);
   return channels.getPubkeysInPublicConversation(id);
 }
 
@@ -800,7 +799,7 @@ export async function removeAllMessagesInConversation(conversationId: string): P
     //   time so we don't use too much memory.
     // eslint-disable-next-line no-await-in-loop
     messages = await getMessagesByConversation(conversationId, {
-      limit: 25,
+      limit: 100,
     });
 
     if (!messages.length) {
@@ -816,7 +815,6 @@ export async function removeAllMessagesInConversation(conversationId: string): P
 
     // eslint-disable-next-line no-await-in-loop
     await channels.removeMessage(ids);
-    await sleepFor(10);
   } while (messages.length > 0);
 }
 

ts/interactions/message.ts

@@ -109,7 +109,6 @@ export async function removeSenderFromModerator(sender: string, convoId: string)
     const pubKeyToRemove = PubKey.cast(sender);
     const convo = ConversationController.getInstance().getOrThrow(convoId);
 
-    // FIXME audric removeModerator not working serverside
     const roomInfo = convo.toOpenGroupV2();
     const res = await ApiV2.removeModerator(pubKeyToRemove, roomInfo);
     if (!res) {

ts/opengroup/opengroupV2/ApiAuth.ts

@@ -35,20 +35,25 @@ async function claimAuthToken(
   return authToken;
 }
 
-async function oneAtATimeGetAuth({
-  serverUrl,
-  roomId,
-  roomDetails,
-}: OpenGroupRequestCommonType & { roomDetails: OpenGroupV2Room }) {
+async function oneAtATimeGetAuth({ serverUrl, roomId }: OpenGroupRequestCommonType) {
   return allowOnlyOneAtATime(`getAuthToken${serverUrl}:${roomId}`, async () => {
     try {
+      // first try to fetch from db a saved token.
+      const roomDetails = await getV2OpenGroupRoomByRoomId({ serverUrl, roomId });
+      if (!roomDetails) {
+        window?.log?.warn('getAuthToken Room does not exist.');
+        return null;
+      }
+
+      if (roomDetails?.token) {
+        return roomDetails.token;
+      }
+
       window?.log?.info(
         `Triggering getAuthToken with serverUrl:'${serverUrl}'; roomId: '${roomId}'`
       );
       const token = await requestNewAuthToken({ serverUrl, roomId });
-      if (roomId === 'lokinet') {
-        debugger;
-      }
+
       if (!token) {
         window?.log?.warn('invalid new auth token', token);
         return;
@@ -56,24 +61,18 @@ async function oneAtATimeGetAuth({
 
       window?.log?.info(`Got AuthToken for serverUrl:'${serverUrl}'; roomId: '${roomId}'`);
       const claimedToken = await claimAuthToken(token, serverUrl, roomId);
+
       if (!claimedToken) {
         window?.log?.warn('Failed to claim token', claimedToken);
       } else {
         window?.log?.info(`Claimed AuthToken for serverUrl:'${serverUrl}'; roomId: '${roomId}'`);
       }
-      console.error('Saving token to claimed token for ', roomDetails.roomId);
       // still save it to the db. just to mark it as to be refreshed later
-      if (roomId === 'lokinet') {
-        debugger;
-      }
       roomDetails.token = claimedToken || '';
-      if (roomId === 'lokinet') {
-        debugger;
-      }
-
       await saveV2OpenGroupRoom(roomDetails);
 
       window?.log?.info(`AuthToken saved to DB for serverUrl:'${serverUrl}'; roomId: '${roomId}'`);
+
       return claimedToken;
     } catch (e) {
       window?.log?.error('Failed to getAuthToken', e);
@@ -86,43 +85,7 @@ export async function getAuthToken({
   serverUrl,
   roomId,
 }: OpenGroupRequestCommonType): Promise<string | null> {
-  // first try to fetch from db a saved token.
-  const roomDetails = await getV2OpenGroupRoomByRoomId({ serverUrl, roomId });
-  if (!roomDetails) {
-    window?.log?.warn('getAuthToken Room does not exist.');
-    return null;
-  }
-  if (roomDetails?.token) {
-    console.error('Already having a saved token ', roomDetails.roomId);
-
-    return roomDetails.token;
-  }
-
-  const claimedToken = await oneAtATimeGetAuth({ roomDetails, roomId, serverUrl });
-
-  // fetch the data from the db again, which should have been written in the saveV2OpenGroupRoom() call above
-  const refreshedRoomDetails = await getV2OpenGroupRoomByRoomId({
-    serverUrl,
-    roomId,
-  });
-  if (!refreshedRoomDetails) {
-    window?.log?.warn('getAuthToken Room does not exist.');
-    return null;
-  }
-  // if the claimedToken got overriden, save it again
-  if (!refreshedRoomDetails?.token && claimedToken) {
-    refreshedRoomDetails.token = claimedToken;
-    console.error('claimed auth token for overriden. Forcing writing it', roomDetails.roomId);
-
-    await saveV2OpenGroupRoom(refreshedRoomDetails);
-  }
-
-  if (refreshedRoomDetails?.token) {
-    console.error('Returning freshclaimed token for ', roomDetails.roomId);
-
-    return refreshedRoomDetails?.token;
-  }
-  return null;
+  return oneAtATimeGetAuth({ roomId, serverUrl });
 }
 
 export const deleteAuthToken = async ({

ts/opengroup/opengroupV2/ApiUtil.ts

@@ -86,8 +86,8 @@ export const parseMessages = async (
     window?.log?.info('Handling rawMessage chunk of size', currentChunk.length);
     const messagesHandled = await handleChunk(currentChunk);
     allHandledMessages.push(...messagesHandled);
-
-    await sleepFor(10);
+    // as we are not running in a worker, just give some time for UI events
+    await sleepFor(2);
   }
 
   return _.compact(allHandledMessages).sort((a, b) => (a.serverId || 0) - (b.serverId || 0));

ts/opengroup/opengroupV2/OpenGroupAPIV2.ts

@@ -145,7 +145,6 @@ export async function sendApiV2Request(
         window?.log?.warn('Got 401, but this room does not exist');
         return null;
       }
-      console.error('Overriding token to undefined for ', roomDetails.roomId)
       roomDetails.token = undefined;
       // we might need to retry doing the request here, but how to make sure we don't retry indefinetely?
       await saveV2OpenGroupRoom(roomDetails);

ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts

@@ -271,8 +271,8 @@ async function sendOpenGroupV2RequestCompactPoll(
   // this holds only the poll results which are valid
   const roomPollValidResults = results.filter(ret => ret.statusCode === 200);
 
-  if (roomWithTokensToRefresh) {
-    window.log.info('We got those rooms to refresh the token ', roomWithTokensToRefresh);
+  if (roomWithTokensToRefresh?.length) {
+    window.log.info('We got those rooms to refresh the token with:', roomWithTokensToRefresh);
     await Promise.all(
       roomWithTokensToRefresh.map(async roomId => {
         const roomDetails = await getV2OpenGroupRoomByRoomId({
@@ -282,7 +282,6 @@ async function sendOpenGroupV2RequestCompactPoll(
         if (!roomDetails) {
           return;
         }
-        console.error('Overriding token to undefined for ', roomDetails.roomId);
         roomDetails.token = undefined;
         // we might need to retry doing the request here, but how to make sure we don't retry indefinetely?
         await saveV2OpenGroupRoom(roomDetails);

ts/opengroup/opengroupV2/OpenGroupServerPoller.ts

@@ -14,18 +14,17 @@ import {
 import _ from 'lodash';
 import { ConversationModel } from '../../models/conversation';
 import { getMessageIdsFromServerIds, removeMessage } from '../../data/data';
-import { getV2OpenGroupRoom, OpenGroupV2Room, saveV2OpenGroupRoom } from '../../data/opengroups';
+import { getV2OpenGroupRoom, saveV2OpenGroupRoom } from '../../data/opengroups';
 import { OpenGroupMessageV2 } from './OpenGroupMessageV2';
 import { handleOpenGroupV2Message } from '../../receiver/receiver';
 import autoBind from 'auto-bind';
 import { sha256 } from '../../session/crypto';
 import { fromBase64ToArrayBuffer } from '../../session/utils/String';
-import { getAuthToken } from './ApiAuth';
 import { DURATION } from '../../session/constants';
 
 const pollForEverythingInterval = DURATION.SECONDS * 10;
 const pollForRoomAvatarInterval = DURATION.DAYS * 1;
-const pollForMemberCountInterval = DURATION.MINUTES * 30;
+const pollForMemberCountInterval = DURATION.MINUTES * 10;
 
 /**
  * An OpenGroupServerPollerV2 polls for everything for a particular server. We should
@@ -173,20 +172,6 @@ export class OpenGroupServerPoller {
   }
 
   private async triggerPollAfterAdd(room?: OpenGroupRequestCommonType) {
-    if (room) {
-      // this call either get the token from db, or fetch a new one
-      console.warn('getAuthToken with triggerPollAfterAdd with room', room.roomId);
-      await getAuthToken({ roomId: room.roomId, serverUrl: this.serverUrl });
-    } else if (this.roomIdsToPoll.size) {
-      console.warn('getAuthToken with triggerPollAfterAdd with all rooms');
-      await Promise.all(
-        [...this.roomIdsToPoll].map(async r => {
-          // this call either get the token from db, or fetch a new one
-          await getAuthToken({ roomId: r, serverUrl: this.serverUrl });
-        })
-      );
-    }
-
     await this.compactPoll();
     await this.previewPerRoomPoll();
     await this.pollForAllMemberCount();

ts/receiver/closedGroups.ts

@@ -438,7 +438,7 @@ async function performIfValid(
   const convo = ConversationController.getInstance().get(groupPublicKey);
   if (!convo) {
     window?.log?.warn('dropping message for nonexistent group');
-    return;
+    return removeFromCache(envelope);
   }
 
   if (!convo) {

ts/session/conversations/ConversationController.ts

@@ -251,6 +251,10 @@ export class ConversationController {
     return Array.from(this.conversations.models);
   }
 
+  public unsafeDelete(convo: ConversationModel) {
+    this.conversations.remove(convo);
+  }
+
   public async load() {
     window?.log?.info('ConversationController: starting initial fetch');
 

ts/session/onions/onionPath.ts

@@ -60,9 +60,12 @@ let guardNodes: Array<SnodePool.Snode> = [];
 
 export const ed25519Str = (ed25519Key: string) => `(...${ed25519Key.substr(58)})`;
 
+let buildNewOnionPathsWorkerRetry = 0;
+
 export async function buildNewOnionPathsOneAtATime() {
   // this function may be called concurrently make sure we only have one inflight
   return allowOnlyOneAtATime('buildNewOnionPaths', async () => {
+    buildNewOnionPathsWorkerRetry = 0;
     await buildNewOnionPathsWorker();
   });
 }
@@ -363,9 +366,21 @@ async function buildNewOnionPathsWorker() {
     // this is a recursive call limited to only one call at a time. we use the timeout
     // here to make sure we retry this call if we cannot get enough otherNodes
 
-    setTimeout(async () => {
-      await buildNewOnionPathsOneAtATime();
-    }, 100);
+    // how to handle failing to rety
+    buildNewOnionPathsWorkerRetry = buildNewOnionPathsWorkerRetry + 1;
+    window.log.warn(
+      'buildNewOnionPathsWorker failed to get otherNodes. Current retry:',
+      buildNewOnionPathsWorkerRetry
+    );
+    if (buildNewOnionPathsWorkerRetry >= 3) {
+      // we failed enough. Something is wrong. Lets get out of that function and get a new fresh call.
+      window.log.warn(
+        `buildNewOnionPathsWorker failed to get otherNodes even after retries... Exiting after ${buildNewOnionPathsWorkerRetry} retries`
+      );
+
+      return;
+    }
+    await buildNewOnionPathsWorker();
     return;
   }
 

ts/session/utils/Promise.ts

@@ -25,7 +25,6 @@ export async function allowOnlyOneAtATime(
 ) {
   // if currently not in progress
   if (snodeGlobalLocks[name] === undefined) {
-    console.warn(`${name} not already running, creating it`);
     // set lock
     snodeGlobalLocks[name] = new Promise(async (resolve, reject) => {
       // set up timeout feature
@@ -72,8 +71,6 @@ export async function allowOnlyOneAtATime(
       // release the kraken
       resolve(innerRetVal);
     });
-  } else {
-    console.warn(`${name} already running, returning it`);
   }
   return snodeGlobalLocks[name];
 }