From ecf409bed57f457082d77c82a84d14130e23059c Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 4 Jun 2021 11:38:51 +1000 Subject: [PATCH] fix slow app while removing v1 convoss --- app/sql_channel.js | 1 + js/background.js | 9 --- ts/components/session/ActionsPanel.tsx | 17 +++-- ts/data/data.ts | 4 +- ts/interactions/message.ts | 1 - ts/opengroup/opengroupV2/ApiAuth.ts | 69 +++++-------------- ts/opengroup/opengroupV2/ApiUtil.ts | 4 +- ts/opengroup/opengroupV2/OpenGroupAPIV2.ts | 1 - .../opengroupV2/OpenGroupAPIV2CompactPoll.ts | 5 +- .../opengroupV2/OpenGroupServerPoller.ts | 19 +---- ts/receiver/closedGroups.ts | 2 +- .../conversations/ConversationController.ts | 4 ++ ts/session/onions/onionPath.ts | 21 +++++- ts/session/utils/Promise.ts | 3 - 14 files changed, 60 insertions(+), 100 deletions(-) diff --git a/app/sql_channel.js b/app/sql_channel.js index f7cb999dc..569e44a42 100644 --- a/app/sql_channel.js +++ b/app/sql_channel.js @@ -28,6 +28,7 @@ function initialize() { } const result = await fn(...args); + event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result); } catch (error) { const errorForDisplay = error && error.stack ? error.stack : error; diff --git a/js/background.js b/js/background.js index bfa337fc8..a35b84626 100644 --- a/js/background.js +++ b/js/background.js @@ -280,15 +280,6 @@ ); window.log.info('Cleanup: complete'); - const observer = new PerformanceObserver(list => { - console.warn('Long Task detected! 🚩️'); - const entries = list.getEntries(); - console.warn(entries); - // debugger; - }); - - observer.observe({ entryTypes: ['longtask'] }); - window.log.info('listening for registration events'); Whisper.events.on('registration_done', async () => { window.log.info('handling registration event'); diff --git a/ts/components/session/ActionsPanel.tsx b/ts/components/session/ActionsPanel.tsx index 1557a3b81..c09c58315 100644 --- a/ts/components/session/ActionsPanel.tsx +++ b/ts/components/session/ActionsPanel.tsx @@ -14,6 +14,7 @@ import { getItemById, hasSyncedInitialConfigurationItem, lastAvatarUploadTimestamp, + removeConversation, } from '../../data/data'; import { OnionPaths } from '../../session/onions'; import { getMessageQueue } from '../../session/sending'; @@ -43,6 +44,8 @@ import { IMAGE_JPEG } from '../../types/MIME'; import { FSv2 } from '../../fileserver'; import { debounce } from 'lodash'; import { DURATION } from '../../session/constants'; +import { actions as conversationActions } from '../../state/ducks/conversations'; + // tslint:disable-next-line: no-import-side-effect no-submodule-imports export enum SectionType { @@ -160,14 +163,20 @@ const triggerSyncIfNeeded = async () => { const removeAllV1OpenGroups = async () => { const allV1Convos = (await getAllOpenGroupV1Conversations()).models || []; - // do this sequentially to avoid hurting the db + // do not remove messages of opengroupv1 for now. We have to find a way of doing it without making the whole app extremely slow // tslint:disable-next-line: prefer-for-of for (let index = 0; index < allV1Convos.length; index++) { const v1Convo = allV1Convos[index]; try { - window.log.info('removing v1Convo: ', v1Convo.id); - // calling this here rather than in a migration because it takes care of removing the attachments, the messages, the profile images, etc. - await ConversationController.getInstance().deleteContact(v1Convo.id); + await removeConversation(v1Convo.id); + window.log.info(`deleting v1convo : ${v1Convo.id}`); + ConversationController.getInstance().unsafeDelete(v1Convo); + if (window.inboxStore) { + window.inboxStore?.dispatch(conversationActions.conversationRemoved(v1Convo.id)); + window.inboxStore?.dispatch( + conversationActions.conversationChanged(v1Convo.id, v1Convo.getProps()) + ); + } } catch (e) { window.log.warn(`failed to delete opengroupv1 ${v1Convo.id}`, e); } diff --git a/ts/data/data.ts b/ts/data/data.ts index 3d35f9201..7d6b39916 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -596,7 +596,6 @@ export async function getAllOpenGroupV1Conversations(): Promise> { - window?.log?.info(`getPubkeysInPublicConversation in '${id}'`); return channels.getPubkeysInPublicConversation(id); } @@ -800,7 +799,7 @@ export async function removeAllMessagesInConversation(conversationId: string): P // time so we don't use too much memory. // eslint-disable-next-line no-await-in-loop messages = await getMessagesByConversation(conversationId, { - limit: 25, + limit: 100, }); if (!messages.length) { @@ -816,7 +815,6 @@ export async function removeAllMessagesInConversation(conversationId: string): P // eslint-disable-next-line no-await-in-loop await channels.removeMessage(ids); - await sleepFor(10); } while (messages.length > 0); } diff --git a/ts/interactions/message.ts b/ts/interactions/message.ts index 4766a8300..7aa20b24b 100644 --- a/ts/interactions/message.ts +++ b/ts/interactions/message.ts @@ -109,7 +109,6 @@ export async function removeSenderFromModerator(sender: string, convoId: string) const pubKeyToRemove = PubKey.cast(sender); const convo = ConversationController.getInstance().getOrThrow(convoId); - // FIXME audric removeModerator not working serverside const roomInfo = convo.toOpenGroupV2(); const res = await ApiV2.removeModerator(pubKeyToRemove, roomInfo); if (!res) { diff --git a/ts/opengroup/opengroupV2/ApiAuth.ts b/ts/opengroup/opengroupV2/ApiAuth.ts index 10cd62ba2..cc3744a87 100644 --- a/ts/opengroup/opengroupV2/ApiAuth.ts +++ b/ts/opengroup/opengroupV2/ApiAuth.ts @@ -35,20 +35,25 @@ async function claimAuthToken( return authToken; } -async function oneAtATimeGetAuth({ - serverUrl, - roomId, - roomDetails, -}: OpenGroupRequestCommonType & { roomDetails: OpenGroupV2Room }) { +async function oneAtATimeGetAuth({ serverUrl, roomId }: OpenGroupRequestCommonType) { return allowOnlyOneAtATime(`getAuthToken${serverUrl}:${roomId}`, async () => { try { + // first try to fetch from db a saved token. + const roomDetails = await getV2OpenGroupRoomByRoomId({ serverUrl, roomId }); + if (!roomDetails) { + window?.log?.warn('getAuthToken Room does not exist.'); + return null; + } + + if (roomDetails?.token) { + return roomDetails.token; + } + window?.log?.info( `Triggering getAuthToken with serverUrl:'${serverUrl}'; roomId: '${roomId}'` ); const token = await requestNewAuthToken({ serverUrl, roomId }); - if (roomId === 'lokinet') { - debugger; - } + if (!token) { window?.log?.warn('invalid new auth token', token); return; @@ -56,24 +61,18 @@ async function oneAtATimeGetAuth({ window?.log?.info(`Got AuthToken for serverUrl:'${serverUrl}'; roomId: '${roomId}'`); const claimedToken = await claimAuthToken(token, serverUrl, roomId); + if (!claimedToken) { window?.log?.warn('Failed to claim token', claimedToken); } else { window?.log?.info(`Claimed AuthToken for serverUrl:'${serverUrl}'; roomId: '${roomId}'`); } - console.error('Saving token to claimed token for ', roomDetails.roomId); // still save it to the db. just to mark it as to be refreshed later - if (roomId === 'lokinet') { - debugger; - } roomDetails.token = claimedToken || ''; - if (roomId === 'lokinet') { - debugger; - } - await saveV2OpenGroupRoom(roomDetails); window?.log?.info(`AuthToken saved to DB for serverUrl:'${serverUrl}'; roomId: '${roomId}'`); + return claimedToken; } catch (e) { window?.log?.error('Failed to getAuthToken', e); @@ -86,43 +85,7 @@ export async function getAuthToken({ serverUrl, roomId, }: OpenGroupRequestCommonType): Promise { - // first try to fetch from db a saved token. - const roomDetails = await getV2OpenGroupRoomByRoomId({ serverUrl, roomId }); - if (!roomDetails) { - window?.log?.warn('getAuthToken Room does not exist.'); - return null; - } - if (roomDetails?.token) { - console.error('Already having a saved token ', roomDetails.roomId); - - return roomDetails.token; - } - - const claimedToken = await oneAtATimeGetAuth({ roomDetails, roomId, serverUrl }); - - // fetch the data from the db again, which should have been written in the saveV2OpenGroupRoom() call above - const refreshedRoomDetails = await getV2OpenGroupRoomByRoomId({ - serverUrl, - roomId, - }); - if (!refreshedRoomDetails) { - window?.log?.warn('getAuthToken Room does not exist.'); - return null; - } - // if the claimedToken got overriden, save it again - if (!refreshedRoomDetails?.token && claimedToken) { - refreshedRoomDetails.token = claimedToken; - console.error('claimed auth token for overriden. Forcing writing it', roomDetails.roomId); - - await saveV2OpenGroupRoom(refreshedRoomDetails); - } - - if (refreshedRoomDetails?.token) { - console.error('Returning freshclaimed token for ', roomDetails.roomId); - - return refreshedRoomDetails?.token; - } - return null; + return oneAtATimeGetAuth({ roomId, serverUrl }); } export const deleteAuthToken = async ({ diff --git a/ts/opengroup/opengroupV2/ApiUtil.ts b/ts/opengroup/opengroupV2/ApiUtil.ts index 598b02101..1f688fe0e 100644 --- a/ts/opengroup/opengroupV2/ApiUtil.ts +++ b/ts/opengroup/opengroupV2/ApiUtil.ts @@ -86,8 +86,8 @@ export const parseMessages = async ( window?.log?.info('Handling rawMessage chunk of size', currentChunk.length); const messagesHandled = await handleChunk(currentChunk); allHandledMessages.push(...messagesHandled); - - await sleepFor(10); + // as we are not running in a worker, just give some time for UI events + await sleepFor(2); } return _.compact(allHandledMessages).sort((a, b) => (a.serverId || 0) - (b.serverId || 0)); diff --git a/ts/opengroup/opengroupV2/OpenGroupAPIV2.ts b/ts/opengroup/opengroupV2/OpenGroupAPIV2.ts index 7928c790f..0f45ad126 100644 --- a/ts/opengroup/opengroupV2/OpenGroupAPIV2.ts +++ b/ts/opengroup/opengroupV2/OpenGroupAPIV2.ts @@ -145,7 +145,6 @@ export async function sendApiV2Request( window?.log?.warn('Got 401, but this room does not exist'); return null; } - console.error('Overriding token to undefined for ', roomDetails.roomId) roomDetails.token = undefined; // we might need to retry doing the request here, but how to make sure we don't retry indefinetely? await saveV2OpenGroupRoom(roomDetails); diff --git a/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts b/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts index 280941c79..d1a747eef 100644 --- a/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts +++ b/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts @@ -271,8 +271,8 @@ async function sendOpenGroupV2RequestCompactPoll( // this holds only the poll results which are valid const roomPollValidResults = results.filter(ret => ret.statusCode === 200); - if (roomWithTokensToRefresh) { - window.log.info('We got those rooms to refresh the token ', roomWithTokensToRefresh); + if (roomWithTokensToRefresh?.length) { + window.log.info('We got those rooms to refresh the token with:', roomWithTokensToRefresh); await Promise.all( roomWithTokensToRefresh.map(async roomId => { const roomDetails = await getV2OpenGroupRoomByRoomId({ @@ -282,7 +282,6 @@ async function sendOpenGroupV2RequestCompactPoll( if (!roomDetails) { return; } - console.error('Overriding token to undefined for ', roomDetails.roomId); roomDetails.token = undefined; // we might need to retry doing the request here, but how to make sure we don't retry indefinetely? await saveV2OpenGroupRoom(roomDetails); diff --git a/ts/opengroup/opengroupV2/OpenGroupServerPoller.ts b/ts/opengroup/opengroupV2/OpenGroupServerPoller.ts index 087814408..b999b43a6 100644 --- a/ts/opengroup/opengroupV2/OpenGroupServerPoller.ts +++ b/ts/opengroup/opengroupV2/OpenGroupServerPoller.ts @@ -14,18 +14,17 @@ import { import _ from 'lodash'; import { ConversationModel } from '../../models/conversation'; import { getMessageIdsFromServerIds, removeMessage } from '../../data/data'; -import { getV2OpenGroupRoom, OpenGroupV2Room, saveV2OpenGroupRoom } from '../../data/opengroups'; +import { getV2OpenGroupRoom, saveV2OpenGroupRoom } from '../../data/opengroups'; import { OpenGroupMessageV2 } from './OpenGroupMessageV2'; import { handleOpenGroupV2Message } from '../../receiver/receiver'; import autoBind from 'auto-bind'; import { sha256 } from '../../session/crypto'; import { fromBase64ToArrayBuffer } from '../../session/utils/String'; -import { getAuthToken } from './ApiAuth'; import { DURATION } from '../../session/constants'; const pollForEverythingInterval = DURATION.SECONDS * 10; const pollForRoomAvatarInterval = DURATION.DAYS * 1; -const pollForMemberCountInterval = DURATION.MINUTES * 30; +const pollForMemberCountInterval = DURATION.MINUTES * 10; /** * An OpenGroupServerPollerV2 polls for everything for a particular server. We should @@ -173,20 +172,6 @@ export class OpenGroupServerPoller { } private async triggerPollAfterAdd(room?: OpenGroupRequestCommonType) { - if (room) { - // this call either get the token from db, or fetch a new one - console.warn('getAuthToken with triggerPollAfterAdd with room', room.roomId); - await getAuthToken({ roomId: room.roomId, serverUrl: this.serverUrl }); - } else if (this.roomIdsToPoll.size) { - console.warn('getAuthToken with triggerPollAfterAdd with all rooms'); - await Promise.all( - [...this.roomIdsToPoll].map(async r => { - // this call either get the token from db, or fetch a new one - await getAuthToken({ roomId: r, serverUrl: this.serverUrl }); - }) - ); - } - await this.compactPoll(); await this.previewPerRoomPoll(); await this.pollForAllMemberCount(); diff --git a/ts/receiver/closedGroups.ts b/ts/receiver/closedGroups.ts index 9dab0f735..1e35a6e03 100644 --- a/ts/receiver/closedGroups.ts +++ b/ts/receiver/closedGroups.ts @@ -438,7 +438,7 @@ async function performIfValid( const convo = ConversationController.getInstance().get(groupPublicKey); if (!convo) { window?.log?.warn('dropping message for nonexistent group'); - return; + return removeFromCache(envelope); } if (!convo) { diff --git a/ts/session/conversations/ConversationController.ts b/ts/session/conversations/ConversationController.ts index b347327d7..c40b4ccaf 100644 --- a/ts/session/conversations/ConversationController.ts +++ b/ts/session/conversations/ConversationController.ts @@ -251,6 +251,10 @@ export class ConversationController { return Array.from(this.conversations.models); } + public unsafeDelete(convo: ConversationModel) { + this.conversations.remove(convo); + } + public async load() { window?.log?.info('ConversationController: starting initial fetch'); diff --git a/ts/session/onions/onionPath.ts b/ts/session/onions/onionPath.ts index 40670e24e..49cc8f89d 100644 --- a/ts/session/onions/onionPath.ts +++ b/ts/session/onions/onionPath.ts @@ -60,9 +60,12 @@ let guardNodes: Array = []; export const ed25519Str = (ed25519Key: string) => `(...${ed25519Key.substr(58)})`; +let buildNewOnionPathsWorkerRetry = 0; + export async function buildNewOnionPathsOneAtATime() { // this function may be called concurrently make sure we only have one inflight return allowOnlyOneAtATime('buildNewOnionPaths', async () => { + buildNewOnionPathsWorkerRetry = 0; await buildNewOnionPathsWorker(); }); } @@ -363,9 +366,21 @@ async function buildNewOnionPathsWorker() { // this is a recursive call limited to only one call at a time. we use the timeout // here to make sure we retry this call if we cannot get enough otherNodes - setTimeout(async () => { - await buildNewOnionPathsOneAtATime(); - }, 100); + // how to handle failing to rety + buildNewOnionPathsWorkerRetry = buildNewOnionPathsWorkerRetry + 1; + window.log.warn( + 'buildNewOnionPathsWorker failed to get otherNodes. Current retry:', + buildNewOnionPathsWorkerRetry + ); + if (buildNewOnionPathsWorkerRetry >= 3) { + // we failed enough. Something is wrong. Lets get out of that function and get a new fresh call. + window.log.warn( + `buildNewOnionPathsWorker failed to get otherNodes even after retries... Exiting after ${buildNewOnionPathsWorkerRetry} retries` + ); + + return; + } + await buildNewOnionPathsWorker(); return; } diff --git a/ts/session/utils/Promise.ts b/ts/session/utils/Promise.ts index 891dc2b96..1642ceda6 100644 --- a/ts/session/utils/Promise.ts +++ b/ts/session/utils/Promise.ts @@ -25,7 +25,6 @@ export async function allowOnlyOneAtATime( ) { // if currently not in progress if (snodeGlobalLocks[name] === undefined) { - console.warn(`${name} not already running, creating it`); // set lock snodeGlobalLocks[name] = new Promise(async (resolve, reject) => { // set up timeout feature @@ -72,8 +71,6 @@ export async function allowOnlyOneAtATime( // release the kraken resolve(innerRetVal); }); - } else { - console.warn(`${name} already running, returning it`); } return snodeGlobalLocks[name]; }