From 13bc1a21d9ed0699dfd66f937aae0838d0f92dac Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 25 Jun 2021 15:22:26 +1000 Subject: [PATCH] add test for swarmPolling variable rate --- .../conversations/ConversationController.ts | 6 +- ts/session/snode_api/swarmPolling.ts | 173 +++++----- .../unit/swarm_polling/SwarmPolling_test.ts | 307 ++++++++++++++++++ ts/test/test-utils/utils/pubkey.ts | 9 + ts/util/blockedNumberController.ts | 2 +- 5 files changed, 416 insertions(+), 81 deletions(-) create mode 100644 ts/test/session/unit/swarm_polling/SwarmPolling_test.ts diff --git a/ts/session/conversations/ConversationController.ts b/ts/session/conversations/ConversationController.ts index fdf7faed1..5d249c1b4 100644 --- a/ts/session/conversations/ConversationController.ts +++ b/ts/session/conversations/ConversationController.ts @@ -116,7 +116,7 @@ export class ConversationController { conversation.initialPromise = create(); conversation.initialPromise.then(async () => { - if (window.inboxStore) { + if (window?.inboxStore) { window.inboxStore?.dispatch( conversationActions.conversationAdded(conversation.id, conversation.getProps()) ); @@ -242,7 +242,7 @@ export class ConversationController { window.log.info(`deleteContact !isPrivate, convo removed from DB: ${id}`); this.conversations.remove(conversation); - if (window.inboxStore) { + if (window?.inboxStore) { window.inboxStore?.dispatch(conversationActions.conversationRemoved(conversation.id)); window.inboxStore?.dispatch( conversationActions.conversationChanged(conversation.id, conversation.getProps()) @@ -310,7 +310,7 @@ export class ConversationController { public reset() { this._initialPromise = Promise.resolve(); this._initialFetchComplete = false; - if (window.inboxStore) { + if (window?.inboxStore) { window.inboxStore?.dispatch(conversationActions.removeAllConversations()); } this.conversations.reset([]); diff --git a/ts/session/snode_api/swarmPolling.ts b/ts/session/snode_api/swarmPolling.ts index 99d170767..87239c427 100644 --- a/ts/session/snode_api/swarmPolling.ts +++ b/ts/session/snode_api/swarmPolling.ts @@ -1,5 +1,5 @@ import { PubKey } from '../types'; -import { getSwarmFor } from './snodePool'; +import * as snodePool from './snodePool'; import { retrieveNextMessages } from './SNodeAPI'; import { SignalService } from '../../protobuf'; import * as Receiver from '../../receiver/receiver'; @@ -13,10 +13,9 @@ import { } from '../../../ts/data/data'; import { StringUtils, UserUtils } from '../../session/utils'; -import { getConversationController } from '../conversations'; import { ConversationModel } from '../../models/conversation'; import { DURATION, SWARM_POLLING_TIMEOUT } from '../constants'; -import { ConversationController } from '../conversations/ConversationController'; +import { getConversationController } from '../conversations'; type PubkeyToHash = { [key: string]: string }; @@ -51,7 +50,7 @@ export const getSwarmPollingInstance = () => { return instance; }; -class SwarmPolling { +export class SwarmPolling { private ourPubkey: PubKey | undefined; private groupPolling: Array<{ pubkey: PubKey; lastPolledTimestamp: number }>; private readonly lastHashes: { [key: string]: PubkeyToHash }; @@ -61,10 +60,22 @@ class SwarmPolling { this.lastHashes = {}; } - public start(): void { + public async start(waitForFirstPoll = false): Promise { this.ourPubkey = UserUtils.getOurPubKeyFromCache(); this.loadGroupIds(); - void this.pollForAllKeys(); + if (waitForFirstPoll) { + await this.TEST_pollForAllKeys(); + } else { + void this.TEST_pollForAllKeys(); + } + } + + /** + * Used fo testing only + */ + public TEST_reset() { + this.ourPubkey = undefined; + this.groupPolling = []; } public addGroupId(pubkey: PubKey) { @@ -84,12 +95,89 @@ class SwarmPolling { this.groupPolling = this.groupPolling.filter(group => !pubkey.isEqual(group.pubkey)); } - private async pollOnceForKey(pubkey: PubKey, isGroup: boolean) { + /** + * Only public for testing + * As of today, we pull closed group pubkeys as follow: + * if activeAt is not set, poll only once per hour + * if activeAt is less than an hour old, poll every 5 seconds or so + * if activeAt is less than a day old, poll every minutes only. + * If activeAt is more than a day old, poll only once per hour + */ + public TEST_getPollingTimeout(convoId: PubKey) { + const convo = getConversationController().get(convoId.key); + if (!convo) { + return SWARM_POLLING_TIMEOUT.INACTIVE; + } + const activeAt = convo.get('active_at'); + if (!activeAt) { + return SWARM_POLLING_TIMEOUT.INACTIVE; + } + + const currentTimestamp = Date.now(); + + // consider that this is an active group if activeAt is less than an hour old + if (currentTimestamp - activeAt <= DURATION.HOURS * 1) { + return SWARM_POLLING_TIMEOUT.ACTIVE; + } + + if (currentTimestamp - activeAt <= DURATION.DAYS * 1) { + return SWARM_POLLING_TIMEOUT.MEDIUM_ACTIVE; + } + + return SWARM_POLLING_TIMEOUT.INACTIVE; + } + + /** + * Only public for testing + */ + public async TEST_pollForAllKeys() { + // we always poll as often as possible for our pubkey + const directPromise = this.ourPubkey + ? this.TEST_pollOnceForKey(this.ourPubkey, false) + : Promise.resolve(); + + const now = Date.now(); + const groupPromises = this.groupPolling.map(async group => { + const convoPollingTimeout = this.TEST_getPollingTimeout(group.pubkey); + + const diff = now - group.lastPolledTimestamp; + + const loggingId = + getConversationController() + .get(group.pubkey.key) + ?.idForLogging() || group.pubkey.key; + + if (diff >= convoPollingTimeout) { + (window?.log?.info || console.warn)( + `Polling for ${loggingId}; timeout: ${convoPollingTimeout} ; diff: ${diff}` + ); + return this.TEST_pollOnceForKey(group.pubkey, true); + } + (window?.log?.info || console.warn)( + `Not polling for ${loggingId}; timeout: ${convoPollingTimeout} ; diff: ${diff}` + ); + + return Promise.resolve(); + }); + try { + await Promise.all(_.concat(directPromise, groupPromises)); + } catch (e) { + (window?.log?.info || console.warn)('pollForAllKeys swallowing exception: ', e); + throw e; + } finally { + setTimeout(this.TEST_pollForAllKeys.bind(this), SWARM_POLLING_TIMEOUT.ACTIVE); + } + } + + /** + * Only exposed as public for testing + */ + public async TEST_pollOnceForKey(pubkey: PubKey, isGroup: boolean) { // NOTE: sometimes pubkey is string, sometimes it is object, so // accept both until this is fixed: const pkStr = pubkey.key; - const snodes = await getSwarmFor(pkStr); + const snodes = await snodePool.getSwarmFor(pkStr); // Select nodes for which we already have lastHashes const alreadyPolled = snodes.filter((n: Snode) => this.lastHashes[n.pubkey_ed25519]); @@ -198,75 +286,6 @@ class SwarmPolling { return newMessages; } - /** - * As of today, we pull closed group pubkeys as follow: - * if activeAt is not set, poll only once per hour - * if activeAt is less than an hour old, poll every 5 seconds or so - * if activeAt is less than a day old, poll every minutes only. - * If activeAt is more than a day old, poll only once per hour - */ - private getPollingTimeout(convoId: PubKey) { - const convo = getConversationController().get(convoId.key); - if (!convo) { - return this.pollOnceForKey(convoId, true); - } - const activeAt = convo.get('active_at'); - if (!activeAt) { - return SWARM_POLLING_TIMEOUT.INACTIVE; - } - - const currentTimestamp = Date.now(); - - // consider that this is an active group if activeAt is less than an hour old - if (currentTimestamp - activeAt <= DURATION.HOURS * 1) { - return SWARM_POLLING_TIMEOUT.ACTIVE; - } - - if (currentTimestamp - activeAt <= DURATION.DAYS * 1) { - return SWARM_POLLING_TIMEOUT.MEDIUM_ACTIVE; - } - - return SWARM_POLLING_TIMEOUT.INACTIVE; - } - - private async pollForAllKeys() { - // we always poll as often as possible for our pubkey - const directPromise = this.ourPubkey - ? this.pollOnceForKey(this.ourPubkey, false) - : Promise.resolve(); - - const now = Date.now(); - const groupPromises = this.groupPolling.map(async group => { - const convoPollingTimeout = this.getPollingTimeout(group.pubkey); - - const diff = now - group.lastPolledTimestamp; - - const loggingId = getConversationController() - .get(group.pubkey.key) - .idForLogging(); - - if (diff >= convoPollingTimeout) { - window?.log?.info( - `Polling for ${loggingId}; timeout: ${convoPollingTimeout} ; diff: ${diff}` - ); - return this.pollOnceForKey(group.pubkey, true); - } - window?.log?.info( - `Not polling for ${loggingId}; timeout: ${convoPollingTimeout} ; diff: ${diff}` - ); - - return Promise.resolve(); - }); - try { - await Promise.all(_.concat(directPromise, groupPromises)); - } catch (e) { - window?.log?.warn('pollForAllKeys swallowing exception: ', e); - throw e; - } finally { - setTimeout(this.pollForAllKeys.bind(this), SWARM_POLLING_TIMEOUT.ACTIVE); - } - } - private async updateLastHash( edkey: string, pubkey: PubKey, diff --git a/ts/test/session/unit/swarm_polling/SwarmPolling_test.ts b/ts/test/session/unit/swarm_polling/SwarmPolling_test.ts new file mode 100644 index 000000000..5ff3eca58 --- /dev/null +++ b/ts/test/session/unit/swarm_polling/SwarmPolling_test.ts @@ -0,0 +1,307 @@ +// tslint:disable: no-implicit-dependencies max-func-body-length no-unused-expression + +import chai from 'chai'; +import Sinon, * as sinon from 'sinon'; +import _, { noop } from 'lodash'; +import { describe } from 'mocha'; + +import chaiAsPromised from 'chai-as-promised'; +import { TestUtils } from '../../../test-utils'; +import { UserUtils } from '../../../../session/utils'; +import { getConversationController } from '../../../../session/conversations'; +import * as Data from '../../../../../ts/data/data'; +import { getSwarmPollingInstance, SnodePool } from '../../../../session/snode_api'; +import { SwarmPolling } from '../../../../session/snode_api/swarmPolling'; +import { SWARM_POLLING_TIMEOUT } from '../../../../session/constants'; +import { + ConversationCollection, + ConversationModel, + ConversationTypeEnum, +} from '../../../../models/conversation'; +import { PubKey } from '../../../../session/types'; +// tslint:disable: chai-vague-errors + +chai.use(chaiAsPromised as any); +chai.should(); + +const { expect } = chai; + +// tslint:disable-next-line: max-func-body-length +describe('SwarmPolling', () => { + // Initialize new stubbed cache + const sandbox = sinon.createSandbox(); + const ourPubkey = TestUtils.generateFakePubKey(); + const ourNumber = ourPubkey.key; + + let pollOnceForKeySpy: Sinon.SinonSpy; + + let swarmPolling: SwarmPolling; + + let clock: Sinon.SinonFakeTimers; + beforeEach(async () => { + // Utils Stubs + sandbox.stub(UserUtils, 'getOurPubKeyStrFromCache').returns(ourNumber); + + sandbox.stub(Data, 'getAllConversations').resolves(new ConversationCollection()); + sandbox.stub(Data, 'getItemById').resolves(); + sandbox.stub(Data, 'saveConversation').resolves(); + sandbox.stub(Data, 'getSwarmNodesForPubkey').resolves(); + sandbox.stub(SnodePool, 'getSwarmFor').resolves([]); + TestUtils.stubWindow('profileImages', { removeImagesNotInArray: noop, hasImage: noop }); + TestUtils.stubWindow('inboxStore', undefined); + const convoController = getConversationController(); + await convoController.load(); + getConversationController().getOrCreate(ourPubkey.key, ConversationTypeEnum.PRIVATE); + + swarmPolling = getSwarmPollingInstance(); + swarmPolling.TEST_reset(); + pollOnceForKeySpy = sandbox.spy(swarmPolling, 'TEST_pollOnceForKey'); + + clock = sinon.useFakeTimers(Date.now()); + }); + + afterEach(() => { + TestUtils.restoreStubs(); + sandbox.restore(); + getConversationController().reset(); + clock.restore(); + }); + + describe('getPollingTimeout', () => { + it('returns INACTIVE for non existing convo', () => { + const fakeConvo = TestUtils.generateFakePubKey(); + + expect(swarmPolling.TEST_getPollingTimeout(fakeConvo)).to.eq(SWARM_POLLING_TIMEOUT.INACTIVE); + }); + + it('returns ACTIVE for convo with less than an hour old activeAt', () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + convo.set('active_at', Date.now() - 3555); + expect(swarmPolling.TEST_getPollingTimeout(PubKey.cast(convo.id))).to.eq( + SWARM_POLLING_TIMEOUT.ACTIVE + ); + }); + + it('returns INACTIVE for convo with undefined activeAt', () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + convo.set('active_at', undefined); + expect(swarmPolling.TEST_getPollingTimeout(PubKey.cast(convo.id))).to.eq( + SWARM_POLLING_TIMEOUT.INACTIVE + ); + }); + + it('returns MEDIUM_ACTIVE for convo with activeAt of less than a day', () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + convo.set('active_at', Date.now() - 1000 * 3600 * 23); + expect(swarmPolling.TEST_getPollingTimeout(PubKey.cast(convo.id))).to.eq( + SWARM_POLLING_TIMEOUT.MEDIUM_ACTIVE + ); + }); + + it('returns INACTIVE for convo with activeAt of more than a day', () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + convo.set('active_at', Date.now() - 1000 * 3600 * 25); + expect(swarmPolling.TEST_getPollingTimeout(PubKey.cast(convo.id))).to.eq( + SWARM_POLLING_TIMEOUT.INACTIVE + ); + }); + }); + + describe('pollForAllKeys', () => { + it('does run for our pubkey even if activeAt is really old ', async () => { + const convo = getConversationController().getOrCreate( + ourNumber, + ConversationTypeEnum.PRIVATE + ); + convo.set('active_at', Date.now() - 1000 * 3600 * 25); + await swarmPolling.start(true); + + expect(pollOnceForKeySpy.callCount).to.eq(1); + expect(pollOnceForKeySpy.firstCall.args).to.deep.eq([ourPubkey, false]); + }); + + it('does run for our pubkey even if activeAt is recent ', async () => { + const convo = getConversationController().getOrCreate( + ourNumber, + ConversationTypeEnum.PRIVATE + ); + convo.set('active_at', Date.now()); + await swarmPolling.start(true); + + expect(pollOnceForKeySpy.callCount).to.eq(1); + expect(pollOnceForKeySpy.firstCall.args).to.deep.eq([ourPubkey, false]); + }); + + it('does run for group pubkey on start no matter the recent timestamp ', async () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + convo.set('active_at', Date.now()); + const groupConvoPubkey = PubKey.cast(convo.id); + swarmPolling.addGroupId(groupConvoPubkey); + await swarmPolling.start(true); + + // our pubkey will be polled for, hence the 2 + expect(pollOnceForKeySpy.callCount).to.eq(2); + expect(pollOnceForKeySpy.firstCall.args).to.deep.eq([ourPubkey, false]); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([groupConvoPubkey, true]); + }); + + it('does run for group pubkey on start no matter the old timestamp ', async () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + + convo.set('active_at', 1); + const groupConvoPubkey = PubKey.cast(convo.id); + swarmPolling.addGroupId(groupConvoPubkey); + await swarmPolling.start(true); + + // our pubkey will be polled for, hence the 2 + expect(pollOnceForKeySpy.callCount).to.eq(2); + expect(pollOnceForKeySpy.firstCall.args).to.deep.eq([ourPubkey, false]); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([groupConvoPubkey, true]); + }); + + it('does run for group pubkey on start but not another time if activeAt is old ', async () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + + convo.set('active_at', 1); + const groupConvoPubkey = PubKey.cast(convo.id); + swarmPolling.addGroupId(groupConvoPubkey); + await swarmPolling.start(true); + + await swarmPolling.TEST_pollForAllKeys(); + + expect(pollOnceForKeySpy.callCount).to.eq(3); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([groupConvoPubkey, true]); + expect(pollOnceForKeySpy.thirdCall.args).to.deep.eq([ourPubkey, false]); + }); + + it('does run twice if activeAt less than one hour ', async () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + + convo.set('active_at', Date.now()); + const groupConvoPubkey = PubKey.cast(convo.id); + swarmPolling.addGroupId(groupConvoPubkey); + await swarmPolling.start(true); + clock.tick(6000); + // no need to do that as the tick will trigger a call in all cases after 5 secs + // await swarmPolling.TEST_pollForAllKeys(); + + expect(pollOnceForKeySpy.callCount).to.eq(4); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([groupConvoPubkey, true]); + expect(pollOnceForKeySpy.thirdCall.args).to.deep.eq([ourPubkey, false]); + expect(pollOnceForKeySpy.lastCall.args).to.deep.eq([groupConvoPubkey, true]); + }); + + it('does run once only if activeAt is more than one hour', async () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + + convo.set('active_at', Date.now()); + const groupConvoPubkey = PubKey.cast(convo.id); + swarmPolling.addGroupId(groupConvoPubkey); + await swarmPolling.start(true); + + // more than hour old, we should not tick after just 5 seconds + convo.set('active_at', Date.now() - 3605 * 1000); + + clock.tick(6000); + + expect(pollOnceForKeySpy.callCount).to.eq(3); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([groupConvoPubkey, true]); + expect(pollOnceForKeySpy.thirdCall.args).to.deep.eq([ourPubkey, false]); + }); + + it('does run once if activeAt is more than 1 days old ', async () => { + const convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + + convo.set('active_at', Date.now()); + const groupConvoPubkey = PubKey.cast(convo.id); + swarmPolling.addGroupId(groupConvoPubkey); + await swarmPolling.start(true); + + // more than hour old, we should not tick after just 5 seconds + convo.set('active_at', Date.now() - 25 * 3600 * 1000); + + clock.tick(6 * 1000); // active + + expect(pollOnceForKeySpy.callCount).to.eq(3); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([groupConvoPubkey, true]); + expect(pollOnceForKeySpy.thirdCall.args).to.deep.eq([ourPubkey, false]); + }); + + describe('multiple runs', () => { + let convo: ConversationModel; + let groupConvoPubkey: PubKey; + + beforeEach(async () => { + convo = getConversationController().getOrCreate( + TestUtils.generateFakePubKeyStr(), + ConversationTypeEnum.GROUP + ); + + convo.set('active_at', Date.now()); + groupConvoPubkey = PubKey.cast(convo.id); + swarmPolling.addGroupId(groupConvoPubkey); + await swarmPolling.start(true); + }); + + it('does run twice if activeAt is more than 1 hour old and we tick more than one minute ', async () => { + pollOnceForKeySpy.resetHistory(); + // more than hour old but less than a day, we should tick after just 60 seconds + convo.set('active_at', Date.now() - 3605 * 1000); + + clock.tick(61 * 1000); // medium_active + + await swarmPolling.TEST_pollForAllKeys(); + expect(pollOnceForKeySpy.callCount).to.eq(3); + // first two calls are our pubkey + expect(pollOnceForKeySpy.firstCall.args).to.deep.eq([ourPubkey, false]); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([ourPubkey, false]); + + expect(pollOnceForKeySpy.thirdCall.args).to.deep.eq([groupConvoPubkey, true]); + }); + + it('does run twice if activeAt is more than 1 day old and we tick more than one hour ', async () => { + pollOnceForKeySpy.resetHistory(); + convo.set('active_at', Date.now() - 25 * 3600 * 1000); + + clock.tick(3700 * 1000); // inactive + + await swarmPolling.TEST_pollForAllKeys(); + expect(pollOnceForKeySpy.callCount).to.eq(3); + // first two calls are our pubkey + expect(pollOnceForKeySpy.firstCall.args).to.deep.eq([ourPubkey, false]); + expect(pollOnceForKeySpy.secondCall.args).to.deep.eq([ourPubkey, false]); + expect(pollOnceForKeySpy.thirdCall.args).to.deep.eq([groupConvoPubkey, true]); + }); + }); + }); +}); diff --git a/ts/test/test-utils/utils/pubkey.ts b/ts/test/test-utils/utils/pubkey.ts index 04f17bfc6..b73ac6d43 100644 --- a/ts/test/test-utils/utils/pubkey.ts +++ b/ts/test/test-utils/utils/pubkey.ts @@ -11,6 +11,15 @@ export function generateFakePubKey(): PubKey { return new PubKey(pubkeyString); } +export function generateFakePubKeyStr(): string { + // Generates a mock pubkey for testing + const numBytes = PubKey.PUBKEY_LEN / 2 - 1; + const hexBuffer = crypto.randomBytes(numBytes).toString('hex'); + const pubkeyString = `05${hexBuffer}`; + + return pubkeyString; +} + export function generateFakeECKeyPair(): ECKeyPair { const pubkey = generateFakePubKey().toArray(); const privKey = new Uint8Array(crypto.randomBytes(64)); diff --git a/ts/util/blockedNumberController.ts b/ts/util/blockedNumberController.ts index abdfc4b06..281f79a8c 100644 --- a/ts/util/blockedNumberController.ts +++ b/ts/util/blockedNumberController.ts @@ -1,4 +1,4 @@ -import { createOrUpdateItem, getItemById } from '../../ts/data/data'; +import { getItemById, createOrUpdateItem } from '../data/data'; import { PubKey } from '../session/types'; import { UserUtils } from '../session/utils';