From 958f64e27f35d8e558dd8ee4c2384430b594cd7f Mon Sep 17 00:00:00 2001 From: William Grant Date: Mon, 25 Sep 2023 16:52:08 +1000 Subject: [PATCH] feat: get_expiries implementation done now just need to make sure to call both endpoints in the correct places --- ts/receiver/configMessage.ts | 58 ++++- ts/session/apis/snode_api/batchRequest.ts | 4 +- ts/session/apis/snode_api/expireRequest.ts | 2 +- .../apis/snode_api/getExpiriesRequest.ts | 220 +++++++----------- ts/session/constants.ts | 13 +- .../messages/outgoing/ExpirableMessage.ts | 1 + 6 files changed, 150 insertions(+), 148 deletions(-) diff --git a/ts/receiver/configMessage.ts b/ts/receiver/configMessage.ts index f7fc30684..934dcb900 100644 --- a/ts/receiver/configMessage.ts +++ b/ts/receiver/configMessage.ts @@ -54,6 +54,7 @@ import { addKeyPairToCacheAndDBIfNeeded, handleNewClosedGroup } from './closedGr import { HexKeyPair } from './keypairs'; import { queueAllCachedFromSource } from './receiver'; import { EnvelopePlus } from './types'; +import { getExpiriesFromSnode } from '../session/apis/snode_api/getExpiriesRequest'; function groupByVariant( incomingConfigs: Array> @@ -396,7 +397,7 @@ async function handleContactsUpdate(result: IncomingConfResult): Promise 0 + ) { + const messages2Expire = await Data.getUnreadByConversation(convoId, lastReadMessageTimestamp); + + if (messages2Expire.length) { + const messageHashes = compact( + messages2Expire + .filter(m => + Boolean( + m.get('expirationType') && + m.get('expirationType') !== 'deleteAfterSend' && + m.get('expireTimer') > 0 + ) + ) + .map(m => m.get('messageHash')) + ); + window.log.debug( + `WIP: [applyConvoVolatileUpdateFromWrapper]\nmessages2Expire: ${JSON.stringify( + messages2Expire + )}` + ); + const currentExpiryTimestamps = await getExpiriesFromSnode({ + messageHashes, + timestamp: lastReadMessageTimestamp, + }); + + window.log.debug( + `WIP: [applyConvoVolatileUpdateFromWrapper] currentExpiryTimestamps: ${JSON.stringify( + currentExpiryTimestamps + )} ` + ); + + if (currentExpiryTimestamps.length) { + for (let index = 0; index < messages2Expire.length; index++) { + if (currentExpiryTimestamps[index] === -1) { + window.log.debug( + `WIP: [applyConvoVolatileUpdateFromWrapper] invalid expiry value returned from snode. We will keep the local value.\nmessageHash: ${messageHashes[index]},` + ); + continue; + } + messages2Expire.at(index).set('expires_at', currentExpiryTimestamps[index]); + } + window.log.debug( + `WIP: [applyConvoVolatileUpdateFromWrapper] disappear after read messages updated!` + ); + } + } + } + // window.log.debug( // `applyConvoVolatileUpdateFromWrapper: ${convoId}: forcedUnread:${forcedUnread}, lastReadMessage:${lastReadMessageTimestamp}` // ); diff --git a/ts/session/apis/snode_api/batchRequest.ts b/ts/session/apis/snode_api/batchRequest.ts index 86f6b7695..99f3748c9 100644 --- a/ts/session/apis/snode_api/batchRequest.ts +++ b/ts/session/apis/snode_api/batchRequest.ts @@ -21,8 +21,8 @@ export async function doSnodeBatchRequest( associatedWith: string | null, method: 'batch' | 'sequence' = 'batch' ): Promise { - // console.warn( - // `doSnodeBatchRequest "${method}":`, + // window.log.debug( + // `WIP: [doSnodeBatchRequest] "${method}":`, // subRequests.map(m => m.method), // subRequests // ); diff --git a/ts/session/apis/snode_api/expireRequest.ts b/ts/session/apis/snode_api/expireRequest.ts index 0d74af711..91eea545c 100644 --- a/ts/session/apis/snode_api/expireRequest.ts +++ b/ts/session/apis/snode_api/expireRequest.ts @@ -350,7 +350,7 @@ export async function expireMessageOnSnode( } catch (e) { const snodeStr = snode ? `${snode.ip}:${snode.port}` : 'null'; window?.log?.warn( - `WIP: loki_message:::expireMessage - ${e.code ? `${e.code} ` : ''}${ + `WIP: [expireMessageOnSnode] ${e.code ? `${e.code} ` : ''}${ e.message } by ${ourPubKey} for ${messageHash} via snode:${snodeStr}` ); diff --git a/ts/session/apis/snode_api/getExpiriesRequest.ts b/ts/session/apis/snode_api/getExpiriesRequest.ts index a6b20e7eb..01adf378f 100644 --- a/ts/session/apis/snode_api/getExpiriesRequest.ts +++ b/ts/session/apis/snode_api/getExpiriesRequest.ts @@ -2,127 +2,68 @@ import { isEmpty, sample } from 'lodash'; import pRetry from 'p-retry'; import { Snode } from '../../../data/data'; -import { getSodiumRenderer } from '../../crypto'; -import { StringUtils, UserUtils } from '../../utils'; -import { fromBase64ToArray, fromHexToArray } from '../../utils/String'; +import { UserUtils } from '../../utils'; import { EmptySwarmError } from '../../utils/errors'; -import { UpdateExpiryOnNodeSubRequest } from './SnodeRequestTypes'; +import { GetExpiriesFromNodeSubRequest } from './SnodeRequestTypes'; import { doSnodeBatchRequest } from './batchRequest'; -import { GetNetworkTime } from './getNetworkTime'; import { getSwarmFor } from './snodePool'; import { SnodeSignature } from './snodeSignatures'; -import { ExpireMessageResultItem, ExpireMessagesResultsContent } from './types'; +import { GetExpiriesResultsContent } from './types'; import { SeedNodeAPI } from '../seed_node_api'; -async function verifyExpireMsgsResponseSignature({ - pubkey, - snodePubkey, - messageHashes, - expiry, - signature, - updated, - unchanged, -}: ExpireMessageResultItem & { - pubkey: string; - snodePubkey: any; - messageHashes: Array; -}): Promise { - if (!expiry || isEmpty(messageHashes) || isEmpty(signature)) { - window.log.warn('WIP: [verifyExpireMsgsSignature] missing argument'); - return false; - } - - const edKeyPrivBytes = fromHexToArray(snodePubkey); - const hashes = [...messageHashes, ...updated]; - if (unchanged && Object.keys(unchanged).length > 0) { - hashes.push( - ...Object.entries(unchanged) - .map(([key, value]: [string, number]) => { - return `${key}${value}`; - }) - .sort() - ); - } - - const verificationString = `${pubkey}${expiry}${hashes.join('')}`; - const verificationData = StringUtils.encode(verificationString, 'utf8'); - // window.log.debug('WIP: [verifyExpireMsgsSignature] verificationString', verificationString); - - const sodium = await getSodiumRenderer(); - try { - const isValid = sodium.crypto_sign_verify_detached( - fromBase64ToArray(signature), - new Uint8Array(verificationData), - edKeyPrivBytes - ); - - return isValid; - } catch (e) { - window.log.warn('WIP: [verifyExpireMsgsSignature] failed with: ', e.message); - return false; - } -} - -type ExpireRequestResponseResults = Record; expiry: number }>; +type GetExpiriesRequestResponseResults = Record; -async function processExpireRequestResponse( - pubkey: string, +async function processGetExpiriesRequestResponse( targetNode: Snode, - swarm: ExpireMessagesResultsContent, + expiries: GetExpiriesResultsContent, messageHashes: Array -): Promise { - if (isEmpty(swarm)) { - throw Error(`[expireOnNodes] failed! ${messageHashes}`); +): Promise { + if (isEmpty(expiries)) { + throw Error( + `[processExpireRequestResponse] Expiries are missing! ${JSON.stringify(messageHashes)}` + ); } - const results: ExpireRequestResponseResults = {}; - // window.log.debug(`WIP: [processExpireRequestResponse] initial results: `, swarm, messageHashes); + const results: GetExpiriesRequestResponseResults = {}; + // window.log.debug( + // `WIP: [processGetExpiriesRequestResponse] initial results:\nexpiries:${JSON.stringify( + // expiries + // )}` + // ); - for (const nodeKey of Object.keys(swarm)) { - if (!isEmpty(swarm[nodeKey].failed)) { - const reason = 'Unknown'; - const statusCode = '404'; - window?.log?.warn( - `WIP: loki_message:::expireMessage - Couldn't delete data from: ${ + for (const messageHash of Object.keys(expiries)) { + if (!expiries[messageHash]) { + window.log.warn( + `WIP: [processExpireRequestResponse] Expiries result failure on ${ targetNode.pubkey_ed25519 - }${reason && statusCode && ` due to an error ${reason} (${statusCode})`}` + } for messageHash ${messageHash}\n${JSON.stringify(expiries[messageHash])}` ); - // Make sure to clear the result since it failed - results[nodeKey] = { hashes: [], expiry: 0 }; + continue; } - const updatedHashes = swarm[nodeKey].updated; - const unchangedHashes = swarm[nodeKey].unchanged; - const expiry = swarm[nodeKey].expiry; - const signature = swarm[nodeKey].signature; - - // eslint-disable-next-line no-await-in-loop - const isValid = await verifyExpireMsgsResponseSignature({ - pubkey, - snodePubkey: nodeKey, - messageHashes, - expiry, - signature, - updated: updatedHashes, - unchanged: unchangedHashes, - }); - - if (!isValid) { + const expiryMs = expiries[messageHash]; + + if (!expiryMs) { window.log.warn( - 'WIP: loki_message:::expireMessage - Signature verification failed!', - messageHashes + `WIP: [processGetExpiriesRequestResponse] Missing expiry value on ${ + targetNode.pubkey_ed25519 + } so we will ignore this result (${messageHash}) and trust in the force.\n${JSON.stringify( + expiries[messageHash] + )}` ); + results[messageHash] = -1; // explicit failure value + } else { + results[messageHash] = expiryMs; } - results[nodeKey] = { hashes: updatedHashes, expiry }; } return results; } -async function expireOnNodes( +async function getExpiriesFromNodes( targetNode: Snode, - expireRequest: UpdateExpiryOnNodeSubRequest -): Promise { + expireRequest: GetExpiriesFromNodeSubRequest +): Promise> { try { const result = await doSnodeBatchRequest( [expireRequest], @@ -132,13 +73,15 @@ async function expireOnNodes( 'batch' ); + window.log.debug(`WIP: [getExpiriesFromNodes] result: ${JSON.stringify(result)}`); + if (!result || result.length !== 1) { window?.log?.warn( - `WIP: [expireOnNodes] There was an issue with the results. sessionRpc ${targetNode.ip}:${ - targetNode.port - } expireRequest ${JSON.stringify(expireRequest)}` + `WIP: [getExpiriesFromNodes] There was an issue with the results. sessionRpc ${ + targetNode.ip + }:${targetNode.port} expireRequest ${JSON.stringify(expireRequest)}` ); - return null; + return []; } // TODOLATER make sure that this code still works once disappearing messages is merged @@ -146,43 +89,41 @@ async function expireOnNodes( const firstResult = result[0]; if (firstResult.code !== 200) { - window?.log?.warn(`WIP: [expireOnNods] result is not 200 but ${firstResult.code}`); - return null; + window?.log?.warn(`WIP: [getExpiriesFromNodes] result is not 200 but ${firstResult.code}`); + return []; } try { const bodyFirstResult = firstResult.body; - const expirationResults = await processExpireRequestResponse( - expireRequest.params.pubkey, + const expirationResults = await processGetExpiriesRequestResponse( targetNode, - bodyFirstResult.swarm as ExpireMessagesResultsContent, + bodyFirstResult.expiries as GetExpiriesResultsContent, expireRequest.params.messages ); - const firstExpirationResult = Object.entries(expirationResults).at(0); - if (!firstExpirationResult) { + + if (!Object.keys(expirationResults).length) { window?.log?.warn( - 'WIP: [expireOnNodes] failed to parse "swarm" result. firstExpirationResult is null' + 'WIP: [getExpiriesFromNodes] failed to parse "get_expiries" results. expirationResults is empty' ); - throw new Error('firstExpirationResult is null'); + throw new Error('expirationResults is empty'); } - const messageHash = firstExpirationResult[0]; - const expiry = firstExpirationResult[1].expiry; + const expiryTimestamps: Array = Object.values(expirationResults); window.log.debug( - `WIP: [expireOnNodes] Success!\nHere are the results from one of the snodes.\nmessageHash: ${messageHash} \nexpiry: ${expiry} \nexpires at: ${new Date( - expiry - ).toUTCString()}\nnow: ${new Date(GetNetworkTime.getNowWithNetworkOffset()).toUTCString()}` + `WIP: [getExpiriesFromNodes] Success!\nHere are the results.\nexpirationResults: ${Object.entries( + expirationResults + )}` ); - return expiry; + return expiryTimestamps; } catch (e) { - window?.log?.warn('WIP: [expireOnNodes] Failed to parse "swarm" result: ', e.msg); + window?.log?.warn('WIP: [getExpiriesFromNodes] Failed to parse "swarm" result: ', e); } - return null; + return []; } catch (e) { window?.log?.warn( - 'WIP: [expireOnNodes] - send error:', + 'WIP: [getExpiriesFromNodes] - send error:', e, `destination ${targetNode.ip}:${targetNode.port}` ); @@ -197,7 +138,7 @@ type GetExpiriesFromSnodeProps = { async function buildGetExpiriesRequest( props: GetExpiriesFromSnodeProps -): Promise { +): Promise { const { messageHashes, timestamp } = props; const ourPubKey = UserUtils.getOurPubKeyStrFromCache(); @@ -207,7 +148,7 @@ async function buildGetExpiriesRequest( } window.log.debug( - `WIP: [buildGetExpiriesRequest] gettig expiries for messageHashes: ${messageHashes} from ${new Date( + `WIP: [buildGetExpiriesRequest] starting\nlastReadMessageTimestamp: ${new Date( timestamp ).toUTCString()}` ); @@ -224,42 +165,41 @@ async function buildGetExpiriesRequest( return null; } - const expireParams: UpdateExpiryOnNodeSubRequest = { - method: 'expire', + const getExpiriesParams: GetExpiriesFromNodeSubRequest = { + method: 'get_expiries', params: { pubkey: ourPubKey, pubkey_ed25519: signResult.pubkey_ed25519.toUpperCase(), - // TODO better testing for failed case - messages: [messageHashes], - expiry, - extend: extend || undefined, - shorten: shorten || undefined, + messages: messageHashes, + timestamp, signature: signResult?.signature, }, }; window.log.debug( - `WIP: [buildGetExpiriesRequest] ${messageHashes}\n${JSON.stringify(expireParams)}` + `WIP: [buildGetExpiriesRequest] getExpiriesParams: ${JSON.stringify(getExpiriesParams)}` ); - return expireParams; + return getExpiriesParams; } /** - * Sends an 'expire' request to the user's swarm for a specific message. - * This supports both extending and shortening a message's TTL. - * The returned TTL should be assigned to the message to expire. + * Sends an 'get_expiries' request which retrieves the current expiry timestamps of the given messages. + * + * The returned TTLs should be assigned to the given disappearing messages. * @param messageHashes the hashes of the messages we want the current expiries for * @param timestamp the time (ms) the request was initiated, must be within ±60s of the current time so using the server time is recommended. - * @returns the TTL of the message as set by the server + * @returns an arrray of the expiry timestamps (TTL) for the given messages */ -export async function getExpiriesFromSnode(props: GetExpiriesFromSnodeProps) { +export async function getExpiriesFromSnode( + props: GetExpiriesFromSnodeProps +): Promise> { const { messageHashes } = props; const ourPubKey = UserUtils.getOurPubKeyStrFromCache(); if (!ourPubKey) { window.log.error('WIP: [getExpiriesFromSnode] No pubkey found', messageHashes); - return null; + return []; } let snode: Snode | undefined; @@ -287,17 +227,17 @@ export async function getExpiriesFromSnode(props: GetExpiriesFromSnodeProps) { try { const expireRequestParams = await buildGetExpiriesRequest(props); if (!expireRequestParams) { - throw new Error(`Failed to build expire request ${JSON.stringify(props)}`); + throw new Error(`Failed to build get_expiries request ${JSON.stringify(props)}`); } - let newTTL = null; + let expiryTimestamps: Array = []; await pRetry( async () => { if (!snode) { throw new Error(`No snode found.\n${JSON.stringify(props)}`); } - newTTL = await expireOnNodes(snode, expireRequestParams); + expiryTimestamps = await getExpiriesFromNodes(snode, expireRequestParams); }, { retries: 3, @@ -311,11 +251,11 @@ export async function getExpiriesFromSnode(props: GetExpiriesFromSnodeProps) { } ); - return newTTL; + return expiryTimestamps; } catch (e) { const snodeStr = snode ? `${snode.ip}:${snode.port}` : 'null'; window?.log?.warn( - `WIP: loki_message:::expireMessage - ${e.code ? `${e.code} ` : ''}${ + `WIP: [getExpiriesFromSnode] ${e.code ? `${e.code} ` : ''}${ e.message } by ${ourPubKey} for ${messageHashes} via snode:${snodeStr}` ); diff --git a/ts/session/constants.ts b/ts/session/constants.ts index ee9aa9d3c..ef4aa107b 100644 --- a/ts/session/constants.ts +++ b/ts/session/constants.ts @@ -3,11 +3,16 @@ const minutes = seconds * 60; const hours = minutes * 60; const days = hours * 24; +/** in millisecond */ export const DURATION = { - SECONDS: seconds, // in ms - MINUTES: minutes, // in ms - HOURS: hours, // in ms - DAYS: days, // in ms + /** 1000ms */ + SECONDS: seconds, + /** 60 * 1000 = 60,000 ms */ + MINUTES: minutes, + /** 60 * 60 * 1000 = 3,600,000 ms */ + HOURS: hours, + /** 24 * 60 * 60 * 1000 = 86,400,000 ms */ + DAYS: days, }; export const TTL_DEFAULT = { diff --git a/ts/session/messages/outgoing/ExpirableMessage.ts b/ts/session/messages/outgoing/ExpirableMessage.ts index 348a32d9f..54505de7f 100644 --- a/ts/session/messages/outgoing/ExpirableMessage.ts +++ b/ts/session/messages/outgoing/ExpirableMessage.ts @@ -11,6 +11,7 @@ export interface ExpirableMessageParams extends MessageParams { export class ExpirableMessage extends ContentMessage { public readonly expirationType?: DisappearingMessageType; + /** in seconds, 0 means no expiration */ public readonly expireTimer?: number; constructor(params: ExpirableMessageParams) {