From 3c58f9c1e494fbe8e9f0c4f27ea15e74102359c3 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Tue, 21 Feb 2023 17:09:08 +1100 Subject: [PATCH] feat: add a hidden flag for convos and use it with the contactswrapper --- ts/components/leftpane/ActionsPanel.tsx | 5 +- .../leftpane/overlay/OverlayMessage.tsx | 8 +- ts/data/data.ts | 7 +- ts/interactions/conversationInteractions.ts | 4 +- ts/models/conversation.ts | 17 +- ts/models/conversationAttributes.ts | 3 + ts/node/database_utility.ts | 3 + ts/node/migration/sessionMigrations.ts | 254 +++++++++++++++++- ts/node/sql.ts | 16 +- ts/receiver/configMessage.ts | 12 +- ts/receiver/contentMessage.ts | 22 +- ts/receiver/queuedJob.ts | 7 +- .../opengroupV2/OpenGroupManagerV2.ts | 1 + .../conversations/ConversationController.ts | 29 +- ts/session/group/closed-group.ts | 2 + ts/session/utils/calling/CallManager.ts | 5 +- .../job_runners/jobs/AvatarDownloadJob.ts | 8 +- .../utils/libsession/libsession_utils.ts | 8 +- .../libsession/libsession_utils_contacts.ts | 66 ++--- ts/session/utils/sync/syncUtils.ts | 51 ++-- ts/state/ducks/conversations.ts | 1 + ts/state/ducks/search.ts | 20 +- ts/state/selectors/conversations.ts | 3 +- .../unit/selectors/conversations_test.ts | 10 + ts/test/session/unit/utils/SyncUtils_test.ts | 2 +- ts/test/util/blockedNumberController_test.ts | 4 +- ts/types/sqlSharedTypes.ts | 54 ++++ ts/util/blockedNumberController.ts | 18 +- 28 files changed, 492 insertions(+), 148 deletions(-) diff --git a/ts/components/leftpane/ActionsPanel.tsx b/ts/components/leftpane/ActionsPanel.tsx index 397e4073e..470f8c668 100644 --- a/ts/components/leftpane/ActionsPanel.tsx +++ b/ts/components/leftpane/ActionsPanel.tsx @@ -164,11 +164,12 @@ const setupTheme = async () => { // Do this only if we created a new Session ID, or if we already received the initial configuration message const triggerSyncIfNeeded = async () => { + const us = UserUtils.getOurPubKeyStrFromCache(); await getConversationController() - .get(UserUtils.getOurPubKeyStrFromCache()) + .get(us) .setDidApproveMe(true, true); await getConversationController() - .get(UserUtils.getOurPubKeyStrFromCache()) + .get(us) .setIsApproved(true, true); const didWeHandleAConfigurationMessageAlready = (await Data.getItemById(hasSyncedInitialConfigurationItem))?.value || false; diff --git a/ts/components/leftpane/overlay/OverlayMessage.tsx b/ts/components/leftpane/overlay/OverlayMessage.tsx index 724d260b0..8583ab6ac 100644 --- a/ts/components/leftpane/overlay/OverlayMessage.tsx +++ b/ts/components/leftpane/overlay/OverlayMessage.tsx @@ -63,8 +63,12 @@ export const OverlayMessage = () => { ); // we now want to show a conversation we just started on the leftpane, even if we did not send a message to it yet - if (!convo.isActive() || !convo.isApproved()) { - convo.set({ active_at: Date.now(), isApproved: true }); + if (!convo.isActive() || !convo.isApproved() || convo.isHidden()) { + // bump the timestamp only if we were not active before + if (!convo.isActive()) { + convo.set({ active_at: Date.now(), isApproved: true, hidden: false }); + } + convo.set({ isApproved: true, hidden: false }); await convo.commit(); } diff --git a/ts/data/data.ts b/ts/data/data.ts index da761fe80..b2b2458cf 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -7,9 +7,11 @@ import { ConversationCollection, ConversationModel } from '../models/conversatio import { ConversationAttributes } from '../models/conversationAttributes'; import { MessageCollection, MessageModel } from '../models/message'; import { MessageAttributes, MessageDirection } from '../models/messageType'; +import { StorageItem } from '../node/storage_item'; import { HexKeyPair } from '../receiver/keypairs'; import { getSodiumRenderer } from '../session/crypto'; import { PubKey } from '../session/types'; +import { fromArrayBufferToBase64, fromBase64ToArrayBuffer } from '../session/utils/String'; import { AsyncWrapper, MsgDuplicateSearchOpenGroup, @@ -20,8 +22,6 @@ import { ExpirationTimerOptions } from '../util/expiringMessages'; import { Storage } from '../util/storage'; import { channels } from './channels'; import * as dataInit from './dataInit'; -import { StorageItem } from '../node/storage_item'; -import { fromArrayBufferToBase64, fromBase64ToArrayBuffer } from '../session/utils/String'; import { cleanData } from './dataUtils'; const ERASE_SQL_KEY = 'erase-sql-key'; @@ -142,7 +142,7 @@ async function removeAllClosedGroupEncryptionKeyPairs(groupPublicKey: string): P // Conversation async function saveConversation(data: ConversationAttributes): Promise { - const cleaned = cleanData(data); + const cleaned = cleanData(data) as ConversationAttributes; /** * Merging two conversations in `handleMessageRequestResponse` introduced a bug where we would mark conversation active_at to be -Infinity. * The root issue has been fixed, but just to make sure those INVALID DATE does not show up, update those -Infinity active_at conversations to be now(), once., @@ -150,6 +150,7 @@ async function saveConversation(data: ConversationAttributes): Promise { if (cleaned.active_at === -Infinity) { cleaned.active_at = Date.now(); } + await channels.saveConversation(cleaned); } diff --git a/ts/interactions/conversationInteractions.ts b/ts/interactions/conversationInteractions.ts index affa7429b..7190f7418 100644 --- a/ts/interactions/conversationInteractions.ts +++ b/ts/interactions/conversationInteractions.ts @@ -102,14 +102,14 @@ export async function unblockConvoById(conversationId: string) { if (!conversation) { // we assume it's a block contact and not group. // this is to be able to unlock a contact we don't have a conversation with. - await BlockedNumberController.unblock(conversationId); + await BlockedNumberController.unblockAll([conversationId]); ToastUtils.pushToastSuccess('unblocked', window.i18n('unblocked')); return; } if (!conversation.id || conversation.isPublic()) { return; } - await BlockedNumberController.unblock(conversationId); + await BlockedNumberController.unblockAll([conversationId]); ToastUtils.pushToastSuccess('unblocked', window.i18n('unblocked')); await conversation.commit(); } diff --git a/ts/models/conversation.ts b/ts/models/conversation.ts index 42b68a99d..0b9d5e622 100644 --- a/ts/models/conversation.ts +++ b/ts/models/conversation.ts @@ -258,6 +258,10 @@ export class ConversationModel extends Backbone.Model { return Boolean(this.get('active_at')); } + public isHidden() { + return Boolean(this.get('hidden')); + } + public async cleanup() { await deleteExternalFilesOfConversation(this.attributes); } @@ -310,6 +314,7 @@ export class ConversationModel extends Backbone.Model { id: this.id as string, activeAt: this.get('active_at'), type: this.get('type'), + isHidden: !!this.get('hidden'), }; if (isPrivate) { @@ -562,6 +567,12 @@ export class ConversationModel extends Backbone.Model { if (this.isPublic() && !this.isOpenGroupV2()) { throw new Error('Only opengroupv2 are supported now'); } + + // we are trying to send a message to someone. If that convo is hidden in the list, make sure it is not + if (this.isHidden()) { + this.set({ hidden: false }); + await this.commit(); + } // an OpenGroupV2 message is just a visible message const chatMessageParams: VisibleMessageParams = { body, @@ -1682,7 +1693,7 @@ export class ConversationModel extends Backbone.Model { * profileKey MUST be a hex string * @param profileKey MUST be a hex string */ - public async setProfileKey(profileKey?: Uint8Array, autoCommit = true) { + public async setProfileKey(profileKey?: Uint8Array, shouldCommit = true) { if (!profileKey) { return; } @@ -1695,7 +1706,7 @@ export class ConversationModel extends Backbone.Model { profileKey: profileKeyHex, }); - if (autoCommit) { + if (shouldCommit) { await this.commit(); } } @@ -2198,6 +2209,8 @@ export async function commitConversationAndRefreshWrapper(id: string) { } // write to DB // TODO remove duplicates between db and wrapper (except nickname&name as we need them for search) + + // TODO when deleting a contact from the ConversationController, we still need to keep it in the wrapper but mark it as hidden (and we might need to add an hidden convo model field for it) await Data.saveConversation(convo.attributes); const shouldBeSavedToWrapper = SessionUtilContact.filterContactsToStoreInContactsWrapper(convo); diff --git a/ts/models/conversationAttributes.ts b/ts/models/conversationAttributes.ts index 11b3fa958..70964d624 100644 --- a/ts/models/conversationAttributes.ts +++ b/ts/models/conversationAttributes.ts @@ -94,6 +94,8 @@ export interface ConversationAttributes { * When we create a closed group v3 or get promoted to admim, we need to save the private key of that closed group. */ // identityPrivateKey?: string; + + hidden: boolean; } /** @@ -129,5 +131,6 @@ export const fillConvoAttributesWithDefaults = ( mentionedUs: false, isKickedFromGroup: false, left: false, + hidden: true, }); }; diff --git a/ts/node/database_utility.ts b/ts/node/database_utility.ts index 69a68ad85..0339029da 100644 --- a/ts/node/database_utility.ts +++ b/ts/node/database_utility.ts @@ -78,6 +78,7 @@ const allowedKeysFormatRowOfConversation = [ 'conversationIdOrigin', 'identityPrivateKey', 'markedAsUnread', + 'hidden', ]; // tslint:disable: cyclomatic-complexity export function formatRowOfConversation(row?: Record): ConversationAttributes | null { @@ -134,6 +135,7 @@ export function formatRowOfConversation(row?: Record): Conversation convo.writeCapability = Boolean(convo.writeCapability); convo.uploadCapability = Boolean(convo.uploadCapability); convo.markedAsUnread = Boolean(convo.markedAsUnread); + convo.hidden = Boolean(convo.hidden); if (!convo.conversationIdOrigin) { convo.conversationIdOrigin = undefined; @@ -214,6 +216,7 @@ const allowedKeysOfConversationAttributes = [ 'conversationIdOrigin', 'identityPrivateKey', 'markedAsUnread', + 'hidden', ]; /** diff --git a/ts/node/migration/sessionMigrations.ts b/ts/node/migration/sessionMigrations.ts index 6b6695b54..14d581f5f 100644 --- a/ts/node/migration/sessionMigrations.ts +++ b/ts/node/migration/sessionMigrations.ts @@ -1,11 +1,19 @@ import * as BetterSqlite3 from 'better-sqlite3'; -import { compact, map, pick } from 'lodash'; +import { base64_variants, from_base64, to_hex } from 'libsodium-wrappers-sumo'; +import { compact, isArray, isEmpty, map, pick } from 'lodash'; +import { + ContactsConfigWrapperInsideWorker, + UserConfigWrapperInsideWorker, +} from 'session_util_wrapper'; import { ConversationAttributes } from '../../models/conversationAttributes'; +import { fromHexToArray } from '../../session/utils/String'; +import { CONFIG_DUMP_TABLE, getContactInfoFromDBValues } from '../../types/sqlSharedTypes'; import { CLOSED_GROUP_V2_KEY_PAIRS_TABLE, CONVERSATIONS_TABLE, dropFtsAndTriggers, GUARD_NODE_TABLE, + ITEMS_TABLE, jsonToObject, LAST_HASHES_TABLE, MESSAGES_TABLE, @@ -13,6 +21,7 @@ import { objectToJSON, OPEN_GROUP_ROOMS_V2_TABLE, rebuildFtsTable, + toSqliteBoolean, } from '../database_utility'; import { sqlNode } from '../sql'; @@ -1224,6 +1233,118 @@ function updateToSessionSchemaVersion30(currentVersion: number, db: BetterSqlite console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); } +function getIdentityKeysDuringMigration(db: BetterSqlite3.Database) { + const row = db.prepare(`SELECT * FROM ${ITEMS_TABLE} WHERE id = $id;`).get({ + id: 'identityKey', + }); + + if (!row) { + return null; + } + try { + const parsedIdentityKey = jsonToObject(row.json); + if ( + !parsedIdentityKey?.value?.pubKey || + !parsedIdentityKey?.value?.ed25519KeyPair?.privateKey + ) { + return null; + } + const publicKeyBase64 = parsedIdentityKey?.value?.pubKey; + const publicKeyHex = to_hex(from_base64(publicKeyBase64, base64_variants.ORIGINAL)); + + const ed25519PrivateKeyUintArray = parsedIdentityKey?.value?.ed25519KeyPair?.privateKey; + + // TODO migrate the ed25519KeyPair for all the users already logged in to a base64 representation + const privateEd25519 = new Uint8Array(Object.values(ed25519PrivateKeyUintArray)); + + if (!privateEd25519 || isEmpty(privateEd25519)) { + return null; + } + + return { + publicKeyHex, + privateEd25519, + }; + } catch (e) { + return null; + } +} + +function insertContactIntoWrapper( + contact: any, + blockedNumbers: Array, + contactsConfigWrapper: ContactsConfigWrapperInsideWorker +) { + const dbApproved = !!contact.isApproved || false; + const dbApprovedMe = !!contact.didApproveMe || false; + const dbBlocked = blockedNumbers.includes(contact.id); + const hidden = contact.hidden || false; + const isPinned = contact.isPinned; + + const wrapperContact = getContactInfoFromDBValues({ + id: contact.id, + dbApproved, + dbApprovedMe, + dbBlocked, + dbName: contact.displayNameInProfile || undefined, + dbNickname: contact.nickname || undefined, + dbProfileKey: contact.profileKey || undefined, + dbProfileUrl: contact.avatarPointer || undefined, + isPinned, + hidden, + }); + + try { + console.info('Inserting contact into wrapper: ', wrapperContact); + contactsConfigWrapper.set(wrapperContact); + } catch (e) { + console.error( + `contactsConfigWrapper.set during migration failed with ${e.message} for id: ${contact.id}` + ); + // the wrapper did not like something. Try again with just the boolean fields as it's most likely the issue is with one of the strings (which could be recovered) + try { + console.info('Inserting edited contact into wrapper: ', contact.id); + contactsConfigWrapper.set( + getContactInfoFromDBValues({ + id: contact.id, + dbApproved, + dbApprovedMe, + dbBlocked, + dbName: undefined, + dbNickname: undefined, + dbProfileKey: undefined, + dbProfileUrl: undefined, + isPinned: false, + hidden, + }) + ); + } catch (e) { + // there is nothing else we can do here + console.error( + `contactsConfigWrapper.set during migration failed with ${e.message} for id: ${contact.id}. Skipping contact entirely` + ); + } + } +} + +function getBlockedNumbersDuringMigration(db: BetterSqlite3.Database) { + try { + const blockedItem = sqlNode.getItemById('blocked', db); + if (!blockedItem) { + throw new Error('no blocked contacts at all'); + } + const foundBlocked = blockedItem?.value; + console.warn('foundBlockedNumbers during migration', foundBlocked); + if (isArray(foundBlocked)) { + return foundBlocked; + } + return []; + } catch (e) { + console.warn('failed to read blocked numbers. Considering no blocked numbers', e.stack); + return []; + } +} + function updateToSessionSchemaVersion31(currentVersion: number, db: BetterSqlite3.Database) { const targetVersion = 31; if (currentVersion >= targetVersion) { @@ -1235,6 +1356,27 @@ function updateToSessionSchemaVersion31(currentVersion: number, db: BetterSqlite * Create a table to store our sharedConfigMessage dumps */ db.transaction(() => { + // when deleting a contact we now mark it as 'hidden' rather than overriding the `active_at` field. + // by default, conversation are hidden + db.exec( + `ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN hidden INTEGER NOT NULL DEFAULT ${toSqliteBoolean( + true + )} ;` + ); + // mark every "active" private chats as not hidden + db.prepare( + `UPDATE ${CONVERSATIONS_TABLE} SET + hidden = ${toSqliteBoolean(false)} + WHERE type = 'private' AND active_at > 0 AND (didApproveMe OR isApproved);` + ).run({}); + + // mark every not private chats (groups or communities) as not hidden (even if a group was left or we were kicked, we want it visible in the app) + db.prepare( + `UPDATE ${CONVERSATIONS_TABLE} SET + hidden = ${toSqliteBoolean(false)} + WHERE type <> 'private' AND active_at > 0;` + ).run({}); + db.exec(`CREATE TABLE configDump( variant TEXT NOT NULL, publicKey TEXT NOT NULL, @@ -1244,6 +1386,116 @@ function updateToSessionSchemaVersion31(currentVersion: number, db: BetterSqlite ); `); + try { + const keys = getIdentityKeysDuringMigration(db); + + if (!keys || !keys.privateEd25519 || isEmpty(keys.privateEd25519)) { + throw new Error('privateEd25519 was empty. Considering no users are logged in'); + } + const { privateEd25519, publicKeyHex } = keys; + const userProfileWrapper = new UserConfigWrapperInsideWorker(privateEd25519, null); + const contactsConfigWrapper = new ContactsConfigWrapperInsideWorker(privateEd25519, null); + + /** + * Setup up the User profile wrapper with what is stored in our own conversation + */ + + const ourConversation = sqlNode.getConversationById(publicKeyHex, db); + if (!ourConversation) { + throw new Error('Failed to find our logged in conversation while migrating'); + } + + // Insert the user profile into the userWrappoer + const ourDbName = ourConversation.displayNameInProfile || ''; + const ourDbProfileUrl = ourConversation.avatarPointer || ''; + const ourDbProfileKey = fromHexToArray(ourConversation.profileKey || ''); + userProfileWrapper.setName(ourDbName); + + if (ourDbProfileUrl && !isEmpty(ourDbProfileKey)) { + userProfileWrapper.setProfilePicture(ourDbProfileUrl, ourDbProfileKey); + } else { + userProfileWrapper.setProfilePicture('', new Uint8Array()); + } + + // dump the user wrapper content and save it to the DB + const userDump = userProfileWrapper.dump(); + + db.prepare( + `INSERT OR REPLACE INTO ${CONFIG_DUMP_TABLE} ( + publicKey, + variant, + combinedMessageHashes, + data + ) values ( + $publicKey, + $variant, + $combinedMessageHashes, + $data + );` + ).run({ + publicKey: publicKeyHex, + variant: 'UserConfig', + combinedMessageHashes: JSON.stringify([]), + data: userDump, + }); + + /** + * Setup up the Contacts Wrapper with all the contact details which needs to be stored in it. + */ + const blockedNumbers = getBlockedNumbersDuringMigration(db); + + // this filter is based on the `filterContactsToStoreInContactsWrapper` function. + const contactsToWriteInWrapper = db + .prepare( + `SELECT * FROM ${CONVERSATIONS_TABLE} WHERE type = 'private' AND active_at > 0 AND NOT hidden AND (didApproveMe OR isApproved) AND id <> $us AND id NOT LIKE '15%' ;` + ) + .all({ + us: publicKeyHex, + }); + + if (isArray(contactsToWriteInWrapper) && contactsToWriteInWrapper.length) { + console.warn( + '===================== Starting contact inserting into wrapper =======================' + ); + + console.info( + 'Writing contacts to wrapper during migration. length: ', + contactsToWriteInWrapper?.length + ); + + contactsToWriteInWrapper.forEach(contact => { + insertContactIntoWrapper(contact, blockedNumbers, contactsConfigWrapper); + }); + + console.warn('===================== Done with contact inserting ======================='); + } + const contactsDump = contactsConfigWrapper.dump(); + + db.prepare( + `INSERT OR REPLACE INTO ${CONFIG_DUMP_TABLE} ( + publicKey, + variant, + combinedMessageHashes, + data + ) values ( + $publicKey, + $variant, + $combinedMessageHashes, + $data + );` + ).run({ + publicKey: publicKeyHex, + variant: 'ContactsConfig', + combinedMessageHashes: JSON.stringify([]), + data: contactsDump, + }); + + // TODO we've just created the initial dumps. We have to add an initial SyncJob to the database so it is run on the next app start/ + // or find another way of adding one on the next start (store an another item in the DB and check for it on app start?) + } catch (e) { + console.error(`failed to create initial wrapper: `, e.stack); + } + // db.exec(`ALTER TABLE conversations // ADD COLUMN lastReadTimestampMs INTEGER; // ; diff --git a/ts/node/sql.ts b/ts/node/sql.ts index 925a0678d..0815c6e5b 100644 --- a/ts/node/sql.ts +++ b/ts/node/sql.ts @@ -292,8 +292,8 @@ function updateGuardNodes(nodes: Array) { function createOrUpdateItem(data: StorageItem, instance?: BetterSqlite3.Database) { createOrUpdate(ITEMS_TABLE, data, instance); } -function getItemById(id: string) { - return getById(ITEMS_TABLE, id); +function getItemById(id: string, instance?: BetterSqlite3.Database) { + return getById(ITEMS_TABLE, id, instance); } function getAllItems() { const rows = assertGlobalInstance() @@ -443,6 +443,7 @@ function saveConversation(data: ConversationAttributes, instance?: BetterSqlite3 avatarInProfile, displayNameInProfile, conversationIdOrigin, + hidden, // identityPrivateKey, } = formatted; @@ -504,6 +505,7 @@ function saveConversation(data: ConversationAttributes, instance?: BetterSqlite3 displayNameInProfile, conversationIdOrigin, // identityPrivateKey, + hidden, }); } @@ -527,8 +529,8 @@ function removeConversation(id: string | Array) { .run(id); } -function getConversationById(id: string) { - const row = assertGlobalInstance() +function getConversationById(id: string, instance?: BetterSqlite3.Database) { + const row = assertGlobalInstanceOrInstance(instance) .prepare(`SELECT * FROM ${CONVERSATIONS_TABLE} WHERE id = $id;`) .get({ id, @@ -610,7 +612,7 @@ function searchConversations(query: string) { ( displayNameInProfile LIKE $displayNameInProfile OR nickname LIKE $nickname - ) AND active_at IS NOT NULL AND active_at > 0 + ) AND active_at > 0 ORDER BY active_at DESC LIMIT $limit` ) @@ -654,6 +656,10 @@ function searchMessages(query: string, limit: number) { })); } +/** + * Search for matching messages in a specific conversation. + * Currently unused but kept as we want to add it back at some point. + */ function searchMessagesInConversation(query: string, conversationId: string, limit: number) { const rows = assertGlobalInstance() .prepare( diff --git a/ts/receiver/configMessage.ts b/ts/receiver/configMessage.ts index 3cc5f32bc..2c7dd0a2e 100644 --- a/ts/receiver/configMessage.ts +++ b/ts/receiver/configMessage.ts @@ -36,10 +36,12 @@ async function mergeConfigsWithIncomingUpdates( const toMerge = [incomingConfig.message.data]; const wrapperId = LibSessionUtil.kindToVariant(kind); try { + window.log.warn(`${wrapperId} right before merge of ${toMerge.length} arrays`); await GenericWrapperActions.merge(wrapperId, toMerge); + window.log.warn(`${wrapperId} right after merge.`); const needsPush = await GenericWrapperActions.needsPush(wrapperId); const needsDump = await GenericWrapperActions.needsDump(wrapperId); - console.info(`${wrapperId} needsPush:${needsPush} needsDump:${needsDump} `); + window.log.info(`${wrapperId} needsPush:${needsPush} needsDump:${needsDump} `); const messageHashes = [incomingConfig.messageHash]; const latestSentTimestamp = incomingConfig.envelopeTimestamp; @@ -99,15 +101,13 @@ async function handleContactsUpdate(result: IncomingConfResult): Promise