From eabee8f39a2e06da7095fce3478e312f854bd8b2 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 12 Aug 2022 14:48:20 +1000 Subject: [PATCH] fix: split migrations and sql files also address PR reviews --- ts/components/leftpane/ActionsPanel.tsx | 10 +- ts/node/database_utility.ts | 63 + ts/node/migration/sessionMigrations.ts | 1179 ++++++++++ ts/node/migration/signalMigrations.ts | 684 ++++++ ts/node/sql.ts | 1944 +---------------- .../open_group_api/opengroupV2/ApiUtil.ts | 2 +- .../apis/open_group_api/sogsv3/sogsApiV3.ts | 5 + 7 files changed, 1980 insertions(+), 1907 deletions(-) create mode 100644 ts/node/migration/sessionMigrations.ts create mode 100644 ts/node/migration/signalMigrations.ts diff --git a/ts/components/leftpane/ActionsPanel.tsx b/ts/components/leftpane/ActionsPanel.tsx index 154cd9315..83b2e7fea 100644 --- a/ts/components/leftpane/ActionsPanel.tsx +++ b/ts/components/leftpane/ActionsPanel.tsx @@ -223,14 +223,14 @@ const doAppStartUp = () => { void setupTheme(); // this generates the key to encrypt attachments locally void Data.generateAttachmentKeyIfEmpty(); + + /* Postpone a little bit of the polling of sogs messages to let the swarm messages come in first. */ global.setTimeout(() => { void getOpenGroupManager().startPolling(); - }, 10000); - // trigger a sync message if needed for our other devices + }, 5000); - global.setTimeout(() => { - void triggerSyncIfNeeded(); - }, 20000); + // trigger a sync message if needed for our other devices + void triggerSyncIfNeeded(); void getSwarmPollingInstance().start(); void loadDefaultRooms(); diff --git a/ts/node/database_utility.ts b/ts/node/database_utility.ts index d692003be..5a466d9c7 100644 --- a/ts/node/database_utility.ts +++ b/ts/node/database_utility.ts @@ -1,6 +1,23 @@ import { difference, omit, pick } from 'lodash'; import { ConversationAttributes } from '../models/conversationAttributes'; +import * as BetterSqlite3 from 'better-sqlite3'; + +export const CONVERSATIONS_TABLE = 'conversations'; +export const MESSAGES_TABLE = 'messages'; +export const MESSAGES_FTS_TABLE = 'messages_fts'; +export const NODES_FOR_PUBKEY_TABLE = 'nodesForPubkey'; +export const OPEN_GROUP_ROOMS_V2_TABLE = 'openGroupRoomsV2'; +export const IDENTITY_KEYS_TABLE = 'identityKeys'; +export const GUARD_NODE_TABLE = 'guardNodes'; +export const ITEMS_TABLE = 'items'; +export const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; +export const CLOSED_GROUP_V2_KEY_PAIRS_TABLE = 'encryptionKeyPairsForClosedGroupV2'; +export const LAST_HASHES_TABLE = 'lastHashes'; + +export const HEX_KEY = /[^0-9A-Fa-f]/; +// tslint:disable: no-console + export function objectToJSON(data: Record) { return JSON.stringify(data); } @@ -211,3 +228,49 @@ export function assertValidConversationAttributes( return pick(data, allowedKeysOfConversationAttributes) as ConversationAttributes; } + +export function dropFtsAndTriggers(db: BetterSqlite3.Database) { + console.info('dropping fts5 table'); + + db.exec(` + DROP TRIGGER IF EXISTS messages_on_insert; + DROP TRIGGER IF EXISTS messages_on_delete; + DROP TRIGGER IF EXISTS messages_on_update; + DROP TABLE IF EXISTS ${MESSAGES_FTS_TABLE}; + `); +} + +export function rebuildFtsTable(db: BetterSqlite3.Database) { + console.info('rebuildFtsTable'); + db.exec(` + -- Then we create our full-text search table and populate it + CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE} + USING fts5(id UNINDEXED, body); + INSERT INTO ${MESSAGES_FTS_TABLE}(id, body) + SELECT id, body FROM ${MESSAGES_TABLE}; + -- Then we set up triggers to keep the full-text search table up to date + CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN + INSERT INTO ${MESSAGES_FTS_TABLE} ( + id, + body + ) VALUES ( + new.id, + new.body + ); + END; + CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; + END; + CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} BEGIN + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; + INSERT INTO ${MESSAGES_FTS_TABLE}( + id, + body + ) VALUES ( + new.id, + new.body + ); + END; + `); + console.info('rebuildFtsTable built'); +} diff --git a/ts/node/migration/sessionMigrations.ts b/ts/node/migration/sessionMigrations.ts new file mode 100644 index 000000000..83991f247 --- /dev/null +++ b/ts/node/migration/sessionMigrations.ts @@ -0,0 +1,1179 @@ +import * as BetterSqlite3 from 'better-sqlite3'; +import { compact, map, pick } from 'lodash'; +import { ConversationAttributes } from '../../models/conversationAttributes'; +import { + CLOSED_GROUP_V2_KEY_PAIRS_TABLE, + CONVERSATIONS_TABLE, + dropFtsAndTriggers, + GUARD_NODE_TABLE, + jsonToObject, + LAST_HASHES_TABLE, + MESSAGES_TABLE, + NODES_FOR_PUBKEY_TABLE, + objectToJSON, + OPEN_GROUP_ROOMS_V2_TABLE, + rebuildFtsTable, +} from '../database_utility'; + +import { sqlNode } from '../sql'; + +// tslint:disable: no-console quotemark one-variable-per-declaration + +function getSessionSchemaVersion(db: BetterSqlite3.Database) { + const result = db + .prepare( + ` + SELECT MAX(version) as version FROM loki_schema; + ` + ) + .get(); + if (!result || !result.version) { + return 0; + } + return result.version; +} + +function createSessionSchemaTable(db: BetterSqlite3.Database) { + db.transaction(() => { + db.exec(` + CREATE TABLE loki_schema( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + version INTEGER + ); + INSERT INTO loki_schema ( + version + ) values ( + 0 + ); + `); + })(); +} + +const LOKI_SCHEMA_VERSIONS = [ + updateToSessionSchemaVersion1, + updateToSessionSchemaVersion2, + updateToSessionSchemaVersion3, + updateToSessionSchemaVersion4, + updateToSessionSchemaVersion5, + updateToSessionSchemaVersion6, + updateToSessionSchemaVersion7, + updateToSessionSchemaVersion8, + updateToSessionSchemaVersion9, + updateToSessionSchemaVersion10, + updateToSessionSchemaVersion11, + updateToSessionSchemaVersion12, + updateToSessionSchemaVersion13, + updateToSessionSchemaVersion14, + updateToSessionSchemaVersion15, + updateToSessionSchemaVersion16, + updateToSessionSchemaVersion17, + updateToSessionSchemaVersion18, + updateToSessionSchemaVersion19, + updateToSessionSchemaVersion20, + updateToSessionSchemaVersion21, + updateToSessionSchemaVersion22, + updateToSessionSchemaVersion23, + updateToSessionSchemaVersion24, + updateToSessionSchemaVersion25, + updateToSessionSchemaVersion26, + updateToSessionSchemaVersion27, +]; + +function updateToSessionSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 1; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + db.transaction(() => { + db.exec(` + ALTER TABLE ${MESSAGES_TABLE} + ADD COLUMN serverId INTEGER; + + CREATE TABLE servers( + serverUrl STRING PRIMARY KEY ASC, + token TEXT + ); + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion2(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 2; + + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + CREATE TABLE pairingAuthorisations( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + primaryDevicePubKey VARCHAR(255), + secondaryDevicePubKey VARCHAR(255), + isGranted BOOLEAN, + json TEXT, + UNIQUE(primaryDevicePubKey, secondaryDevicePubKey) + ); + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion3(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 3; + + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + CREATE TABLE ${GUARD_NODE_TABLE}( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + ed25519PubKey VARCHAR(64) + ); + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion4(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 4; + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + DROP TABLE ${LAST_HASHES_TABLE}; + CREATE TABLE ${LAST_HASHES_TABLE}( + id TEXT, + snode TEXT, + hash TEXT, + expiresAt INTEGER, + PRIMARY KEY (id, snode) + ); + -- Add senderIdentity field to unprocessed needed for medium size groups + ALTER TABLE unprocessed ADD senderIdentity TEXT; + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion5(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 5; + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + CREATE TABLE ${NODES_FOR_PUBKEY_TABLE} ( + pubkey TEXT PRIMARY KEY, + json TEXT + ); + + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion6(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 6; + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + -- Remove RSS Feed conversations + DELETE FROM ${CONVERSATIONS_TABLE} WHERE + type = 'group' AND + id LIKE 'rss://%'; + + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion7(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 7; + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + -- Remove multi device data + + DELETE FROM pairingAuthorisations; + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion8(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 8; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + + ALTER TABLE ${MESSAGES_TABLE} + ADD COLUMN serverTimestamp INTEGER; + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion9(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 9; + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + db.transaction(() => { + const rows = db + .prepare( + `SELECT * FROM ${CONVERSATIONS_TABLE} WHERE + type = 'group' AND + id LIKE '__textsecure_group__!%'; + ` + ) + .all(); + + const conversationIdRows = db + .prepare(`SELECT id FROM ${CONVERSATIONS_TABLE} ORDER BY id ASC;`) + .all(); + + const allOldConversationIds = map(conversationIdRows, row => row.id); + rows.forEach(o => { + const oldId = o.id; + const newId = oldId.replace('__textsecure_group__!', ''); + console.log(`migrating conversation, ${oldId} to ${newId}`); + + if (allOldConversationIds.includes(newId)) { + console.log( + 'Found a duplicate conversation after prefix removing. We need to take care of it' + ); + // We have another conversation with the same future name. + // We decided to keep only the conversation with the higher number of messages + const countMessagesOld = sqlNode.getMessagesCountByConversation(oldId, db); + const countMessagesNew = sqlNode.getMessagesCountByConversation(newId, db); + + console.log(`countMessagesOld: ${countMessagesOld}, countMessagesNew: ${countMessagesNew}`); + + const deleteId = countMessagesOld > countMessagesNew ? newId : oldId; + db.prepare(`DELETE FROM ${CONVERSATIONS_TABLE} WHERE id = $deleteId;`).run({ deleteId }); + } + + const morphedObject = { + ...o, + id: newId, + }; + + db.prepare( + `UPDATE ${CONVERSATIONS_TABLE} SET + id = $newId, + json = $json + WHERE id = $oldId;` + ).run({ + newId, + json: objectToJSON(morphedObject), + oldId, + }); + }); + + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion10(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 10; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + CREATE TABLE ${CLOSED_GROUP_V2_KEY_PAIRS_TABLE} ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + groupPublicKey TEXT, + timestamp NUMBER, + json TEXT + ); + + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion11(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 11; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + function remove05PrefixFromStringIfNeeded(str: string) { + if (str.length === 66 && str.startsWith('05')) { + return str.substr(2); + } + return str; + } + + db.transaction(() => { + // the migration is called only once, so all current groups not being open groups are v1 closed group. + const allClosedGroupV1Ids = db + .prepare( + `SELECT id FROM ${CONVERSATIONS_TABLE} WHERE + type = 'group' AND + id NOT LIKE 'publicChat:%';` + ) + .all() + .map(m => m.id) as Array; + + allClosedGroupV1Ids.forEach(groupV1Id => { + try { + console.log('Migrating closed group v1 to v2: pubkey', groupV1Id); + const groupV1IdentityKey = sqlNode.getIdentityKeyById(groupV1Id, db); + if (!groupV1IdentityKey) { + return; + } + const encryptionPubKeyWithoutPrefix = remove05PrefixFromStringIfNeeded( + groupV1IdentityKey.id + ); + + // Note: + // this is what we get from getIdentityKeyById: + // { + // id: string; + // secretKey?: string; + // } + + // and this is what we want saved in db: + // { + // publicHex: string; // without prefix + // privateHex: string; + // } + const keyPair = { + publicHex: encryptionPubKeyWithoutPrefix, + privateHex: groupV1IdentityKey.secretKey, + }; + sqlNode.addClosedGroupEncryptionKeyPair(groupV1Id, keyPair, db); + } catch (e) { + console.error(e); + } + }); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion12(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 12; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + CREATE TABLE ${OPEN_GROUP_ROOMS_V2_TABLE} ( + serverUrl TEXT NOT NULL, + roomId TEXT NOT NULL, + conversationId TEXT, + json TEXT, + PRIMARY KEY (serverUrl, roomId) + ); + + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion13(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 13; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + // Clear any already deleted db entries. + // secure_delete = ON will make sure next deleted entries are overwritten with 0 right away + db.transaction(() => { + db.pragma('secure_delete = ON'); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion14(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 14; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + DROP TABLE IF EXISTS servers; + DROP TABLE IF EXISTS sessions; + DROP TABLE IF EXISTS preKeys; + DROP TABLE IF EXISTS contactPreKeys; + DROP TABLE IF EXISTS contactSignedPreKeys; + DROP TABLE IF EXISTS signedPreKeys; + DROP TABLE IF EXISTS senderKeys; + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion15(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 15; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + DROP TABLE pairingAuthorisations; + DROP TRIGGER messages_on_delete; + DROP TRIGGER messages_on_update; + `); + + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion16(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 16; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN serverHash TEXT; + ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN isDeleted BOOLEAN; + + CREATE INDEX messages_serverHash ON ${MESSAGES_TABLE} ( + serverHash + ) WHERE serverHash IS NOT NULL; + + CREATE INDEX messages_isDeleted ON ${MESSAGES_TABLE} ( + isDeleted + ) WHERE isDeleted IS NOT NULL; + + ALTER TABLE unprocessed ADD serverHash TEXT; + CREATE INDEX messages_messageHash ON unprocessed ( + serverHash + ) WHERE serverHash IS NOT NULL; + `); + + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion17(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 17; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + UPDATE ${CONVERSATIONS_TABLE} SET + json = json_set(json, '$.isApproved', 1) + `); + // remove the moderators field. As it was only used for opengroups a long time ago and whatever is there is probably unused + db.exec(` + UPDATE ${CONVERSATIONS_TABLE} SET + json = json_remove(json, '$.moderators', '$.dataMessage', '$.accessKey', '$.profileSharing', '$.sessionRestoreSeen') + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion18(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 18; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + // Dropping all pre-existing schema relating to message searching. + // Recreating the full text search and related triggers + + db.transaction(() => { + dropFtsAndTriggers(db); + rebuildFtsTable(db); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion19(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 19; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + DROP INDEX messages_schemaVersion; + ALTER TABLE ${MESSAGES_TABLE} DROP COLUMN schemaVersion; + `); + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion20(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 20; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + // looking for all private conversations, with a nickname set + const rowsToUpdate = db + .prepare( + `SELECT * FROM ${CONVERSATIONS_TABLE} WHERE type = 'private' AND (name IS NULL or name = '') AND json_extract(json, '$.nickname') <> '';` + ) + .all(); + // tslint:disable-next-line: no-void-expression + (rowsToUpdate || []).forEach(r => { + const obj = jsonToObject(r.json); + + // obj.profile.displayName is the display as this user set it. + if (obj?.nickname?.length && obj?.profile?.displayName?.length) { + // this one has a nickname set, but name is unset, set it to the displayName in the lokiProfile if it's exisitng + obj.name = obj.profile.displayName; + sqlNode.saveConversation(obj as ConversationAttributes, db); + } + }); + writeSessionSchemaVersion(targetVersion, db); + }); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion21(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 21; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + UPDATE ${CONVERSATIONS_TABLE} SET + json = json_set(json, '$.didApproveMe', 1, '$.isApproved', 1) + WHERE type = 'private'; + `); + + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion22(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 22; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(`DROP INDEX messages_duplicate_check;`); + + db.exec(` + ALTER TABLE ${MESSAGES_TABLE} DROP sourceDevice; + `); + db.exec(` + ALTER TABLE unprocessed DROP sourceDevice; + `); + db.exec(` + CREATE INDEX messages_duplicate_check ON ${MESSAGES_TABLE} ( + source, + sent_at + ); + `); + + dropFtsAndTriggers(db); + // we also want to remove the read_by it could have 20 times the same value set in the array + // we do this once, and updated the code to not allow multiple entries in read_by as we do not care about multiple entries + // (read_by is only used in private chats) + db.exec(` + UPDATE ${MESSAGES_TABLE} SET + json = json_remove(json, '$.schemaVersion', '$.recipients', '$.decrypted_at', '$.sourceDevice', '$.read_by') + `); + rebuildFtsTable(db); + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion23(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 23; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec( + ` + ALTER TABLE ${LAST_HASHES_TABLE} RENAME TO ${LAST_HASHES_TABLE}_old; + CREATE TABLE ${LAST_HASHES_TABLE}( + id TEXT, + snode TEXT, + hash TEXT, + expiresAt INTEGER, + namespace INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (id, snode, namespace) + );` + ); + + db.exec( + `INSERT INTO ${LAST_HASHES_TABLE}(id, snode, hash, expiresAt) SELECT id, snode, hash, expiresAt FROM ${LAST_HASHES_TABLE}_old;` + ); + db.exec(`DROP TABLE ${LAST_HASHES_TABLE}_old;`); + + writeSessionSchemaVersion(targetVersion, db); + })(); + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +// tslint:disable-next-line: max-func-body-length +function updateToSessionSchemaVersion24(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 24; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + // it's unlikely there is still a publicChat v1 convo in the db, but run this in a migration to be 100% sure (previously, run on app start instead) + db.prepare( + `DELETE FROM ${CONVERSATIONS_TABLE} WHERE + type = 'group' AND + id LIKE 'publicChat:1@%';` + ).run(); + + db.exec(` + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN zombies TEXT DEFAULT "[]"; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN left INTEGER; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN expireTimer INTEGER; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN mentionedUs INTEGER; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN unreadCount INTEGER; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN lastMessageStatus TEXT; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN lastMessage TEXT; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN lastJoinedTimestamp INTEGER; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN groupAdmins TEXT DEFAULT "[]"; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isKickedFromGroup INTEGER; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN subscriberCount INTEGER; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN is_medium_group INTEGER; + + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarPointer TEXT; -- this is the url of the avatar for that conversation + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarHash TEXT; -- only used for opengroup avatar. + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN nickname TEXT; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN profileKey TEXT; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN triggerNotificationsFor TEXT DEFAULT "all"; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isTrustedForAttachmentDownload INTEGER DEFAULT "FALSE"; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isPinned INTEGER DEFAULT "FALSE"; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isApproved INTEGER DEFAULT "FALSE"; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN didApproveMe INTEGER DEFAULT "FALSE"; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarInProfile TEXT; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarPathInAvatar TEXT; -- this is very temporary, removed right below + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN displayNameInProfile TEXT; + + UPDATE ${CONVERSATIONS_TABLE} SET + zombies = json_extract(json, '$.zombies'), + members = json_extract(json, '$.members'), + left = json_extract(json, '$.left'), + expireTimer = json_extract(json, '$.expireTimer'), + mentionedUs = json_extract(json, '$.mentionedUs'), + unreadCount = json_extract(json, '$.unreadCount'), + lastMessageStatus = json_extract(json, '$.lastMessageStatus'), + lastMessage = json_extract(json, '$.lastMessage'), + lastJoinedTimestamp = json_extract(json, '$.lastJoinedTimestamp'), + groupAdmins = json_extract(json, '$.groupAdmins'), + isKickedFromGroup = json_extract(json, '$.isKickedFromGroup'), + subscriberCount = json_extract(json, '$.subscriberCount'), + is_medium_group = json_extract(json, '$.is_medium_group'), + avatarPointer = json_extract(json, '$.avatarPointer'), + avatarHash = json_extract(json, '$.avatarHash'), + nickname = json_extract(json, '$.nickname'), + profileKey = json_extract(json, '$.profileKey'), + triggerNotificationsFor = json_extract(json, '$.triggerNotificationsFor'), + isTrustedForAttachmentDownload = json_extract(json, '$.isTrustedForAttachmentDownload'), + isPinned = json_extract(json, '$.isPinned'), + isApproved = json_extract(json, '$.isApproved'), + didApproveMe = json_extract(json, '$.didApproveMe'), + avatarInProfile = json_extract(json, '$.profile.avatar'),-- profile.avatar is no longer used. We rely on avatarInProfile only (for private chats and opengroups ) + avatarPathInAvatar = json_extract(json, '$.avatar.path'),-- this is very temporary + displayNameInProfile = json_extract(json, '$.profile.displayName'); + + UPDATE ${CONVERSATIONS_TABLE} SET json = json_remove(json, + '$.zombies', + '$.members', + '$.left', + '$.expireTimer', + '$.mentionedUs', + '$.unreadCount', + '$.lastMessageStatus', + '$.lastJoinedTimestamp', + '$.lastMessage', + '$.groupAdmins', + '$.isKickedFromGroup', + '$.subscriberCount', + '$.is_medium_group', + '$.avatarPointer', + '$.avatarHash', + '$.nickname', + '$.profileKey', + '$.triggerNotificationsFor', + '$.isTrustedForAttachmentDownload', + '$.isPinned', + '$.isApproved', + '$.type', + '$.version', + '$.isMe', + '$.didApproveMe', + '$.active_at', + '$.id', + '$.moderators', + '$.sessionRestoreSeen', + '$.profileName', + '$.timestamp', + '$.profile', + '$.name', + '$.profileAvatar', + '$.avatarPath + '); + + ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN json; + UPDATE ${CONVERSATIONS_TABLE} SET displayNameInProfile = name WHERE + type = 'group' AND + id NOT LIKE 'publicChat:%'; + + ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN profileName; + ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN name; + + -- we want to rely on avatarInProfile only, but it can be set either in avatarInProfile or in avatarPathInAvatar. + -- make sure to override avatarInProfile with the value from avatarPathInAvatar if avatarInProfile is unset + UPDATE ${CONVERSATIONS_TABLE} SET avatarInProfile = avatarPathInAvatar WHERE avatarInProfile IS NULL; + ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN avatarPathInAvatar; + + CREATE INDEX conversation_nickname ON ${CONVERSATIONS_TABLE} ( + nickname + ); + CREATE INDEX conversation_displayNameInProfile ON ${CONVERSATIONS_TABLE} ( + displayNameInProfile + ); + + `); + + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion25(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 25; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + // mark all conversation as read/write/upload capability to be true on migration. + // the next batch poll will update them if needed + db.exec(` + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN readCapability INTEGER DEFAULT 1; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN writeCapability INTEGER DEFAULT 1; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN uploadCapability INTEGER DEFAULT 1; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN conversationIdOrigin TEXT; + ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN avatarHash; + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarImageId INTEGER; + + CREATE INDEX messages_convo_serverID ON ${MESSAGES_TABLE} ( + serverId, + conversationId + ); + `); + + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function updateToSessionSchemaVersion26(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 26; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(` + ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN groupModerators TEXT DEFAULT "[]"; -- those are for sogs only (for closed groups we only need the groupAdmins) + `); + + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +function getRoomIdFromConversationAttributes(attributes?: ConversationAttributes | null) { + if (!attributes) { + return null; + } + const indexSemiColon = attributes.id.indexOf(':'); + const indexAt = attributes.id.indexOf('@'); + if (indexSemiColon < 0 || indexAt < 0 || indexSemiColon >= indexAt) { + return null; + } + const roomId = attributes.id.substring(indexSemiColon, indexAt); + if (roomId.length <= 0) { + return null; + } + return roomId; +} + +function updateToSessionSchemaVersion27(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 27; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + const domainNameToUse = 'open.getsession.org'; + const urlToUse = `https://${domainNameToUse}`; + + const ipToRemove = '116.203.70.33'; + + // defining this function here as this is very specific to this migration and used in a few places + function getNewConvoId(oldConvoId?: string) { + if (!oldConvoId) { + return null; + } + return ( + oldConvoId + ?.replace(`https://${ipToRemove}`, urlToUse) + // tslint:disable-next-line: no-http-string + ?.replace(`http://${ipToRemove}`, urlToUse) + ?.replace(ipToRemove, urlToUse) + ); + } + + getNewConvoId('test'); + + // tslint:disable-next-line: max-func-body-length + db.transaction(() => { + // We want to replace all the occurrences of the sogs server ip url (116.203.70.33 || http://116.203.70.33 || https://116.203.70.33) by its hostname: https://open.getsession.org + // This includes change the conversationTable, the openGroupRooms tables and every single message associated with them. + // Because the conversationId is used to link messages to conversation includes the ip/url in it... + + /** + * First, remove duplicates for the v2 opengroup table, and replace the one without duplicates with their dns name syntax + */ + + // rooms to rename are: crypto, lokinet, oxen, session, session-updates + const allSessionV2RoomsIp = sqlNode + .getAllV2OpenGroupRooms(db) + .filter(m => m.serverUrl.includes(ipToRemove)); + const allSessionV2RoomsDns = sqlNode + .getAllV2OpenGroupRooms(db) + .filter(m => m.serverUrl.includes(domainNameToUse)); + + const duplicatesRoomsIpAndDns = allSessionV2RoomsIp.filter(ip => + allSessionV2RoomsDns.some(dns => dns.roomId === ip.roomId) + ); + + const withIpButNotDuplicateRoom = allSessionV2RoomsIp.filter(ip => { + return !duplicatesRoomsIpAndDns.some(dns => dns.roomId === ip.roomId); + }); + + console.info( + 'allSessionV2RoomsIp', + allSessionV2RoomsIp.map(m => pick(m, ['serverUrl', 'roomId'])) + ); + console.info( + 'allSessionV2RoomsDns', + allSessionV2RoomsDns.map(m => pick(m, ['serverUrl', 'roomId'])) + ); + console.info( + 'duplicatesRoomsIpAndDns', + duplicatesRoomsIpAndDns.map(m => pick(m, ['serverUrl', 'roomId'])) + ); + console.info( + 'withIpButNotDuplicateRoom', + withIpButNotDuplicateRoom.map(m => pick(m, ['serverUrl', 'roomId'])) + ); + console.info( + '========> before room update:', + sqlNode + .getAllV2OpenGroupRooms(db) + .filter(m => m.serverUrl.includes(domainNameToUse) || m.serverUrl.includes(ipToRemove)) + .map(m => pick(m, ['conversationId', 'serverUrl', 'roomId'])) + ); + + // for those with duplicates, delete the one with the IP as we want to rely on the one with the DNS only now + // remove the ip ones completely which are duplicated. + // Note: this also removes the ones not duplicated, but we are recreating them just below with `saveV2OpenGroupRoom` + db.exec(`DELETE FROM ${OPEN_GROUP_ROOMS_V2_TABLE} WHERE serverUrl LIKE '%${ipToRemove}%';`); + + // for those without duplicates, override the value with the Domain Name + withIpButNotDuplicateRoom.forEach(r => { + const newConvoId = getNewConvoId(r.conversationId); + if (!newConvoId) { + return; + } + console.info( + `withIpButNotDuplicateRoom: renaming room old:${r.conversationId} with saveV2OpenGroupRoom() new- conversationId:${newConvoId}: serverUrl:${urlToUse}` + ); + sqlNode.saveV2OpenGroupRoom({ + ...r, + serverUrl: urlToUse, + conversationId: newConvoId, + }); + }); + + console.info( + '<======== after room update:', + sqlNode + .getAllV2OpenGroupRooms(db) + .filter(m => m.serverUrl.includes(domainNameToUse) || m.serverUrl.includes(ipToRemove)) + .map(m => pick(m, ['conversationId', 'serverUrl', 'roomId'])) + ); + + /** + * Then, update the conversations table by doing the same thing + */ + const allSessionV2ConvosIp = compact( + sqlNode.getAllOpenGroupV2Conversations(db).filter(m => m?.id.includes(ipToRemove)) + ); + const allSessionV2ConvosDns = compact( + sqlNode.getAllOpenGroupV2Conversations(db).filter(m => m?.id.includes(domainNameToUse)) + ); + + const duplicatesConvosIpAndDns = allSessionV2ConvosIp.filter(ip => { + const roomId = getRoomIdFromConversationAttributes(ip); + if (!roomId) { + return false; + } + return allSessionV2ConvosDns.some(dns => { + return getRoomIdFromConversationAttributes(dns) === roomId; + }); + }); + const withIpButNotDuplicateConvo = allSessionV2ConvosIp.filter(ip => { + const roomId = getRoomIdFromConversationAttributes(ip); + if (!roomId) { + return false; + } + + return !allSessionV2ConvosDns.some(dns => { + return getRoomIdFromConversationAttributes(dns) === roomId; + }); + }); + console.info('========================================'); + console.info( + 'allSessionV2ConvosIp', + allSessionV2ConvosIp.map(m => m?.id) + ); + console.info( + 'allSessionV2ConvosDns', + allSessionV2ConvosDns.map(m => m?.id) + ); + console.info( + 'duplicatesConvosIpAndDns', + duplicatesConvosIpAndDns.map(m => m?.id) + ); + console.info( + 'withIpButNotDuplicateConvo', + withIpButNotDuplicateConvo.map(m => m?.id) + ); + // for those with duplicates, delete the one with the IP as we want to rely on the one with the DNS only now + // remove the ip ones completely which are duplicated. + // Note: this also removes the ones not duplicated, but we are recreating them just below with `saveConversation` + db.exec(`DELETE FROM ${CONVERSATIONS_TABLE} WHERE id LIKE '%${ipToRemove}%';`); + + // for those without duplicates, override the value with the DNS + const convoIdsToMigrateFromIpToDns: Map = new Map(); + withIpButNotDuplicateConvo.forEach(r => { + if (!r) { + return; + } + const newConvoId = getNewConvoId(r.id); + if (!newConvoId) { + return; + } + console.info( + `withIpButNotDuplicateConvo: renaming convo old:${r.id} with saveConversation() new- conversationId:${newConvoId}` + ); + convoIdsToMigrateFromIpToDns.set(r.id, newConvoId); + sqlNode.saveConversation({ + ...r, + id: newConvoId, + }); + }); + + console.info( + 'after convos update:', + sqlNode + .getAllOpenGroupV2Conversations(db) + .filter(m => m?.id.includes(domainNameToUse) || m?.id.includes(ipToRemove)) + .map(m => m?.id) + ); + + /** + * Lastly, we need to take care of messages. + * For duplicated rooms, we drop all the messages from the IP one. (Otherwise we would need to compare each message id to not break the PRIMARY_KEY on the messageID and those are just sogs messages). + * For non duplicated rooms which got renamed to their dns ID, we override the stored conversationId in the message with the new conversationID + */ + dropFtsAndTriggers(db); + + // let's start with the non duplicateD ones, as doing so will make the duplicated one process easier + console.info('convoIdsToMigrateFromIpToDns', [...convoIdsToMigrateFromIpToDns.entries()]); + [...convoIdsToMigrateFromIpToDns.keys()].forEach(oldConvoId => { + const newConvoId = convoIdsToMigrateFromIpToDns.get(oldConvoId); + if (!newConvoId) { + return; + } + console.info(`About to migrate messages of ${oldConvoId} to ${newConvoId}`); + + db.prepare( + `UPDATE ${MESSAGES_TABLE} SET + conversationId = $newConvoId, + json = json_set(json,'$.conversationId', $newConvoId) + WHERE conversationId = $oldConvoId;` + ).run({ oldConvoId, newConvoId }); + }); + // now, the duplicated ones. We just need to move every message with a convoId matching that ip, because we already took care of the one to migrate to the dns before + console.log( + 'Count of messages to be migrated: ', + db + .prepare( + `SELECT COUNT(*) FROM ${MESSAGES_TABLE} WHERE conversationId LIKE '%${ipToRemove}%';` + ) + .get() + ); + + const messageWithIdsToUpdate = db + .prepare( + `SELECT DISTINCT conversationId FROM ${MESSAGES_TABLE} WHERE conversationID LIKE '%${ipToRemove}%'` + ) + .all(); + console.info('messageWithConversationIdsToUpdate', messageWithIdsToUpdate); + messageWithIdsToUpdate.forEach(oldConvo => { + const newConvoId = getNewConvoId(oldConvo.conversationId); + if (!newConvoId) { + return; + } + console.info('oldConvo.conversationId', oldConvo.conversationId, newConvoId); + db.prepare( + `UPDATE ${MESSAGES_TABLE} SET + conversationId = $newConvoId, + json = json_set(json,'$.conversationId', $newConvoId) + WHERE conversationId = $oldConvoId;` + ).run({ oldConvoId: oldConvo.conversationId, newConvoId }); + }); + + rebuildFtsTable(db); + + console.info( + 'removing lastMessageDeletedServerID & lastMessageFetchedServerID from rooms table' + ); + db.exec( + `UPDATE ${OPEN_GROUP_ROOMS_V2_TABLE} SET + json = json_remove(json, '$.lastMessageDeletedServerID', '$.lastMessageFetchedServerID', '$.token' );` + ); + console.info( + 'removing lastMessageDeletedServerID & lastMessageFetchedServerID from rooms table. done' + ); + writeSessionSchemaVersion(targetVersion, db); + console.log('... done'); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + +// function printTableColumns(table: string, db: BetterSqlite3.Database) { +// console.warn(db.pragma(`table_info('${table}');`)); +// } + +function writeSessionSchemaVersion(newVersion: number, db: BetterSqlite3.Database) { + db.prepare( + `INSERT INTO loki_schema( + version + ) values ( + $newVersion + )` + ).run({ newVersion }); +} + +export function updateSessionSchema(db: BetterSqlite3.Database) { + const result = db + .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';`) + .get(); + + if (!result) { + createSessionSchemaTable(db); + } + const lokiSchemaVersion = getSessionSchemaVersion(db); + console.log( + 'updateSessionSchema:', + `Current loki schema version: ${lokiSchemaVersion};`, + `Most recent schema version: ${LOKI_SCHEMA_VERSIONS.length};` + ); + for (let index = 0, max = LOKI_SCHEMA_VERSIONS.length; index < max; index += 1) { + const runSchemaUpdate = LOKI_SCHEMA_VERSIONS[index]; + runSchemaUpdate(lokiSchemaVersion, db); + } +} diff --git a/ts/node/migration/signalMigrations.ts b/ts/node/migration/signalMigrations.ts new file mode 100644 index 000000000..e407c1d4a --- /dev/null +++ b/ts/node/migration/signalMigrations.ts @@ -0,0 +1,684 @@ +import * as BetterSqlite3 from 'better-sqlite3'; +import { isNumber } from 'lodash'; +import path from 'path'; + +import { + ATTACHMENT_DOWNLOADS_TABLE, + CONVERSATIONS_TABLE, + HEX_KEY, + IDENTITY_KEYS_TABLE, + ITEMS_TABLE, + LAST_HASHES_TABLE, + MESSAGES_FTS_TABLE, + MESSAGES_TABLE, +} from '../database_utility'; +import { getAppRootPath } from '../getRootPath'; +import { updateSessionSchema } from './sessionMigrations'; + +// tslint:disable: no-console quotemark non-literal-fs-path one-variable-per-declaration +const openDbOptions = { + // tslint:disable-next-line: no-constant-condition + verbose: false ? console.log : undefined, + + nativeBinding: path.join( + getAppRootPath(), + 'node_modules', + 'better-sqlite3', + 'build', + 'Release', + 'better_sqlite3.node' + ), +}; + +// tslint:disable: no-console one-variable-per-declaration + +function updateToSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 1) { + return; + } + + console.log('updateToSchemaVersion1: starting...'); + + db.transaction(() => { + db.exec( + `CREATE TABLE ${MESSAGES_TABLE}( + id STRING PRIMARY KEY ASC, + json TEXT, + + unread INTEGER, + expires_at INTEGER, + sent BOOLEAN, + sent_at INTEGER, + schemaVersion INTEGER, + conversationId STRING, + received_at INTEGER, + source STRING, + sourceDevice STRING, + hasAttachments INTEGER, + hasFileAttachments INTEGER, + hasVisualMediaAttachments INTEGER + ); + + CREATE INDEX messages_unread ON ${MESSAGES_TABLE} ( + unread + ); + + CREATE INDEX messages_expires_at ON ${MESSAGES_TABLE} ( + expires_at + ); + + CREATE INDEX messages_receipt ON ${MESSAGES_TABLE} ( + sent_at + ); + + CREATE INDEX messages_schemaVersion ON ${MESSAGES_TABLE} ( + schemaVersion + ); + + CREATE INDEX messages_conversation ON ${MESSAGES_TABLE} ( + conversationId, + received_at + ); + + CREATE INDEX messages_duplicate_check ON ${MESSAGES_TABLE} ( + source, + sourceDevice, + sent_at + ); + + CREATE INDEX messages_hasAttachments ON ${MESSAGES_TABLE} ( + conversationId, + hasAttachments, + received_at + ); + + CREATE INDEX messages_hasFileAttachments ON ${MESSAGES_TABLE} ( + conversationId, + hasFileAttachments, + received_at + ); + + CREATE INDEX messages_hasVisualMediaAttachments ON ${MESSAGES_TABLE} ( + conversationId, + hasVisualMediaAttachments, + received_at + ); + + CREATE TABLE unprocessed( + id STRING, + timestamp INTEGER, + json TEXT + ); + + CREATE INDEX unprocessed_id ON unprocessed ( + id + ); + + CREATE INDEX unprocessed_timestamp ON unprocessed ( + timestamp + ); + + + ` + ); + db.pragma('user_version = 1'); + })(); + + // tslint:disable: no-console + console.log('updateToSchemaVersion1: success!'); +} + +function updateToSchemaVersion2(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 2) { + return; + } + + console.log('updateToSchemaVersion2: starting...'); + + db.transaction(() => { + db.exec(`ALTER TABLE ${MESSAGES_TABLE} + ADD COLUMN expireTimer INTEGER; + + ALTER TABLE ${MESSAGES_TABLE} + ADD COLUMN expirationStartTimestamp INTEGER; + + ALTER TABLE ${MESSAGES_TABLE} + ADD COLUMN type STRING; + + CREATE INDEX messages_expiring ON ${MESSAGES_TABLE} ( + expireTimer, + expirationStartTimestamp, + expires_at + ); + + UPDATE ${MESSAGES_TABLE} SET + expirationStartTimestamp = json_extract(json, '$.expirationStartTimestamp'), + expireTimer = json_extract(json, '$.expireTimer'), + type = json_extract(json, '$.type'); + + + `); + db.pragma('user_version = 2'); + })(); + + console.log('updateToSchemaVersion2: success!'); +} + +function updateToSchemaVersion3(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 3) { + return; + } + + console.log('updateToSchemaVersion3: starting...'); + + db.transaction(() => { + db.exec(` + DROP INDEX messages_expiring; + DROP INDEX messages_unread; + + CREATE INDEX messages_without_timer ON ${MESSAGES_TABLE} ( + expireTimer, + expires_at, + type + ) WHERE expires_at IS NULL AND expireTimer IS NOT NULL; + + CREATE INDEX messages_unread ON ${MESSAGES_TABLE} ( + conversationId, + unread + ) WHERE unread IS NOT NULL; + + ANALYZE; + + `); + db.pragma('user_version = 3'); + })(); + + console.log('updateToSchemaVersion3: success!'); +} + +function updateToSchemaVersion4(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 4) { + return; + } + + console.log('updateToSchemaVersion4: starting...'); + + db.transaction(() => { + db.exec(` + + CREATE TABLE ${CONVERSATIONS_TABLE}( + id STRING PRIMARY KEY ASC, + json TEXT, + + active_at INTEGER, + type STRING, + members TEXT, + name TEXT, + profileName TEXT + ); + + CREATE INDEX conversations_active ON ${CONVERSATIONS_TABLE} ( + active_at + ) WHERE active_at IS NOT NULL; + CREATE INDEX conversations_type ON ${CONVERSATIONS_TABLE} ( + type + ) WHERE type IS NOT NULL; + + `); + + db.pragma('user_version = 4'); + })(); + + console.log('updateToSchemaVersion4: success!'); +} + +function updateToSchemaVersion6(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 6) { + return; + } + console.log('updateToSchemaVersion6: starting...'); + db.transaction(() => { + db.exec(` + CREATE TABLE ${LAST_HASHES_TABLE}( + snode TEXT PRIMARY KEY, + hash TEXT, + expiresAt INTEGER + ); + + CREATE TABLE seenMessages( + hash TEXT PRIMARY KEY, + expiresAt INTEGER + ); + + + CREATE TABLE sessions( + id STRING PRIMARY KEY ASC, + number STRING, + json TEXT + ); + + CREATE INDEX sessions_number ON sessions ( + number + ) WHERE number IS NOT NULL; + + CREATE TABLE groups( + id STRING PRIMARY KEY ASC, + json TEXT + ); + + + CREATE TABLE ${IDENTITY_KEYS_TABLE}( + id STRING PRIMARY KEY ASC, + json TEXT + ); + + CREATE TABLE ${ITEMS_TABLE}( + id STRING PRIMARY KEY ASC, + json TEXT + ); + + + CREATE TABLE preKeys( + id INTEGER PRIMARY KEY ASC, + recipient STRING, + json TEXT + ); + + + CREATE TABLE signedPreKeys( + id INTEGER PRIMARY KEY ASC, + json TEXT + ); + + CREATE TABLE contactPreKeys( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + identityKeyString VARCHAR(255), + keyId INTEGER, + json TEXT + ); + + CREATE UNIQUE INDEX contact_prekey_identity_key_string_keyid ON contactPreKeys ( + identityKeyString, + keyId + ); + + CREATE TABLE contactSignedPreKeys( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + identityKeyString VARCHAR(255), + keyId INTEGER, + json TEXT + ); + + CREATE UNIQUE INDEX contact_signed_prekey_identity_key_string_keyid ON contactSignedPreKeys ( + identityKeyString, + keyId + ); + + `); + db.pragma('user_version = 6'); + })(); + + console.log('updateToSchemaVersion6: success!'); +} + +function updateToSchemaVersion7(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 7) { + return; + } + console.log('updateToSchemaVersion7: starting...'); + + db.transaction(() => { + db.exec(` + -- SQLite has been coercing our STRINGs into numbers, so we force it with TEXT + -- We create a new table then copy the data into it, since we can't modify columns + DROP INDEX sessions_number; + ALTER TABLE sessions RENAME TO sessions_old; + + CREATE TABLE sessions( + id TEXT PRIMARY KEY, + number TEXT, + json TEXT + ); + CREATE INDEX sessions_number ON sessions ( + number + ) WHERE number IS NOT NULL; + INSERT INTO sessions(id, number, json) + SELECT "+" || id, number, json FROM sessions_old; + DROP TABLE sessions_old; + `); + + db.pragma('user_version = 7'); + })(); + + console.log('updateToSchemaVersion7: success!'); +} + +function updateToSchemaVersion8(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 8) { + return; + } + console.log('updateToSchemaVersion8: starting...'); + + db.transaction(() => { + db.exec(` + -- First, we pull a new body field out of the message table's json blob + ALTER TABLE ${MESSAGES_TABLE} + ADD COLUMN body TEXT; + UPDATE ${MESSAGES_TABLE} SET body = json_extract(json, '$.body'); + + -- Then we create our full-text search table and populate it + CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE} + USING fts5(id UNINDEXED, body); + + INSERT INTO ${MESSAGES_FTS_TABLE}(id, body) + SELECT id, body FROM ${MESSAGES_TABLE}; + + -- Then we set up triggers to keep the full-text search table up to date + CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN + INSERT INTO ${MESSAGES_FTS_TABLE} ( + id, + body + ) VALUES ( + new.id, + new.body + ); + END; + CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; + END; + CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} BEGIN + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; + INSERT INTO ${MESSAGES_FTS_TABLE}( + id, + body + ) VALUES ( + new.id, + new.body + ); + END; + + `); + // For formatting search results: + // https://sqlite.org/fts5.html#the_highlight_function + // https://sqlite.org/fts5.html#the_snippet_function + db.pragma('user_version = 8'); + })(); + + console.log('updateToSchemaVersion8: success!'); +} + +function updateToSchemaVersion9(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 9) { + return; + } + console.log('updateToSchemaVersion9: starting...'); + db.transaction(() => { + db.exec(` + CREATE TABLE ${ATTACHMENT_DOWNLOADS_TABLE}( + id STRING primary key, + timestamp INTEGER, + pending INTEGER, + json TEXT + ); + + CREATE INDEX attachment_downloads_timestamp + ON ${ATTACHMENT_DOWNLOADS_TABLE} ( + timestamp + ) WHERE pending = 0; + CREATE INDEX attachment_downloads_pending + ON ${ATTACHMENT_DOWNLOADS_TABLE} ( + pending + ) WHERE pending != 0; + `); + + db.pragma('user_version = 9'); + })(); + + console.log('updateToSchemaVersion9: success!'); +} + +function updateToSchemaVersion10(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 10) { + return; + } + console.log('updateToSchemaVersion10: starting...'); + + db.transaction(() => { + db.exec(` + DROP INDEX unprocessed_id; + DROP INDEX unprocessed_timestamp; + ALTER TABLE unprocessed RENAME TO unprocessed_old; + + CREATE TABLE unprocessed( + id STRING, + timestamp INTEGER, + version INTEGER, + attempts INTEGER, + envelope TEXT, + decrypted TEXT, + source TEXT, + sourceDevice TEXT, + serverTimestamp INTEGER + ); + + CREATE INDEX unprocessed_id ON unprocessed ( + id + ); + CREATE INDEX unprocessed_timestamp ON unprocessed ( + timestamp + ); + + INSERT INTO unprocessed ( + id, + timestamp, + version, + attempts, + envelope, + decrypted, + source, + sourceDevice, + serverTimestamp + ) SELECT + id, + timestamp, + json_extract(json, '$.version'), + json_extract(json, '$.attempts'), + json_extract(json, '$.envelope'), + json_extract(json, '$.decrypted'), + json_extract(json, '$.source'), + json_extract(json, '$.sourceDevice'), + json_extract(json, '$.serverTimestamp') + FROM unprocessed_old; + + DROP TABLE unprocessed_old; + `); + + db.pragma('user_version = 10'); + })(); + console.log('updateToSchemaVersion10: success!'); +} + +function updateToSchemaVersion11(currentVersion: number, db: BetterSqlite3.Database) { + if (currentVersion >= 11) { + return; + } + console.log('updateToSchemaVersion11: starting...'); + db.transaction(() => { + db.exec(` + DROP TABLE groups; + `); + + db.pragma('user_version = 11'); + })(); + console.log('updateToSchemaVersion11: success!'); +} + +const SCHEMA_VERSIONS = [ + updateToSchemaVersion1, + updateToSchemaVersion2, + updateToSchemaVersion3, + updateToSchemaVersion4, + () => null, // version 5 was dropped + updateToSchemaVersion6, + updateToSchemaVersion7, + updateToSchemaVersion8, + updateToSchemaVersion9, + updateToSchemaVersion10, + updateToSchemaVersion11, +]; + +export function updateSchema(db: BetterSqlite3.Database) { + const sqliteVersion = getSQLiteVersion(db); + const sqlcipherVersion = getSQLCipherVersion(db); + const userVersion = getUserVersion(db); + const maxUserVersion = SCHEMA_VERSIONS.length; + const schemaVersion = getSchemaVersion(db); + + console.log('updateSchema:'); + console.log(` Current user_version: ${userVersion}`); + console.log(` Most recent db schema: ${maxUserVersion}`); + console.log(` SQLite version: ${sqliteVersion}`); + console.log(` SQLCipher version: ${sqlcipherVersion}`); + console.log(` (deprecated) schema_version: ${schemaVersion}`); + + for (let index = 0, max = SCHEMA_VERSIONS.length; index < max; index += 1) { + const runSchemaUpdate = SCHEMA_VERSIONS[index]; + runSchemaUpdate(schemaVersion, db); + } + updateSessionSchema(db); +} + +function migrateSchemaVersion(db: BetterSqlite3.Database) { + const userVersion = getUserVersion(db); + if (userVersion > 0) { + return; + } + const schemaVersion = getSchemaVersion(db); + + const newUserVersion = schemaVersion > 18 ? 16 : schemaVersion; + console.log( + 'migrateSchemaVersion: Migrating from schema_version ' + + `${schemaVersion} to user_version ${newUserVersion}` + ); + + setUserVersion(db, newUserVersion); +} + +function getUserVersion(db: BetterSqlite3.Database) { + try { + return db.pragma('user_version', { simple: true }); + } catch (e) { + console.error('getUserVersion error', e); + return 0; + } +} + +function setUserVersion(db: BetterSqlite3.Database, version: number) { + if (!isNumber(version)) { + throw new Error(`setUserVersion: version ${version} is not a number`); + } + + db.pragma(`user_version = ${version}`); +} + +export function openAndMigrateDatabase(filePath: string, key: string) { + let db; + + // First, we try to open the database without any cipher changes + try { + db = new (BetterSqlite3 as any).default(filePath, openDbOptions); + + keyDatabase(db, key); + switchToWAL(db); + migrateSchemaVersion(db); + db.pragma('secure_delete = ON'); + + return db; + } catch (error) { + if (db) { + db.close(); + } + console.log('migrateDatabase: Migration without cipher change failed', error); + } + + // If that fails, we try to open the database with 3.x compatibility to extract the + // user_version (previously stored in schema_version, blown away by cipher_migrate). + + let db1; + try { + db1 = new (BetterSqlite3 as any).default(filePath, openDbOptions); + keyDatabase(db1, key); + + // https://www.zetetic.net/blog/2018/11/30/sqlcipher-400-release/#compatability-sqlcipher-4-0-0 + db1.pragma('cipher_compatibility = 3'); + migrateSchemaVersion(db1); + db1.close(); + } catch (error) { + if (db1) { + db1.close(); + } + console.log('migrateDatabase: migrateSchemaVersion failed', error); + return null; + } + // After migrating user_version -> schema_version, we reopen database, because we can't + // migrate to the latest ciphers after we've modified the defaults. + let db2; + try { + db2 = new (BetterSqlite3 as any).default(filePath, openDbOptions); + keyDatabase(db2, key); + + db2.pragma('cipher_migrate'); + switchToWAL(db2); + + // Because foreign key support is not enabled by default! + db2.pragma('foreign_keys = OFF'); + + return db2; + } catch (error) { + if (db2) { + db2.close(); + } + console.log('migrateDatabase: switchToWAL failed'); + return null; + } +} + +function getSQLiteVersion(db: BetterSqlite3.Database) { + const { sqlite_version } = db.prepare('select sqlite_version() as sqlite_version').get(); + return sqlite_version; +} + +function getSchemaVersion(db: BetterSqlite3.Database) { + return db.pragma('schema_version', { simple: true }); +} + +function getSQLCipherVersion(db: BetterSqlite3.Database) { + return db.pragma('cipher_version', { simple: true }); +} + +export function getSQLCipherIntegrityCheck(db: BetterSqlite3.Database) { + const rows = db.pragma('cipher_integrity_check'); + if (rows.length === 0) { + return undefined; + } + return rows.map((row: any) => row.cipher_integrity_check); +} + +function keyDatabase(db: BetterSqlite3.Database, key: string) { + // https://www.zetetic.net/sqlcipher/sqlcipher-api/#key + // If the password isn't hex then we need to derive a key from it + + const deriveKey = HEX_KEY.test(key); + + const value = deriveKey ? `'${key}'` : `"x'${key}'"`; + + const pragramToRun = `key = ${value}`; + + db.pragma(pragramToRun); +} + +function switchToWAL(db: BetterSqlite3.Database) { + // https://sqlite.org/wal.html + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = FULL'); +} diff --git a/ts/node/sql.ts b/ts/node/sql.ts index 21c48e4e8..6ee2074db 100644 --- a/ts/node/sql.ts +++ b/ts/node/sql.ts @@ -6,7 +6,6 @@ import { app, clipboard, dialog, Notification } from 'electron'; import { chunk, - compact, difference, forEach, fromPairs, @@ -16,1932 +15,82 @@ import { isString, last, map, - pick, } from 'lodash'; import { redactAll } from '../util/privacy'; // checked - only node import { LocaleMessagesType } from './locale'; // checked - only node import { PubKey } from '../session/types/PubKey'; // checked - only node import { StorageItem } from './storage_item'; // checked - only node -import { getAppRootPath } from './getRootPath'; import { ConversationAttributes } from '../models/conversationAttributes'; import { arrayStrToJson, assertValidConversationAttributes, + ATTACHMENT_DOWNLOADS_TABLE, + CLOSED_GROUP_V2_KEY_PAIRS_TABLE, + CONVERSATIONS_TABLE, + dropFtsAndTriggers, formatRowOfConversation, + GUARD_NODE_TABLE, + HEX_KEY, + IDENTITY_KEYS_TABLE, + ITEMS_TABLE, jsonToObject, + LAST_HASHES_TABLE, + MESSAGES_FTS_TABLE, + MESSAGES_TABLE, + NODES_FOR_PUBKEY_TABLE, objectToJSON, + OPEN_GROUP_ROOMS_V2_TABLE, + rebuildFtsTable, toSqliteBoolean, } from './database_utility'; import { UpdateLastHashType } from '../types/sqlSharedTypes'; import { OpenGroupV2Room } from '../data/opengroups'; -// tslint:disable: no-console quotemark non-literal-fs-path one-variable-per-declaration -const openDbOptions = { - // tslint:disable-next-line: no-constant-condition - verbose: false ? console.log : undefined, - - nativeBinding: path.join( - getAppRootPath(), - 'node_modules', - 'better-sqlite3', - 'build', - 'Release', - 'better_sqlite3.node' - ), -}; - -const CONVERSATIONS_TABLE = 'conversations'; -const MESSAGES_TABLE = 'messages'; -const MESSAGES_FTS_TABLE = 'messages_fts'; -const NODES_FOR_PUBKEY_TABLE = 'nodesForPubkey'; -const OPEN_GROUP_ROOMS_V2_TABLE = 'openGroupRoomsV2'; -const IDENTITY_KEYS_TABLE = 'identityKeys'; -const GUARD_NODE_TABLE = 'guardNodes'; -const ITEMS_TABLE = 'items'; -const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; -const CLOSED_GROUP_V2_KEY_PAIRS_TABLE = 'encryptionKeyPairsForClosedGroupV2'; -const LAST_HASHES_TABLE = 'lastHashes'; - -const MAX_PUBKEYS_MEMBERS = 300; - -function getSQLiteVersion(db: BetterSqlite3.Database) { - const { sqlite_version } = db.prepare('select sqlite_version() as sqlite_version').get(); - return sqlite_version; -} - -function getSchemaVersion(db: BetterSqlite3.Database) { - return db.pragma('schema_version', { simple: true }); -} - -function getSQLCipherVersion(db: BetterSqlite3.Database) { - return db.pragma('cipher_version', { simple: true }); -} - -function getSQLCipherIntegrityCheck(db: BetterSqlite3.Database) { - const rows = db.pragma('cipher_integrity_check'); - if (rows.length === 0) { - return undefined; - } - return rows.map((row: any) => row.cipher_integrity_check); -} - -function keyDatabase(db: BetterSqlite3.Database, key: string) { - // https://www.zetetic.net/sqlcipher/sqlcipher-api/#key - // If the password isn't hex then we need to derive a key from it - - const deriveKey = HEX_KEY.test(key); - - const value = deriveKey ? `'${key}'` : `"x'${key}'"`; - - const pragramToRun = `key = ${value}`; - - db.pragma(pragramToRun); -} - -function switchToWAL(db: BetterSqlite3.Database) { - // https://sqlite.org/wal.html - db.pragma('journal_mode = WAL'); - db.pragma('synchronous = FULL'); -} - -function getSQLIntegrityCheck(db: BetterSqlite3.Database) { - const checkResult = db.pragma('quick_check', { simple: true }); - if (checkResult !== 'ok') { - return checkResult; - } - - return undefined; -} - -const HEX_KEY = /[^0-9A-Fa-f]/; - -function migrateSchemaVersion(db: BetterSqlite3.Database) { - const userVersion = getUserVersion(db); - if (userVersion > 0) { - return; - } - const schemaVersion = getSchemaVersion(db); - - const newUserVersion = schemaVersion > 18 ? 16 : schemaVersion; - console.log( - 'migrateSchemaVersion: Migrating from schema_version ' + - `${schemaVersion} to user_version ${newUserVersion}` - ); - - setUserVersion(db, newUserVersion); -} - -function getUserVersion(db: BetterSqlite3.Database) { - try { - return db.pragma('user_version', { simple: true }); - } catch (e) { - console.error('getUserVersion error', e); - return 0; - } -} - -function setUserVersion(db: BetterSqlite3.Database, version: number) { - if (!isNumber(version)) { - throw new Error(`setUserVersion: version ${version} is not a number`); - } - - db.pragma(`user_version = ${version}`); -} - -function openAndMigrateDatabase(filePath: string, key: string) { - let db; - - // First, we try to open the database without any cipher changes - try { - db = new (BetterSqlite3 as any).default(filePath, openDbOptions); - - keyDatabase(db, key); - switchToWAL(db); - migrateSchemaVersion(db); - db.pragma('secure_delete = ON'); - - return db; - } catch (error) { - if (db) { - db.close(); - } - console.log('migrateDatabase: Migration without cipher change failed', error); - } - - // If that fails, we try to open the database with 3.x compatibility to extract the - // user_version (previously stored in schema_version, blown away by cipher_migrate). - - let db1; - try { - db1 = new (BetterSqlite3 as any).default(filePath, openDbOptions); - keyDatabase(db1, key); - - // https://www.zetetic.net/blog/2018/11/30/sqlcipher-400-release/#compatability-sqlcipher-4-0-0 - db1.pragma('cipher_compatibility = 3'); - migrateSchemaVersion(db1); - db1.close(); - } catch (error) { - if (db1) { - db1.close(); - } - console.log('migrateDatabase: migrateSchemaVersion failed', error); - return null; - } - // After migrating user_version -> schema_version, we reopen database, because we can't - // migrate to the latest ciphers after we've modified the defaults. - let db2; - try { - db2 = new (BetterSqlite3 as any).default(filePath, openDbOptions); - keyDatabase(db2, key); - - db2.pragma('cipher_migrate'); - switchToWAL(db2); - - // Because foreign key support is not enabled by default! - db2.pragma('foreign_keys = OFF'); - - return db2; - } catch (error) { - if (db2) { - db2.close(); - } - console.log('migrateDatabase: switchToWAL failed'); - return null; - } -} - -function openAndSetUpSQLCipher(filePath: string, { key }: { key: string }) { - return openAndMigrateDatabase(filePath, key); -} - -function setSQLPassword(password: string) { - if (!globalInstance) { - throw new Error('setSQLPassword: db is not initialized'); - } - - // If the password isn't hex then we need to derive a key from it - const deriveKey = HEX_KEY.test(password); - const value = deriveKey ? `'${password}'` : `"x'${password}'"`; - globalInstance.pragma(`rekey = ${value}`); -} - -function vacuumDatabase(db: BetterSqlite3.Database) { - if (!db) { - throw new Error('vacuum: db is not initialized'); - } - const start = Date.now(); - console.info('Vacuuming DB. This might take a while.'); - db.exec('VACUUM;'); - console.info(`Vacuuming DB Finished in ${Date.now() - start}ms.`); -} - -function updateToSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 1) { - return; - } - - console.log('updateToSchemaVersion1: starting...'); - - db.transaction(() => { - db.exec( - `CREATE TABLE ${MESSAGES_TABLE}( - id STRING PRIMARY KEY ASC, - json TEXT, - - unread INTEGER, - expires_at INTEGER, - sent BOOLEAN, - sent_at INTEGER, - schemaVersion INTEGER, - conversationId STRING, - received_at INTEGER, - source STRING, - sourceDevice STRING, - hasAttachments INTEGER, - hasFileAttachments INTEGER, - hasVisualMediaAttachments INTEGER - ); - - CREATE INDEX messages_unread ON ${MESSAGES_TABLE} ( - unread - ); - - CREATE INDEX messages_expires_at ON ${MESSAGES_TABLE} ( - expires_at - ); - - CREATE INDEX messages_receipt ON ${MESSAGES_TABLE} ( - sent_at - ); - - CREATE INDEX messages_schemaVersion ON ${MESSAGES_TABLE} ( - schemaVersion - ); - - CREATE INDEX messages_conversation ON ${MESSAGES_TABLE} ( - conversationId, - received_at - ); - - CREATE INDEX messages_duplicate_check ON ${MESSAGES_TABLE} ( - source, - sourceDevice, - sent_at - ); - - CREATE INDEX messages_hasAttachments ON ${MESSAGES_TABLE} ( - conversationId, - hasAttachments, - received_at - ); - - CREATE INDEX messages_hasFileAttachments ON ${MESSAGES_TABLE} ( - conversationId, - hasFileAttachments, - received_at - ); - - CREATE INDEX messages_hasVisualMediaAttachments ON ${MESSAGES_TABLE} ( - conversationId, - hasVisualMediaAttachments, - received_at - ); - - CREATE TABLE unprocessed( - id STRING, - timestamp INTEGER, - json TEXT - ); - - CREATE INDEX unprocessed_id ON unprocessed ( - id - ); - - CREATE INDEX unprocessed_timestamp ON unprocessed ( - timestamp - ); - - - ` - ); - db.pragma('user_version = 1'); - })(); - - console.log('updateToSchemaVersion1: success!'); -} - -function updateToSchemaVersion2(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 2) { - return; - } - - console.log('updateToSchemaVersion2: starting...'); - - db.transaction(() => { - db.exec(`ALTER TABLE ${MESSAGES_TABLE} - ADD COLUMN expireTimer INTEGER; - - ALTER TABLE ${MESSAGES_TABLE} - ADD COLUMN expirationStartTimestamp INTEGER; - - ALTER TABLE ${MESSAGES_TABLE} - ADD COLUMN type STRING; - - CREATE INDEX messages_expiring ON ${MESSAGES_TABLE} ( - expireTimer, - expirationStartTimestamp, - expires_at - ); - - UPDATE ${MESSAGES_TABLE} SET - expirationStartTimestamp = json_extract(json, '$.expirationStartTimestamp'), - expireTimer = json_extract(json, '$.expireTimer'), - type = json_extract(json, '$.type'); - - - `); - db.pragma('user_version = 2'); - })(); - - console.log('updateToSchemaVersion2: success!'); -} - -function updateToSchemaVersion3(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 3) { - return; - } - - console.log('updateToSchemaVersion3: starting...'); - - db.transaction(() => { - db.exec(` - DROP INDEX messages_expiring; - DROP INDEX messages_unread; - - CREATE INDEX messages_without_timer ON ${MESSAGES_TABLE} ( - expireTimer, - expires_at, - type - ) WHERE expires_at IS NULL AND expireTimer IS NOT NULL; - - CREATE INDEX messages_unread ON ${MESSAGES_TABLE} ( - conversationId, - unread - ) WHERE unread IS NOT NULL; - - ANALYZE; - - `); - db.pragma('user_version = 3'); - })(); - - console.log('updateToSchemaVersion3: success!'); -} - -function updateToSchemaVersion4(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 4) { - return; - } - - console.log('updateToSchemaVersion4: starting...'); - - db.transaction(() => { - db.exec(` - - CREATE TABLE ${CONVERSATIONS_TABLE}( - id STRING PRIMARY KEY ASC, - json TEXT, - - active_at INTEGER, - type STRING, - members TEXT, - name TEXT, - profileName TEXT - ); - - CREATE INDEX conversations_active ON ${CONVERSATIONS_TABLE} ( - active_at - ) WHERE active_at IS NOT NULL; - CREATE INDEX conversations_type ON ${CONVERSATIONS_TABLE} ( - type - ) WHERE type IS NOT NULL; - - `); - - db.pragma('user_version = 4'); - })(); - - console.log('updateToSchemaVersion4: success!'); -} - -function updateToSchemaVersion6(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 6) { - return; - } - console.log('updateToSchemaVersion6: starting...'); - db.transaction(() => { - db.exec(` - CREATE TABLE ${LAST_HASHES_TABLE}( - snode TEXT PRIMARY KEY, - hash TEXT, - expiresAt INTEGER - ); - - CREATE TABLE seenMessages( - hash TEXT PRIMARY KEY, - expiresAt INTEGER - ); - - - CREATE TABLE sessions( - id STRING PRIMARY KEY ASC, - number STRING, - json TEXT - ); - - CREATE INDEX sessions_number ON sessions ( - number - ) WHERE number IS NOT NULL; - - CREATE TABLE groups( - id STRING PRIMARY KEY ASC, - json TEXT - ); - - - CREATE TABLE ${IDENTITY_KEYS_TABLE}( - id STRING PRIMARY KEY ASC, - json TEXT - ); - - CREATE TABLE ${ITEMS_TABLE}( - id STRING PRIMARY KEY ASC, - json TEXT - ); - - - CREATE TABLE preKeys( - id INTEGER PRIMARY KEY ASC, - recipient STRING, - json TEXT - ); - - - CREATE TABLE signedPreKeys( - id INTEGER PRIMARY KEY ASC, - json TEXT - ); - - CREATE TABLE contactPreKeys( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - identityKeyString VARCHAR(255), - keyId INTEGER, - json TEXT - ); - - CREATE UNIQUE INDEX contact_prekey_identity_key_string_keyid ON contactPreKeys ( - identityKeyString, - keyId - ); - - CREATE TABLE contactSignedPreKeys( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - identityKeyString VARCHAR(255), - keyId INTEGER, - json TEXT - ); - - CREATE UNIQUE INDEX contact_signed_prekey_identity_key_string_keyid ON contactSignedPreKeys ( - identityKeyString, - keyId - ); - - `); - db.pragma('user_version = 6'); - })(); - - console.log('updateToSchemaVersion6: success!'); -} - -function updateToSchemaVersion7(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 7) { - return; - } - console.log('updateToSchemaVersion7: starting...'); - - db.transaction(() => { - db.exec(` - -- SQLite has been coercing our STRINGs into numbers, so we force it with TEXT - -- We create a new table then copy the data into it, since we can't modify columns - DROP INDEX sessions_number; - ALTER TABLE sessions RENAME TO sessions_old; - - CREATE TABLE sessions( - id TEXT PRIMARY KEY, - number TEXT, - json TEXT - ); - CREATE INDEX sessions_number ON sessions ( - number - ) WHERE number IS NOT NULL; - INSERT INTO sessions(id, number, json) - SELECT "+" || id, number, json FROM sessions_old; - DROP TABLE sessions_old; - `); - - db.pragma('user_version = 7'); - })(); - - console.log('updateToSchemaVersion7: success!'); -} - -function updateToSchemaVersion8(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 8) { - return; - } - console.log('updateToSchemaVersion8: starting...'); - - db.transaction(() => { - db.exec(` - -- First, we pull a new body field out of the message table's json blob - ALTER TABLE ${MESSAGES_TABLE} - ADD COLUMN body TEXT; - UPDATE ${MESSAGES_TABLE} SET body = json_extract(json, '$.body'); - - -- Then we create our full-text search table and populate it - CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE} - USING fts5(id UNINDEXED, body); - - INSERT INTO ${MESSAGES_FTS_TABLE}(id, body) - SELECT id, body FROM ${MESSAGES_TABLE}; - - -- Then we set up triggers to keep the full-text search table up to date - CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN - INSERT INTO ${MESSAGES_FTS_TABLE} ( - id, - body - ) VALUES ( - new.id, - new.body - ); - END; - CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN - DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; - END; - CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} BEGIN - DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; - INSERT INTO ${MESSAGES_FTS_TABLE}( - id, - body - ) VALUES ( - new.id, - new.body - ); - END; - - `); - // For formatting search results: - // https://sqlite.org/fts5.html#the_highlight_function - // https://sqlite.org/fts5.html#the_snippet_function - db.pragma('user_version = 8'); - })(); - - console.log('updateToSchemaVersion8: success!'); -} - -function updateToSchemaVersion9(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 9) { - return; - } - console.log('updateToSchemaVersion9: starting...'); - db.transaction(() => { - db.exec(` - CREATE TABLE ${ATTACHMENT_DOWNLOADS_TABLE}( - id STRING primary key, - timestamp INTEGER, - pending INTEGER, - json TEXT - ); - - CREATE INDEX attachment_downloads_timestamp - ON ${ATTACHMENT_DOWNLOADS_TABLE} ( - timestamp - ) WHERE pending = 0; - CREATE INDEX attachment_downloads_pending - ON ${ATTACHMENT_DOWNLOADS_TABLE} ( - pending - ) WHERE pending != 0; - `); - - db.pragma('user_version = 9'); - })(); - - console.log('updateToSchemaVersion9: success!'); -} - -function updateToSchemaVersion10(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 10) { - return; - } - console.log('updateToSchemaVersion10: starting...'); - - db.transaction(() => { - db.exec(` - DROP INDEX unprocessed_id; - DROP INDEX unprocessed_timestamp; - ALTER TABLE unprocessed RENAME TO unprocessed_old; - - CREATE TABLE unprocessed( - id STRING, - timestamp INTEGER, - version INTEGER, - attempts INTEGER, - envelope TEXT, - decrypted TEXT, - source TEXT, - sourceDevice TEXT, - serverTimestamp INTEGER - ); - - CREATE INDEX unprocessed_id ON unprocessed ( - id - ); - CREATE INDEX unprocessed_timestamp ON unprocessed ( - timestamp - ); - - INSERT INTO unprocessed ( - id, - timestamp, - version, - attempts, - envelope, - decrypted, - source, - sourceDevice, - serverTimestamp - ) SELECT - id, - timestamp, - json_extract(json, '$.version'), - json_extract(json, '$.attempts'), - json_extract(json, '$.envelope'), - json_extract(json, '$.decrypted'), - json_extract(json, '$.source'), - json_extract(json, '$.sourceDevice'), - json_extract(json, '$.serverTimestamp') - FROM unprocessed_old; - - DROP TABLE unprocessed_old; - `); - - db.pragma('user_version = 10'); - })(); - console.log('updateToSchemaVersion10: success!'); -} - -function updateToSchemaVersion11(currentVersion: number, db: BetterSqlite3.Database) { - if (currentVersion >= 11) { - return; - } - console.log('updateToSchemaVersion11: starting...'); - db.transaction(() => { - db.exec(` - DROP TABLE groups; - `); - - db.pragma('user_version = 11'); - })(); - console.log('updateToSchemaVersion11: success!'); -} - -const SCHEMA_VERSIONS = [ - updateToSchemaVersion1, - updateToSchemaVersion2, - updateToSchemaVersion3, - updateToSchemaVersion4, - () => null, // version 5 was dropped - updateToSchemaVersion6, - updateToSchemaVersion7, - updateToSchemaVersion8, - updateToSchemaVersion9, - updateToSchemaVersion10, - updateToSchemaVersion11, -]; - -function updateSchema(db: BetterSqlite3.Database) { - const sqliteVersion = getSQLiteVersion(db); - const sqlcipherVersion = getSQLCipherVersion(db); - const userVersion = getUserVersion(db); - const maxUserVersion = SCHEMA_VERSIONS.length; - const schemaVersion = getSchemaVersion(db); - - console.log('updateSchema:'); - console.log(` Current user_version: ${userVersion}`); - console.log(` Most recent db schema: ${maxUserVersion}`); - console.log(` SQLite version: ${sqliteVersion}`); - console.log(` SQLCipher version: ${sqlcipherVersion}`); - console.log(` (deprecated) schema_version: ${schemaVersion}`); - - for (let index = 0, max = SCHEMA_VERSIONS.length; index < max; index += 1) { - const runSchemaUpdate = SCHEMA_VERSIONS[index]; - runSchemaUpdate(schemaVersion, db); - } - updateLokiSchema(db); -} - -const LOKI_SCHEMA_VERSIONS = [ - updateToLokiSchemaVersion1, - updateToLokiSchemaVersion2, - updateToLokiSchemaVersion3, - updateToLokiSchemaVersion4, - updateToLokiSchemaVersion5, - updateToLokiSchemaVersion6, - updateToLokiSchemaVersion7, - updateToLokiSchemaVersion8, - updateToLokiSchemaVersion9, - updateToLokiSchemaVersion10, - updateToLokiSchemaVersion11, - updateToLokiSchemaVersion12, - updateToLokiSchemaVersion13, - updateToLokiSchemaVersion14, - updateToLokiSchemaVersion15, - updateToLokiSchemaVersion16, - updateToLokiSchemaVersion17, - updateToLokiSchemaVersion18, - updateToLokiSchemaVersion19, - updateToLokiSchemaVersion20, - updateToLokiSchemaVersion21, - updateToLokiSchemaVersion22, - updateToLokiSchemaVersion23, - updateToLokiSchemaVersion24, - updateToLokiSchemaVersion25, - updateToLokiSchemaVersion26, - updateToLokiSchemaVersion27, -]; - -function updateToLokiSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 1; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - db.transaction(() => { - db.exec(` - ALTER TABLE ${MESSAGES_TABLE} - ADD COLUMN serverId INTEGER; - - CREATE TABLE servers( - serverUrl STRING PRIMARY KEY ASC, - token TEXT - ); - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion2(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 2; - - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - CREATE TABLE pairingAuthorisations( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - primaryDevicePubKey VARCHAR(255), - secondaryDevicePubKey VARCHAR(255), - isGranted BOOLEAN, - json TEXT, - UNIQUE(primaryDevicePubKey, secondaryDevicePubKey) - ); - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion3(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 3; - - if (currentVersion >= targetVersion) { - return; - } - - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - CREATE TABLE ${GUARD_NODE_TABLE}( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - ed25519PubKey VARCHAR(64) - ); - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion4(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 4; - if (currentVersion >= targetVersion) { - return; - } - - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - DROP TABLE ${LAST_HASHES_TABLE}; - CREATE TABLE ${LAST_HASHES_TABLE}( - id TEXT, - snode TEXT, - hash TEXT, - expiresAt INTEGER, - PRIMARY KEY (id, snode) - ); - -- Add senderIdentity field to unprocessed needed for medium size groups - ALTER TABLE unprocessed ADD senderIdentity TEXT; - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion5(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 5; - if (currentVersion >= targetVersion) { - return; - } - - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - CREATE TABLE ${NODES_FOR_PUBKEY_TABLE} ( - pubkey TEXT PRIMARY KEY, - json TEXT - ); - - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion6(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 6; - if (currentVersion >= targetVersion) { - return; - } - - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - -- Remove RSS Feed conversations - DELETE FROM ${CONVERSATIONS_TABLE} WHERE - type = 'group' AND - id LIKE 'rss://%'; - - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion7(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 7; - if (currentVersion >= targetVersion) { - return; - } - - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - -- Remove multi device data - - DELETE FROM pairingAuthorisations; - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion8(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 8; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - - ALTER TABLE ${MESSAGES_TABLE} - ADD COLUMN serverTimestamp INTEGER; - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion9(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 9; - if (currentVersion >= targetVersion) { - return; - } - - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - db.transaction(() => { - const rows = db - .prepare( - `SELECT * FROM ${CONVERSATIONS_TABLE} WHERE - type = 'group' AND - id LIKE '__textsecure_group__!%'; - ` - ) - .all(); - - const conversationIdRows = db - .prepare(`SELECT id FROM ${CONVERSATIONS_TABLE} ORDER BY id ASC;`) - .all(); - - const allOldConversationIds = map(conversationIdRows, row => row.id); - rows.forEach(o => { - const oldId = o.id; - const newId = oldId.replace('__textsecure_group__!', ''); - console.log(`migrating conversation, ${oldId} to ${newId}`); - - if (allOldConversationIds.includes(newId)) { - console.log( - 'Found a duplicate conversation after prefix removing. We need to take care of it' - ); - // We have another conversation with the same future name. - // We decided to keep only the conversation with the higher number of messages - const countMessagesOld = getMessagesCountByConversation(oldId, db); - const countMessagesNew = getMessagesCountByConversation(newId, db); - - console.log(`countMessagesOld: ${countMessagesOld}, countMessagesNew: ${countMessagesNew}`); - - const deleteId = countMessagesOld > countMessagesNew ? newId : oldId; - db.prepare(`DELETE FROM ${CONVERSATIONS_TABLE} WHERE id = $deleteId;`).run({ deleteId }); - } - - const morphedObject = { - ...o, - id: newId, - }; - - db.prepare( - `UPDATE ${CONVERSATIONS_TABLE} SET - id = $newId, - json = $json - WHERE id = $oldId;` - ).run({ - newId, - json: objectToJSON(morphedObject), - oldId, - }); - }); - - writeLokiSchemaVersion(targetVersion, db); - })(); - - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion10(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 10; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - CREATE TABLE ${CLOSED_GROUP_V2_KEY_PAIRS_TABLE} ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - groupPublicKey TEXT, - timestamp NUMBER, - json TEXT - ); - - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion11(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 11; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - // the migration is called only once, so all current groups not being open groups are v1 closed group. - const allClosedGroupV1Ids = db - .prepare( - `SELECT id FROM ${CONVERSATIONS_TABLE} WHERE - type = 'group' AND - id NOT LIKE 'publicChat:%';` - ) - .all() - .map(m => m.id) as Array; - - allClosedGroupV1Ids.forEach(groupV1Id => { - try { - console.log('Migrating closed group v1 to v2: pubkey', groupV1Id); - const groupV1IdentityKey = getIdentityKeyById(groupV1Id, db); - if (!groupV1IdentityKey) { - return; - } - const encryptionPubKeyWithoutPrefix = remove05PrefixFromStringIfNeeded( - groupV1IdentityKey.id - ); - - // Note: - // this is what we get from getIdentityKeyById: - // { - // id: string; - // secretKey?: string; - // } - - // and this is what we want saved in db: - // { - // publicHex: string; // without prefix - // privateHex: string; - // } - const keyPair = { - publicHex: encryptionPubKeyWithoutPrefix, - privateHex: groupV1IdentityKey.secretKey, - }; - addClosedGroupEncryptionKeyPair(groupV1Id, keyPair, db); - } catch (e) { - console.error(e); - } - }); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion12(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 12; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - CREATE TABLE ${OPEN_GROUP_ROOMS_V2_TABLE} ( - serverUrl TEXT NOT NULL, - roomId TEXT NOT NULL, - conversationId TEXT, - json TEXT, - PRIMARY KEY (serverUrl, roomId) - ); - - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion13(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 13; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - // Clear any already deleted db entries. - // secure_delete = ON will make sure next deleted entries are overwritten with 0 right away - db.transaction(() => { - db.pragma('secure_delete = ON'); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion14(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 14; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - DROP TABLE IF EXISTS servers; - DROP TABLE IF EXISTS sessions; - DROP TABLE IF EXISTS preKeys; - DROP TABLE IF EXISTS contactPreKeys; - DROP TABLE IF EXISTS contactSignedPreKeys; - DROP TABLE IF EXISTS signedPreKeys; - DROP TABLE IF EXISTS senderKeys; - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion15(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 15; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - DROP TABLE pairingAuthorisations; - DROP TRIGGER messages_on_delete; - DROP TRIGGER messages_on_update; - `); - - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion16(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 16; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN serverHash TEXT; - ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN isDeleted BOOLEAN; - - CREATE INDEX messages_serverHash ON ${MESSAGES_TABLE} ( - serverHash - ) WHERE serverHash IS NOT NULL; - - CREATE INDEX messages_isDeleted ON ${MESSAGES_TABLE} ( - isDeleted - ) WHERE isDeleted IS NOT NULL; - - ALTER TABLE unprocessed ADD serverHash TEXT; - CREATE INDEX messages_messageHash ON unprocessed ( - serverHash - ) WHERE serverHash IS NOT NULL; - `); - - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion17(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 17; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - db.transaction(() => { - db.exec(` - UPDATE ${CONVERSATIONS_TABLE} SET - json = json_set(json, '$.isApproved', 1) - `); - // remove the moderators field. As it was only used for opengroups a long time ago and whatever is there is probably unused - db.exec(` - UPDATE ${CONVERSATIONS_TABLE} SET - json = json_remove(json, '$.moderators', '$.dataMessage', '$.accessKey', '$.profileSharing', '$.sessionRestoreSeen') - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function dropFtsAndTriggers(db: BetterSqlite3.Database) { - console.info('dropping fts5 table'); - - db.exec(` - DROP TRIGGER IF EXISTS messages_on_insert; - DROP TRIGGER IF EXISTS messages_on_delete; - DROP TRIGGER IF EXISTS messages_on_update; - DROP TABLE IF EXISTS ${MESSAGES_FTS_TABLE}; -`); -} - -function rebuildFtsTable(db: BetterSqlite3.Database) { - console.info('rebuildFtsTable'); - db.exec(` - -- Then we create our full-text search table and populate it - CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE} - USING fts5(id UNINDEXED, body); - INSERT INTO ${MESSAGES_FTS_TABLE}(id, body) - SELECT id, body FROM ${MESSAGES_TABLE}; - -- Then we set up triggers to keep the full-text search table up to date - CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN - INSERT INTO ${MESSAGES_FTS_TABLE} ( - id, - body - ) VALUES ( - new.id, - new.body - ); - END; - CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN - DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; - END; - CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} BEGIN - DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; - INSERT INTO ${MESSAGES_FTS_TABLE}( - id, - body - ) VALUES ( - new.id, - new.body - ); - END; - `); - console.info('rebuildFtsTable built'); -} - -function updateToLokiSchemaVersion18(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 18; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - // Dropping all pre-existing schema relating to message searching. - // Recreating the full text search and related triggers - - db.transaction(() => { - dropFtsAndTriggers(db); - rebuildFtsTable(db); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion19(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 19; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - DROP INDEX messages_schemaVersion; - ALTER TABLE ${MESSAGES_TABLE} DROP COLUMN schemaVersion; - `); - writeLokiSchemaVersion(targetVersion, db); - })(); - - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion20(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 20; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - // looking for all private conversations, with a nickname set - const rowsToUpdate = db - .prepare( - `SELECT * FROM ${CONVERSATIONS_TABLE} WHERE type = 'private' AND (name IS NULL or name = '') AND json_extract(json, '$.nickname') <> '';` - ) - .all(); - // tslint:disable-next-line: no-void-expression - (rowsToUpdate || []).forEach(r => { - const obj = jsonToObject(r.json); - - // obj.profile.displayName is the display as this user set it. - if (obj?.nickname?.length && obj?.profile?.displayName?.length) { - // this one has a nickname set, but name is unset, set it to the displayName in the lokiProfile if it's exisitng - obj.name = obj.profile.displayName; - saveConversation(obj as ConversationAttributes, db); - } - }); - writeLokiSchemaVersion(targetVersion, db); - }); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion21(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 21; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(` - UPDATE ${CONVERSATIONS_TABLE} SET - json = json_set(json, '$.didApproveMe', 1, '$.isApproved', 1) - WHERE type = 'private'; - `); - - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion22(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 22; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec(`DROP INDEX messages_duplicate_check;`); - - db.exec(` - ALTER TABLE ${MESSAGES_TABLE} DROP sourceDevice; - `); - db.exec(` - ALTER TABLE unprocessed DROP sourceDevice; - `); - db.exec(` - CREATE INDEX messages_duplicate_check ON ${MESSAGES_TABLE} ( - source, - sent_at - ); - `); - - dropFtsAndTriggers(db); - // we also want to remove the read_by it could have 20 times the same value set in the array - // we do this once, and updated the code to not allow multiple entries in read_by as we do not care about multiple entries - // (read_by is only used in private chats) - db.exec(` - UPDATE ${MESSAGES_TABLE} SET - json = json_remove(json, '$.schemaVersion', '$.recipients', '$.decrypted_at', '$.sourceDevice', '$.read_by') - `); - rebuildFtsTable(db); - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion23(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 23; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - db.exec( - ` - ALTER TABLE ${LAST_HASHES_TABLE} RENAME TO ${LAST_HASHES_TABLE}_old; - CREATE TABLE ${LAST_HASHES_TABLE}( - id TEXT, - snode TEXT, - hash TEXT, - expiresAt INTEGER, - namespace INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (id, snode, namespace) - );` - ); - - db.exec( - `INSERT INTO ${LAST_HASHES_TABLE}(id, snode, hash, expiresAt) SELECT id, snode, hash, expiresAt FROM ${LAST_HASHES_TABLE}_old;` - ); - db.exec(`DROP TABLE ${LAST_HASHES_TABLE}_old;`); - - writeLokiSchemaVersion(targetVersion, db); - })(); - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -// tslint:disable-next-line: max-func-body-length -function updateToLokiSchemaVersion24(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 24; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - // it's unlikely there is still a publicChat v1 convo in the db, but run this in a migration to be 100% sure (previously, run on app start instead) - db.prepare( - `DELETE FROM ${CONVERSATIONS_TABLE} WHERE - type = 'group' AND - id LIKE 'publicChat:1@%';` - ).run(); - - db.exec(` - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN zombies TEXT DEFAULT "[]"; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN left INTEGER; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN expireTimer INTEGER; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN mentionedUs INTEGER; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN unreadCount INTEGER; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN lastMessageStatus TEXT; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN lastMessage TEXT; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN lastJoinedTimestamp INTEGER; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN groupAdmins TEXT DEFAULT "[]"; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isKickedFromGroup INTEGER; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN subscriberCount INTEGER; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN is_medium_group INTEGER; - - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarPointer TEXT; -- this is the url of the avatar for that conversation - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarHash TEXT; -- only used for opengroup avatar. - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN nickname TEXT; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN profileKey TEXT; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN triggerNotificationsFor TEXT DEFAULT "all"; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isTrustedForAttachmentDownload INTEGER DEFAULT "FALSE"; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isPinned INTEGER DEFAULT "FALSE"; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN isApproved INTEGER DEFAULT "FALSE"; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN didApproveMe INTEGER DEFAULT "FALSE"; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarInProfile TEXT; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarPathInAvatar TEXT; -- this is very temporary, removed right below - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN displayNameInProfile TEXT; - - UPDATE ${CONVERSATIONS_TABLE} SET - zombies = json_extract(json, '$.zombies'), - members = json_extract(json, '$.members'), - left = json_extract(json, '$.left'), - expireTimer = json_extract(json, '$.expireTimer'), - mentionedUs = json_extract(json, '$.mentionedUs'), - unreadCount = json_extract(json, '$.unreadCount'), - lastMessageStatus = json_extract(json, '$.lastMessageStatus'), - lastMessage = json_extract(json, '$.lastMessage'), - lastJoinedTimestamp = json_extract(json, '$.lastJoinedTimestamp'), - groupAdmins = json_extract(json, '$.groupAdmins'), - isKickedFromGroup = json_extract(json, '$.isKickedFromGroup'), - subscriberCount = json_extract(json, '$.subscriberCount'), - is_medium_group = json_extract(json, '$.is_medium_group'), - avatarPointer = json_extract(json, '$.avatarPointer'), - avatarHash = json_extract(json, '$.avatarHash'), - nickname = json_extract(json, '$.nickname'), - profileKey = json_extract(json, '$.profileKey'), - triggerNotificationsFor = json_extract(json, '$.triggerNotificationsFor'), - isTrustedForAttachmentDownload = json_extract(json, '$.isTrustedForAttachmentDownload'), - isPinned = json_extract(json, '$.isPinned'), - isApproved = json_extract(json, '$.isApproved'), - didApproveMe = json_extract(json, '$.didApproveMe'), - avatarInProfile = json_extract(json, '$.profile.avatar'),-- profile.avatar is no longer used. We rely on avatarInProfile only (for private chats and opengroups ) - avatarPathInAvatar = json_extract(json, '$.avatar.path'),-- this is very temporary - displayNameInProfile = json_extract(json, '$.profile.displayName'); - - UPDATE ${CONVERSATIONS_TABLE} SET json = json_remove(json, - '$.zombies', - '$.members', - '$.left', - '$.expireTimer', - '$.mentionedUs', - '$.unreadCount', - '$.lastMessageStatus', - '$.lastJoinedTimestamp', - '$.lastMessage', - '$.groupAdmins', - '$.isKickedFromGroup', - '$.subscriberCount', - '$.is_medium_group', - '$.avatarPointer', - '$.avatarHash', - '$.nickname', - '$.profileKey', - '$.triggerNotificationsFor', - '$.isTrustedForAttachmentDownload', - '$.isPinned', - '$.isApproved', - '$.type', - '$.version', - '$.isMe', - '$.didApproveMe', - '$.active_at', - '$.id', - '$.moderators', - '$.sessionRestoreSeen', - '$.profileName', - '$.timestamp', - '$.profile', - '$.name', - '$.profileAvatar', - '$.avatarPath - '); - - ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN json; - UPDATE ${CONVERSATIONS_TABLE} SET displayNameInProfile = name WHERE - type = 'group' AND - id NOT LIKE 'publicChat:%'; - - ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN profileName; - ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN name; - - -- we want to rely on avatarInProfile only, but it can be set either in avatarInProfile or in avatarPathInAvatar. - -- make sure to override avatarInProfile with the value from avatarPathInAvatar if avatarInProfile is unset - UPDATE ${CONVERSATIONS_TABLE} SET avatarInProfile = avatarPathInAvatar WHERE avatarInProfile IS NULL; - ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN avatarPathInAvatar; - - CREATE INDEX conversation_nickname ON ${CONVERSATIONS_TABLE} ( - nickname - ); - CREATE INDEX conversation_displayNameInProfile ON ${CONVERSATIONS_TABLE} ( - displayNameInProfile - ); - - `); - - writeLokiSchemaVersion(targetVersion, db); - })(); - - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -function updateToLokiSchemaVersion25(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 25; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - - db.transaction(() => { - // mark all conversation as read/write/upload capability to be true on migration. - // the next batch poll will update them if needed - db.exec(` - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN readCapability INTEGER DEFAULT 1; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN writeCapability INTEGER DEFAULT 1; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN uploadCapability INTEGER DEFAULT 1; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN conversationIdOrigin TEXT; - ALTER TABLE ${CONVERSATIONS_TABLE} DROP COLUMN avatarHash; - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN avatarImageId INTEGER; - - CREATE INDEX messages_convo_serverID ON ${MESSAGES_TABLE} ( - serverId, - conversationId - ); - `); +import { + getSQLCipherIntegrityCheck, + openAndMigrateDatabase, + updateSchema, +} from './migration/signalMigrations'; - writeLokiSchemaVersion(targetVersion, db); - })(); +// tslint:disable: no-console function-name non-literal-fs-path - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} +const MAX_PUBKEYS_MEMBERS = 300; -function updateToLokiSchemaVersion26(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 26; - if (currentVersion >= targetVersion) { - return; +function getSQLIntegrityCheck(db: BetterSqlite3.Database) { + const checkResult = db.pragma('quick_check', { simple: true }); + if (checkResult !== 'ok') { + return checkResult; } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - db.transaction(() => { - db.exec(` - ALTER TABLE ${CONVERSATIONS_TABLE} ADD COLUMN groupModerators TEXT DEFAULT "[]"; -- those are for sogs only (for closed groups we only need the groupAdmins) - `); - - writeLokiSchemaVersion(targetVersion, db); - })(); - - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); + return undefined; } -function getRoomIdFromConversationAttributes(attributes?: ConversationAttributes | null) { - if (!attributes) { - return null; - } - const indexSemiColon = attributes.id.indexOf(':'); - const indexAt = attributes.id.indexOf('@'); - if (indexSemiColon < 0 || indexAt < 0 || indexSemiColon >= indexAt) { - return null; - } - const roomId = attributes.id.substring(indexSemiColon, indexAt); - if (roomId.length <= 0) { - return null; - } - return roomId; +function openAndSetUpSQLCipher(filePath: string, { key }: { key: string }) { + return openAndMigrateDatabase(filePath, key); } -function updateToLokiSchemaVersion27(currentVersion: number, db: BetterSqlite3.Database) { - const targetVersion = 27; - if (currentVersion >= targetVersion) { - return; - } - console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); - const domainNameToUse = 'open.getsession.org'; - const urlToUse = `https://${domainNameToUse}`; - - const ipToRemove = '116.203.70.33'; - - // defining this function here as this is very specific to this migration and used in a few places - function getNewConvoId(oldConvoId?: string) { - if (!oldConvoId) { - return null; - } - return ( - oldConvoId - ?.replace(`https://${ipToRemove}`, urlToUse) - // tslint:disable-next-line: no-http-string - ?.replace(`http://${ipToRemove}`, urlToUse) - ?.replace(ipToRemove, urlToUse) - ); +function setSQLPassword(password: string) { + if (!globalInstance) { + throw new Error('setSQLPassword: db is not initialized'); } - // tslint:disable-next-line: max-func-body-length - db.transaction(() => { - // We want to replace all the occurrences of the sogs server ip url (116.203.70.33 || http://116.203.70.33 || https://116.203.70.33) by its hostname: https://open.getsession.org - // This includes change the conversationTable, the openGroupRooms tables and every single message associated with them. - // Because the conversationId is used to link messages to conversation includes the ip/url in it... - - /** - * First, remove duplicates for the v2 opengroup table, and replace the one without duplicates with their dns name syntax - */ - - // rooms to rename are: crypto, lokinet, oxen, session, session-updates - const allSessionV2RoomsIp = getAllV2OpenGroupRooms(db).filter(m => - m.serverUrl.includes(ipToRemove) - ); - const allSessionV2RoomsDns = getAllV2OpenGroupRooms(db).filter(m => - m.serverUrl.includes(domainNameToUse) - ); - - const duplicatesRoomsIpAndDns = allSessionV2RoomsIp.filter(ip => - allSessionV2RoomsDns.some(dns => dns.roomId === ip.roomId) - ); - - const withIpButNotDuplicateRoom = allSessionV2RoomsIp.filter(ip => { - return !duplicatesRoomsIpAndDns.some(dns => dns.roomId === ip.roomId); - }); - console.info( - 'allSessionV2RoomsIp', - allSessionV2RoomsIp.map(m => pick(m, ['serverUrl', 'roomId'])) - ); - console.info( - 'allSessionV2RoomsDns', - allSessionV2RoomsDns.map(m => pick(m, ['serverUrl', 'roomId'])) - ); - console.info( - 'duplicatesRoomsIpAndDns', - duplicatesRoomsIpAndDns.map(m => pick(m, ['serverUrl', 'roomId'])) - ); - console.info( - 'withIpButNotDuplicateRoom', - withIpButNotDuplicateRoom.map(m => pick(m, ['serverUrl', 'roomId'])) - ); - - console.info( - '========> before room update:', - getAllV2OpenGroupRooms(db) - .filter(m => m.serverUrl.includes(domainNameToUse) || m.serverUrl.includes(ipToRemove)) - .map(m => pick(m, ['conversationId', 'serverUrl', 'roomId'])) - ); - - // for those with duplicates, delete the one with the IP as we want to rely on the one with the DNS only now - // remove the ip ones completely which are in double. - // Note: this does also remove the ones not in double, but we are recreating them just below with `saveV2OpenGroupRoom` - db.exec(`DELETE FROM ${OPEN_GROUP_ROOMS_V2_TABLE} WHERE serverUrl LIKE '%${ipToRemove}%';`); - - // for those without duplicates, override the value with the Domain Name - withIpButNotDuplicateRoom.forEach(r => { - const newConvoId = getNewConvoId(r.conversationId); - if (!newConvoId) { - return; - } - console.info( - `withIpButNotDuplicateRoom: renaming room old:${r.conversationId} with saveV2OpenGroupRoom() new- conversationId:${newConvoId}: serverUrl:${urlToUse}` - ); - saveV2OpenGroupRoom({ - ...r, - serverUrl: urlToUse, - conversationId: newConvoId, - }); - }); - - console.info( - '<======== after room update:', - getAllV2OpenGroupRooms(db) - .filter(m => m.serverUrl.includes(domainNameToUse) || m.serverUrl.includes(ipToRemove)) - .map(m => pick(m, ['conversationId', 'serverUrl', 'roomId'])) - ); - - /** - * Then, update the conversations table by doing the same thing - */ - const allSessionV2ConvosIp = compact( - getAllOpenGroupV2Conversations(db).filter(m => m?.id.includes(ipToRemove)) - ); - const allSessionV2ConvosDns = compact( - getAllOpenGroupV2Conversations(db).filter(m => m?.id.includes(domainNameToUse)) - ); - - const duplicatesConvosIpAndDns = allSessionV2ConvosIp.filter(ip => { - const roomId = getRoomIdFromConversationAttributes(ip); - if (!roomId) { - return false; - } - return allSessionV2ConvosDns.some(dns => { - return getRoomIdFromConversationAttributes(dns) === roomId; - }); - }); - const withIpButNotDuplicateConvo = allSessionV2ConvosIp.filter(ip => { - const roomId = getRoomIdFromConversationAttributes(ip); - if (!roomId) { - return false; - } - - return !allSessionV2ConvosDns.some(dns => { - return getRoomIdFromConversationAttributes(dns) === roomId; - }); - }); - console.info('========================================'); - console.info( - 'allSessionV2ConvosIp', - allSessionV2ConvosIp.map(m => m?.id) - ); - console.info( - 'allSessionV2ConvosDns', - allSessionV2ConvosDns.map(m => m?.id) - ); - console.info( - 'duplicatesConvosIpAndDns', - duplicatesConvosIpAndDns.map(m => m?.id) - ); - console.info( - 'withIpButNotDuplicateConvo', - withIpButNotDuplicateConvo.map(m => m?.id) - ); - // for those with duplicates, delete the one with the IP as we want to rely on the one with the DNS only now - // remove the ip ones completely which are in double. - // Note: this does also remove the ones not in double, but we are recreating them just below with `saveConversation` - db.exec(`DELETE FROM ${CONVERSATIONS_TABLE} WHERE id LIKE '%${ipToRemove}%';`); - - // for those without duplicates, override the value with the DNS - const convoIdsToMigrateFromIpToDns: Map = new Map(); - withIpButNotDuplicateConvo.forEach(r => { - if (!r) { - return; - } - const newConvoId = getNewConvoId(r.id); - if (!newConvoId) { - return; - } - console.info( - `withIpButNotDuplicateConvo: renaming convo old:${r.id} with saveConversation() new- conversationId:${newConvoId}` - ); - convoIdsToMigrateFromIpToDns.set(r.id, newConvoId); - saveConversation({ - ...r, - id: newConvoId, - }); - }); - - console.info( - 'after convos update:', - getAllOpenGroupV2Conversations(db) - .filter(m => m?.id.includes(domainNameToUse) || m?.id.includes(ipToRemove)) - .map(m => m?.id) - ); - - /** - * Lastly, we need to take care of messages. - * For duplicated rooms, we drop all the messages from the IP one. (Otherwise we would need to compare each message id to not break the PRIMARY_KEY on the messageID and those are just sogs messages). - * For non duplicated rooms which got renamed to their dns ID, we override the stored conversationId in the message with the new conversationID - */ - dropFtsAndTriggers(db); - - // let's start with the non duplicateD ones, as doing so will make the duplicated one process easier - console.info('convoIdsToMigrateFromIpToDns', [...convoIdsToMigrateFromIpToDns.entries()]); - [...convoIdsToMigrateFromIpToDns.keys()].forEach(oldConvoId => { - const newConvoId = convoIdsToMigrateFromIpToDns.get(oldConvoId); - if (!newConvoId) { - return; - } - console.info(`About to migrate messages of ${oldConvoId} to ${newConvoId}`); - - db.prepare( - `UPDATE ${MESSAGES_TABLE} SET - conversationId = $newConvoId, - json = json_set(json,'$.conversationId', $newConvoId) - WHERE conversationId = $oldConvoId;` - ).run({ oldConvoId, newConvoId }); - }); - // now, the duplicated ones. We just need to move every message with a convoId matching that ip, because we already took care of the one to migrate to the dns before - console.log( - 'Count of messages to be migrated: ', - db - .prepare( - `SELECT COUNT(*) FROM ${MESSAGES_TABLE} WHERE conversationId LIKE '%${ipToRemove}%';` - ) - .get() - ); - - const messageWithIdsToUpdate = db - .prepare( - `SELECT DISTINCT conversationId FROM ${MESSAGES_TABLE} WHERE conversationID LIKE '%${ipToRemove}%'` - ) - .all(); - console.info('messageWithConversationIdsToUpdate', messageWithIdsToUpdate); - messageWithIdsToUpdate.forEach(oldConvo => { - const newConvoId = getNewConvoId(oldConvo.conversationId); - if (!newConvoId) { - return; - } - console.info('oldConvo.conversationId', oldConvo.conversationId, newConvoId); - db.prepare( - `UPDATE ${MESSAGES_TABLE} SET - conversationId = $newConvoId, - json = json_set(json,'$.conversationId', $newConvoId) - WHERE conversationId = $oldConvoId;` - ).run({ oldConvoId: oldConvo.conversationId, newConvoId }); - }); - - rebuildFtsTable(db); - - console.info( - 'removing lastMessageDeletedServerID & lastMessageFetchedServerID from rooms table' - ); - db.exec( - `UPDATE ${OPEN_GROUP_ROOMS_V2_TABLE} SET - json = json_remove(json, '$.lastMessageDeletedServerID', '$.lastMessageFetchedServerID', '$.token' );` - ); - console.info( - 'removing lastMessageDeletedServerID & lastMessageFetchedServerID from rooms table. done' - ); - writeLokiSchemaVersion(targetVersion, db); - console.log('... done'); - })(); - - console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); -} - -// function printTableColumns(table: string, db: BetterSqlite3.Database) { -// console.warn(db.pragma(`table_info('${table}');`)); -// } - -function writeLokiSchemaVersion(newVersion: number, db: BetterSqlite3.Database) { - db.prepare( - `INSERT INTO loki_schema( - version - ) values ( - $newVersion - )` - ).run({ newVersion }); -} - -function updateLokiSchema(db: BetterSqlite3.Database) { - const result = db - .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';`) - .get(); - - if (!result) { - createLokiSchemaTable(db); - } - const lokiSchemaVersion = getLokiSchemaVersion(db); - console.log( - 'updateLokiSchema:', - `Current loki schema version: ${lokiSchemaVersion};`, - `Most recent schema version: ${LOKI_SCHEMA_VERSIONS.length};` - ); - for (let index = 0, max = LOKI_SCHEMA_VERSIONS.length; index < max; index += 1) { - const runSchemaUpdate = LOKI_SCHEMA_VERSIONS[index]; - runSchemaUpdate(lokiSchemaVersion, db); - } + // If the password isn't hex then we need to derive a key from it + const deriveKey = HEX_KEY.test(password); + const value = deriveKey ? `'${password}'` : `"x'${password}'"`; + globalInstance.pragma(`rekey = ${value}`); } -function getLokiSchemaVersion(db: BetterSqlite3.Database) { - const result = db - .prepare( - ` - SELECT MAX(version) as version FROM loki_schema; - ` - ) - .get(); - if (!result || !result.version) { - return 0; +function vacuumDatabase(db: BetterSqlite3.Database) { + if (!db) { + throw new Error('vacuum: db is not initialized'); } - return result.version; + const start = Date.now(); + console.info('Vacuuming DB. This might take a while.'); + db.exec('VACUUM;'); + console.info(`Vacuuming DB Finished in ${Date.now() - start}ms.`); } -function createLokiSchemaTable(db: BetterSqlite3.Database) { - db.transaction(() => { - db.exec(` - CREATE TABLE loki_schema( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - version INTEGER - ); - INSERT INTO loki_schema ( - version - ) values ( - 0 - ); - `); - })(); -} let globalInstance: BetterSqlite3.Database | null = null; function assertGlobalInstance(): BetterSqlite3.Database { @@ -1964,7 +113,6 @@ function assertGlobalInstanceOrInstance( let databaseFilePath: string | undefined; -// tslint:disable-next-line: function-name function _initializePaths(configDir: string) { const dbDir = path.join(configDir, 'sql'); fs.mkdirSync(dbDir, { recursive: true }); @@ -3667,13 +1815,6 @@ function getMessagesCountByConversation( return row ? row['count(*)'] : 0; } -function remove05PrefixFromStringIfNeeded(str: string) { - if (str.length === 66 && str.startsWith('05')) { - return str.substr(2); - } - return str; -} - /** * The returned array is ordered based on the timestamp, the latest is at the end. * @param groupPublicKey string | PubKey @@ -4266,6 +2407,7 @@ export const sqlNode = { getMessagesWithVisualMediaAttachments, getMessagesWithFileAttachments, + getMessagesCountByConversation, getAllEncryptionKeyPairsForGroup, getLatestClosedGroupEncryptionKeyPair, diff --git a/ts/session/apis/open_group_api/opengroupV2/ApiUtil.ts b/ts/session/apis/open_group_api/opengroupV2/ApiUtil.ts index ff8161f0f..e47e2151f 100644 --- a/ts/session/apis/open_group_api/opengroupV2/ApiUtil.ts +++ b/ts/session/apis/open_group_api/opengroupV2/ApiUtil.ts @@ -62,7 +62,7 @@ export function isSessionRunOpenGroup(server: string): boolean { throw new Error('Could not parse URL from serverURL'); } } catch (e) { - // plain ip are not recognized are url, but we want to allow the, + // plain ip are not recognized are url, but we want to allow them serverHost = lowerCased; } diff --git a/ts/session/apis/open_group_api/sogsv3/sogsApiV3.ts b/ts/session/apis/open_group_api/sogsv3/sogsApiV3.ts index 56e7f04bc..2c5bf5c5a 100644 --- a/ts/session/apis/open_group_api/sogsv3/sogsApiV3.ts +++ b/ts/session/apis/open_group_api/sogsv3/sogsApiV3.ts @@ -455,6 +455,11 @@ export const handleBatchPollResults = async ( await handleCapabilities(subrequestOptionsLookup, batchPollResults, serverUrl); if (batchPollResults && isArray(batchPollResults.body)) { + /** + * We run those calls sequentially rather than with a Promise.all call because if we were running those in parallel + * one call might overwrite the changes to the DB of the other one, + * Doing those sequentially makes sure that the cache got from the second call is up to date, before writing it. + */ for (let index = 0; index < batchPollResults.body.length; index++) { const subResponse = batchPollResults.body[index] as any; // using subreqOptions as request type lookup,