From a7d44d334455a4f9977ebcadfd36e77ffa5db6e8 Mon Sep 17 00:00:00 2001 From: Scott Nonnenberg Date: Fri, 20 Apr 2018 14:55:33 -0700 Subject: [PATCH] Backup and end-to-end test! --- js/modules/backup.js | 145 ++++++++++++++-- js/modules/types/message.js | 37 +++- package.json | 1 + preload.js | 14 ++ test/backup_test.js | 269 +++++++++++++++++++++++++++++ test/modules/types/message_test.js | 37 ++++ 6 files changed, 476 insertions(+), 27 deletions(-) diff --git a/js/modules/backup.js b/js/modules/backup.js index 92069abcc..aba881d99 100644 --- a/js/modules/backup.js +++ b/js/modules/backup.js @@ -468,6 +468,7 @@ async function readAttachment(dir, attachment, name, options) { const data = await readFileAsArrayBuffer(targetPath); const isEncrypted = !_.isUndefined(key); + if (isEncrypted) { attachment.data = await crypto.decryptSymmetric(key, data); } else { @@ -475,7 +476,7 @@ async function readAttachment(dir, attachment, name, options) { } } -async function writeAttachment(attachment, options) { +async function writeThumbnail(attachment, options) { const { dir, message, @@ -483,28 +484,77 @@ async function writeAttachment(attachment, options) { key, newKey, } = options; - const filename = _getAnonymousAttachmentFileName(message, index); + const filename = `${_getAnonymousAttachmentFileName(message, index)}-thumbnail`; const target = path.join(dir, filename); - if (fs.existsSync(target)) { - if (newKey) { - console.log(`Deleting attachment ${filename}; key has changed`); - fs.unlinkSync(target); - } else { - console.log(`Skipping attachment ${filename}; already exists`); - return; + const { thumbnail } = attachment; + + if (!thumbnail || !thumbnail.data) { + return; + } + + await writeEncryptedAttachment(target, thumbnail.data, { + key, + newKey, + filename, + dir, + }); +} + +async function writeThumbnails(rawQuotedAttachments, options) { + const { name } = options; + + const { loadAttachmentData } = Signal.Migrations; + const promises = rawQuotedAttachments.map(async (attachment) => { + if (!attachment || !attachment.thumbnail || !attachment.thumbnail.path) { + return attachment; } + + return Object.assign( + {}, + attachment, + { thumbnail: await loadAttachmentData(attachment.thumbnail) } + ); + }); + + const attachments = await Promise.all(promises); + try { + await Promise.all(_.map( + attachments, + (attachment, index) => writeThumbnail(attachment, Object.assign({}, options, { + index, + })) + )); + } catch (error) { + console.log( + 'writeThumbnails: error exporting conversation', + name, + ':', + error && error.stack ? error.stack : error + ); + throw error; } +} +async function writeAttachment(attachment, options) { + const { + dir, + message, + index, + key, + newKey, + } = options; + const filename = _getAnonymousAttachmentFileName(message, index); + const target = path.join(dir, filename); if (!Attachment.hasData(attachment)) { throw new TypeError("'attachment.data' is required"); } - const ciphertext = await crypto.encryptSymmetric(key, attachment.data); - - const writer = await createFileAndWriter(dir, filename); - const stream = createOutputStream(writer); - stream.write(Buffer.from(ciphertext)); - await stream.close(); + await writeEncryptedAttachment(target, attachment.data, { + key, + newKey, + filename, + dir, + }); } async function writeAttachments(rawAttachments, options) { @@ -531,6 +581,32 @@ async function writeAttachments(rawAttachments, options) { } } +async function writeEncryptedAttachment(target, data, options = {}) { + const { + key, + newKey, + filename, + dir, + } = options; + + if (fs.existsSync(target)) { + if (newKey) { + console.log(`Deleting attachment ${filename}; key has changed`); + fs.unlinkSync(target); + } else { + console.log(`Skipping attachment ${filename}; already exists`); + return; + } + } + + const ciphertext = await crypto.encryptSymmetric(key, data); + + const writer = await createFileAndWriter(dir, filename); + const stream = createOutputStream(writer); + stream.write(Buffer.from(ciphertext)); + await stream.close(); +} + function _sanitizeFileName(filename) { return filename.toString().replace(/[^a-z0-9.,+()'#\- ]/gi, '_'); } @@ -542,6 +618,7 @@ async function exportConversation(db, conversation, options) { dir, attachmentsDir, key, + newKey, } = options; if (!name) { throw new Error('Need a name!'); @@ -610,6 +687,7 @@ async function exportConversation(db, conversation, options) { } // eliminate attachment data from the JSON, since it will go to disk + // Note: this is for legacy messages only, which stored attachment data in the db message.attachments = _.map( attachments, attachment => _.omit(attachment, ['data']) @@ -629,18 +707,34 @@ async function exportConversation(db, conversation, options) { const jsonString = JSON.stringify(stringify(message)); stream.write(jsonString); + console.log({ backupMessage: message }); if (attachments && attachments.length > 0) { const exportAttachments = () => writeAttachments(attachments, { dir: attachmentsDir, name, message, key, + newKey, }); // eslint-disable-next-line more/no-then promiseChain = promiseChain.then(exportAttachments); } + const quoteThumbnails = message.quote && message.quote.attachments; + if (quoteThumbnails && quoteThumbnails.length > 0) { + const exportQuoteThumbnails = () => writeThumbnails(quoteThumbnails, { + dir: attachmentsDir, + name, + message, + key, + newKey, + }); + + // eslint-disable-next-line more/no-then + promiseChain = promiseChain.then(exportQuoteThumbnails); + } + count += 1; cursor.continue(); } else { @@ -701,6 +795,7 @@ function exportConversations(db, options) { messagesDir, attachmentsDir, key, + newKey, } = options; if (!messagesDir) { @@ -747,6 +842,7 @@ function exportConversations(db, options) { dir, attachmentsDir, key, + newKey, }); }; @@ -808,11 +904,23 @@ function loadAttachments(dir, getName, options) { options = options || {}; const { message } = options; - const promises = _.map(message.attachments, (attachment, index) => { + const attachmentPromises = _.map(message.attachments, (attachment, index) => { const name = getName(message, index, attachment); return readAttachment(dir, attachment, name, options); }); - return Promise.all(promises); + + const quoteAttachments = message.quote && message.quote.attachments; + const thumbnailPromises = _.map(quoteAttachments, (attachment, index) => { + const thumbnail = attachment && attachment.thumbnail; + if (!thumbnail) { + return null; + } + + const name = `${getName(message, index, thumbnail)}-thumbnail`; + return readAttachment(dir, thumbnail, name, options); + }); + + return Promise.all(attachmentPromises.concat(thumbnailPromises)); } function saveMessage(db, message) { @@ -922,7 +1030,8 @@ async function importConversation(db, dir, options) { return false; } - if (message.attachments && message.attachments.length) { + if ((message.attachments && message.attachments.length) || + (message.quote && message.quote.attachments && message.quote.attachments.length)) { const importMessage = async () => { const getName = attachmentsDir ? _getAnonymousAttachmentFileName diff --git a/js/modules/types/message.js b/js/modules/types/message.js index ec84e7299..f9b36a27a 100644 --- a/js/modules/types/message.js +++ b/js/modules/types/message.js @@ -243,9 +243,12 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => { const message = exports.initializeSchemaVersion(rawMessage); - const { attachments } = message; - const hasAttachments = attachments && attachments.length > 0; - if (!hasAttachments) { + const { attachments, quote } = message; + const hasFilesToWrite = + (quote && quote.attachments && quote.attachments.length > 0) || + (attachments && attachments.length > 0); + + if (!hasFilesToWrite) { return message; } @@ -256,7 +259,7 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => { return message; } - attachments.forEach((attachment) => { + (attachments || []).forEach((attachment) => { if (!Attachment.hasData(attachment)) { throw new TypeError("'attachment.data' is required during message import"); } @@ -266,13 +269,29 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => { } }); - const messageWithoutAttachmentData = Object.assign({}, message, { - attachments: await Promise.all(attachments.map(async (attachment) => { - await writeExistingAttachmentData(attachment); - return omit(attachment, ['data']); - })), + const writeThumbnails = exports._mapQuotedAttachments(async (thumbnail) => { + const { data, path } = thumbnail; + + // we want to be bulletproof to thumbnails without data + if (!data || !path) { + return thumbnail; + } + + await writeExistingAttachmentData(thumbnail); + return omit(thumbnail, ['data']); }); + const messageWithoutAttachmentData = Object.assign( + {}, + await writeThumbnails(message), + { + attachments: await Promise.all((attachments || []).map(async (attachment) => { + await writeExistingAttachmentData(attachment); + return omit(attachment, ['data']); + })), + } + ); + return messageWithoutAttachmentData; }; }; diff --git a/package.json b/package.json index a5c01ecc1..7c99afda6 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "eslint-plugin-mocha": "^4.12.1", "eslint-plugin-more": "^0.3.1", "extract-zip": "^1.6.6", + "glob": "^7.1.2", "grunt": "^1.0.1", "grunt-cli": "^1.2.0", "grunt-contrib-concat": "^1.0.1", diff --git a/preload.js b/preload.js index bdce373ec..e5a766d7a 100644 --- a/preload.js +++ b/preload.js @@ -205,3 +205,17 @@ window.Signal.Workflow.MessageDataMigrator = // We pull this in last, because the native module involved appears to be sensitive to // /tmp mounted as noexec on Linux. require('./js/spell_check'); + +if (window.config.environment === 'test') { + /* eslint-disable global-require, import/no-extraneous-dependencies */ + window.test = { + fs: require('fs'), + glob: require('glob'), + fse: require('fs-extra'), + tmp: require('tmp'), + path: require('path'), + basePath: __dirname, + attachmentsPath, + }; + /* eslint-enable global-require, import/no-extraneous-dependencies */ +} diff --git a/test/backup_test.js b/test/backup_test.js index 6019a85ac..36939ce9d 100644 --- a/test/backup_test.js +++ b/test/backup_test.js @@ -1,5 +1,8 @@ /* global Signal: false */ +/* global Whisper: false */ /* global assert: false */ +/* global textsecure: false */ +/* global _: false */ 'use strict'; @@ -223,4 +226,270 @@ describe('Backup', () => { ); }); }); + + describe('end-to-end', () => { + it('exports then imports to produce the same data we started with', async () => { + const { + attachmentsPath, + fs, + fse, + glob, + path, + tmp, + } = window.test; + const { + upgradeMessageSchema, + loadAttachmentData, + } = window.Signal.Migrations; + + const key = new Uint8Array([ + 1, 3, 4, 5, 6, 7, 8, 11, + 23, 34, 1, 34, 3, 5, 45, 45, + 1, 3, 4, 5, 6, 7, 8, 11, + 23, 34, 1, 34, 3, 5, 45, 45, + ]); + const attachmentsPattern = path.join(attachmentsPath, '**'); + + const OUR_NUMBER = '+12025550000'; + const CONTACT_ONE_NUMBER = '+12025550001'; + + async function wrappedLoadAttachment(attachment) { + return _.omit(await loadAttachmentData(attachment), ['path']); + } + + async function clearAllData() { + await textsecure.storage.protocol.removeAllData(); + await fse.emptyDir(attachmentsPath); + } + + function removeId(model) { + return _.omit(model, ['id']); + } + + const twoSlashes = /.*\/.*\/.*/; + function removeDirs(dirs) { + return _.filter(dirs, (fullDir) => { + const dir = fullDir.replace(attachmentsPath, ''); + return twoSlashes.test(dir); + }); + } + + function _mapQuotedAttachments(mapper) { + return async (message, context) => { + if (!message.quote) { + return message; + } + + const wrappedMapper = async (attachment) => { + if (!attachment || !attachment.thumbnail) { + return attachment; + } + + return Object.assign({}, attachment, { + thumbnail: await mapper(attachment.thumbnail, context), + }); + }; + + const quotedAttachments = (message.quote && message.quote.attachments) || []; + + return Object.assign({}, message, { + quote: Object.assign({}, message.quote, { + attachments: await Promise.all(quotedAttachments.map(wrappedMapper)), + }), + }); + }; + } + + async function loadAllFilesFromDisk(message) { + const loadThumbnails = _mapQuotedAttachments((thumbnail) => { + // we want to be bulletproof to thumbnails without data + if (!thumbnail.path) { + return thumbnail; + } + + return wrappedLoadAttachment(thumbnail); + }); + + const promises = (message.attachments || []).map(attachment => + wrappedLoadAttachment(attachment)); + + return Object.assign( + {}, + await loadThumbnails(message), + { + attachments: await Promise.all(promises), + } + ); + } + + let backupDir; + try { + const ATTACHMENT_COUNT = 2; + const MESSAGE_COUNT = 1; + const CONVERSATION_COUNT = 1; + + const messageWithAttachments = { + conversationId: CONTACT_ONE_NUMBER, + body: 'Totally!', + source: OUR_NUMBER, + received_at: 1524185933350, + timestamp: 1524185933350, + errors: [], + attachments: [{ + contentType: 'image/gif', + fileName: 'sad_cat.gif', + data: new Uint8Array([ + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + ]).buffer, + }], + quote: { + text: "Isn't it cute?", + author: CONTACT_ONE_NUMBER, + id: 12345678, + attachments: [{ + contentType: 'audio/mp3', + fileName: 'song.mp3', + }, { + contentType: 'image/gif', + fileName: 'happy_cat.gif', + thumbnail: { + contentType: 'image/png', + data: new Uint8Array([ + 2, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + ]).buffer, + }, + }], + }, + }; + + console.log('Backup test: Clear all data'); + await clearAllData(); + + console.log('Backup test: Create models, save to db/disk'); + const message = await upgradeMessageSchema(messageWithAttachments); + console.log({ message }); + const messageModel = new Whisper.Message(message); + await window.wrapDeferred(messageModel.save()); + + const conversation = { + active_at: 1524185933350, + color: 'orange', + expireTimer: 0, + id: CONTACT_ONE_NUMBER, + lastMessage: 'Heyo!', + name: 'Someone Somewhere', + profileAvatar: { + contentType: 'image/jpeg', + data: new Uint8Array([ + 3, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + ]).buffer, + size: 64, + }, + profileKey: new Uint8Array([ + 4, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8, + ]).buffer, + profileName: 'Someone! 🤔', + profileSharing: true, + timestamp: 1524185933350, + tokens: [ + 'someone somewhere', + 'someone', + 'somewhere', + '2025550001', + '12025550001', + ], + type: 'private', + unreadCount: 0, + verified: 0, + }; + console.log({ conversation }); + const conversationModel = new Whisper.Conversation(conversation); + await window.wrapDeferred(conversationModel.save()); + + console.log('Backup test: Ensure that all attachments were saved to disk'); + const attachmentFiles = removeDirs(glob.sync(attachmentsPattern)); + console.log({ attachmentFiles }); + assert.strictEqual(ATTACHMENT_COUNT, attachmentFiles.length); + + console.log('Backup test: Export!'); + backupDir = tmp.dirSync().name; + console.log({ backupDir }); + await Signal.Backup.exportToDirectory(backupDir, { key }); + + console.log('Backup test: Ensure that messages.zip exists'); + const zipPath = path.join(backupDir, 'messages.zip'); + const messageZipExists = fs.existsSync(zipPath); + assert.strictEqual(true, messageZipExists); + + console.log('Backup test: Ensure that all attachments made it to backup dir'); + const backupAttachmentPattern = path.join(backupDir, 'attachments/*'); + const backupAttachments = glob.sync(backupAttachmentPattern); + console.log({ backupAttachments }); + assert.strictEqual(ATTACHMENT_COUNT, backupAttachments.length); + + console.log('Backup test: Clear all data'); + await clearAllData(); + + console.log('Backup test: Import!'); + await Signal.Backup.importFromDirectory(backupDir, { key }); + + console.log('Backup test: ensure that all attachments were imported'); + const recreatedAttachmentFiles = removeDirs(glob.sync(attachmentsPattern)); + console.log({ recreatedAttachmentFiles }); + assert.strictEqual(ATTACHMENT_COUNT, recreatedAttachmentFiles.length); + assert.deepEqual(attachmentFiles, recreatedAttachmentFiles); + + console.log('Backup test: Check messages'); + const messageCollection = new Whisper.MessageCollection(); + await window.wrapDeferred(messageCollection.fetch()); + assert.strictEqual(messageCollection.length, MESSAGE_COUNT); + const messageFromDB = removeId(messageCollection.at(0).attributes); + console.log({ messageFromDB, message }); + assert.deepEqual(messageFromDB, message); + + console.log('Backup test: check that all attachments were successfully imported'); + const messageWithAttachmentsFromDB = await loadAllFilesFromDisk(messageFromDB); + console.log({ messageWithAttachmentsFromDB, messageWithAttachments }); + assert.deepEqual( + _.omit(messageWithAttachmentsFromDB, ['schemaVersion']), + messageWithAttachments + ); + + console.log('Backup test: check conversations'); + const conversationCollection = new Whisper.ConversationCollection(); + await window.wrapDeferred(conversationCollection.fetch()); + assert.strictEqual(conversationCollection.length, CONVERSATION_COUNT); + + const conversationFromDB = conversationCollection.at(0).attributes; + console.log({ conversationFromDB, conversation }); + assert.deepEqual( + conversationFromDB, + _.omit(conversation, ['profileAvatar']) + ); + + console.log('Backup test: Clear all data'); + await clearAllData(); + + console.log('Backup test: Complete!'); + } finally { + if (backupDir) { + console.log({ backupDir }); + console.log('Deleting', backupDir); + await fse.remove(backupDir); + } + } + }); + }); }); diff --git a/test/modules/types/message_test.js b/test/modules/types/message_test.js index 7a40dfcd9..e884b063e 100644 --- a/test/modules/types/message_test.js +++ b/test/modules/types/message_test.js @@ -67,6 +67,43 @@ describe('Message', () => { await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input); assert.deepEqual(actual, expected); }); + + it('should process quote attachment thumbnails', async () => { + const input = { + body: 'Imagine there is no heaven…', + schemaVersion: 4, + attachments: [], + quote: { + attachments: [{ + thumbnail: { + path: 'ab/abcdefghi', + data: stringToArrayBuffer('It’s easy if you try'), + }, + }], + }, + }; + const expected = { + body: 'Imagine there is no heaven…', + schemaVersion: 4, + attachments: [], + quote: { + attachments: [{ + thumbnail: { + path: 'ab/abcdefghi', + }, + }], + }, + }; + + const writeExistingAttachmentData = (attachment) => { + assert.equal(attachment.path, 'ab/abcdefghi'); + assert.deepEqual(attachment.data, stringToArrayBuffer('It’s easy if you try')); + }; + + const actual = + await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input); + assert.deepEqual(actual, expected); + }); }); describe('initializeSchemaVersion', () => {