remove backup_test, backup and crypto related to backup only

pull/1783/head
Audric Ackermann 4 years ago
parent 1b2a644e7a
commit 85e0c9cf36
No known key found for this signature in database
GPG Key ID: 999F434D76324AD4

File diff suppressed because it is too large Load Diff

@ -5,93 +5,28 @@
module.exports = {
arrayBufferToBase64,
base64ToArrayBuffer,
bytesFromString,
concatenateBytes,
constantTimeEqual,
decryptAesCtr,
decryptAttachment,
decryptFile,
decryptSymmetric,
deriveAccessKey,
encryptAesCtr,
encryptAttachment,
encryptFile,
encryptSymmetric,
fromEncodedBinaryToArrayBuffer,
getRandomBytes,
getViewOfArrayBuffer,
getZeroes,
highBitsToInt,
hmacSha256,
intsToByteHighAndLow,
splitBytes,
stringFromBytes,
trimBytes,
};
function arrayBufferToBase64(arrayBuffer) {
return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('base64');
}
function base64ToArrayBuffer(base64string) {
return dcodeIO.ByteBuffer.wrap(base64string, 'base64').toArrayBuffer();
}
function fromEncodedBinaryToArrayBuffer(key) {
return dcodeIO.ByteBuffer.wrap(key, 'binary').toArrayBuffer();
}
function bytesFromString(string) {
return dcodeIO.ByteBuffer.wrap(string, 'utf8').toArrayBuffer();
}
function stringFromBytes(buffer) {
return dcodeIO.ByteBuffer.wrap(buffer).toString('utf8');
}
// High-level Operations
// Path structure: 'fa/facdf99c22945b1c9393345599a276f4b36ad7ccdc8c2467f5441b742c2d11fa'
function getAttachmentLabel(path) {
const filename = path.slice(3);
return base64ToArrayBuffer(filename);
}
const PUB_KEY_LENGTH = 32;
async function encryptAttachment(staticPublicKey, path, plaintext) {
const uniqueId = getAttachmentLabel(path);
return encryptFile(staticPublicKey, uniqueId, plaintext);
}
async function decryptAttachment(staticPrivateKey, path, data) {
const uniqueId = getAttachmentLabel(path);
return decryptFile(staticPrivateKey, uniqueId, data);
}
async function encryptFile(staticPublicKey, uniqueId, plaintext) {
const ephemeralKeyPair = await libsignal.KeyHelper.generateIdentityKeyPair();
const agreement = await libsignal.Curve.async.calculateAgreement(
staticPublicKey,
ephemeralKeyPair.privKey
);
const key = await hmacSha256(agreement, uniqueId);
const prefix = ephemeralKeyPair.pubKey.slice(1);
return concatenateBytes(prefix, await encryptSymmetric(key, plaintext));
}
async function decryptFile(staticPrivateKey, uniqueId, data) {
const ephemeralPublicKey = _getFirstBytes(data, PUB_KEY_LENGTH);
const ciphertext = _getBytes(data, PUB_KEY_LENGTH, data.byteLength);
const agreement = await libsignal.Curve.async.calculateAgreement(
ephemeralPublicKey,
staticPrivateKey
);
const key = await hmacSha256(agreement, uniqueId);
return decryptSymmetric(key, ciphertext);
}
async function deriveAccessKey(profileKey) {
const iv = getZeroes(12);
const plaintext = getZeroes(16);
@ -206,19 +141,6 @@ async function encryptAesCtr(key, plaintext, counter) {
return ciphertext;
}
async function decryptAesCtr(key, ciphertext, counter) {
const extractable = false;
const algorithm = {
name: 'AES-CTR',
counter: new Uint8Array(counter),
length: 128,
};
const cryptoKey = await crypto.subtle.importKey('raw', key, algorithm, extractable, ['decrypt']);
const plaintext = await crypto.subtle.decrypt(algorithm, cryptoKey, ciphertext);
return plaintext;
}
async function _encrypt_aes_gcm(key, iv, plaintext) {
const algorithm = {
name: 'AES-GCM',
@ -249,24 +171,6 @@ function getZeroes(n) {
return result;
}
function highBitsToInt(byte) {
return (byte & 0xff) >> 4;
}
function intsToByteHighAndLow(highValue, lowValue) {
return ((highValue << 4) | lowValue) & 0xff;
}
function trimBytes(buffer, length) {
return _getFirstBytes(buffer, length);
}
function getViewOfArrayBuffer(buffer, start, finish) {
const source = new Uint8Array(buffer);
const result = source.slice(start, finish);
return result.buffer;
}
function concatenateBytes(...elements) {
const length = elements.reduce((total, element) => total + element.byteLength, 0);
@ -285,32 +189,6 @@ function concatenateBytes(...elements) {
return result.buffer;
}
function splitBytes(buffer, ...lengths) {
const total = lengths.reduce((acc, length) => acc + length, 0);
if (total !== buffer.byteLength) {
throw new Error(
`Requested lengths total ${total} does not match source total ${buffer.byteLength}`
);
}
const source = new Uint8Array(buffer);
const results = [];
let position = 0;
for (let i = 0, max = lengths.length; i < max; i += 1) {
const length = lengths[i];
const result = new Uint8Array(length);
const section = source.slice(position, position + length);
result.set(section);
position += result.byteLength;
results.push(result);
}
return results;
}
// Internal-only
function _getFirstBytes(data, n) {

@ -345,8 +345,6 @@ window.Signal.Data = require('./ts/data/data');
window.getMessageController = () => window.libsession.Messages.getMessageController();
// Pulling these in separately since they access filesystem, electron
window.Signal.Backup = require('./js/modules/backup');
window.Signal.Logs = require('./js/modules/logs');
window.addEventListener('contextmenu', e => {

@ -1,524 +0,0 @@
/* global Signal, assert, _, libsignal */
/* eslint-disable no-console */
'use strict';
describe('Backup', () => {
describe('_sanitizeFileName', () => {
it('leaves a basic string alone', () => {
const initial = "Hello, how are you #5 ('fine' + great).jpg";
const expected = initial;
assert.strictEqual(Signal.Backup._sanitizeFileName(initial), expected);
});
it('replaces all unknown characters', () => {
const initial = '!@$%^&*=';
const expected = '________';
assert.strictEqual(Signal.Backup._sanitizeFileName(initial), expected);
});
});
describe('_trimFileName', () => {
it('handles a file with no extension', () => {
const initial = '0123456789012345678901234567890123456789';
const expected = '012345678901234567890123456789';
assert.strictEqual(Signal.Backup._trimFileName(initial), expected);
});
it('handles a file with a long extension', () => {
const initial = '0123456789012345678901234567890123456789.01234567890123456789';
const expected = '012345678901234567890123456789';
assert.strictEqual(Signal.Backup._trimFileName(initial), expected);
});
it('handles a file with a normal extension', () => {
const initial = '01234567890123456789012345678901234567890123456789.jpg';
const expected = '012345678901234567890123.jpg';
assert.strictEqual(Signal.Backup._trimFileName(initial), expected);
});
});
describe('_getExportAttachmentFileName', () => {
it('uses original filename if attachment has one', () => {
const message = {
body: 'something',
};
const index = 0;
const attachment = {
fileName: 'blah.jpg',
};
const expected = 'blah.jpg';
const actual = Signal.Backup._getExportAttachmentFileName(message, index, attachment);
assert.strictEqual(actual, expected);
});
it('uses attachment id if no filename', () => {
const message = {
body: 'something',
};
const index = 0;
const attachment = {
id: '123',
};
const expected = '123';
const actual = Signal.Backup._getExportAttachmentFileName(message, index, attachment);
assert.strictEqual(actual, expected);
});
it('uses filename and contentType if available', () => {
const message = {
body: 'something',
};
const index = 0;
const attachment = {
id: '123',
contentType: 'image/jpeg',
};
const expected = '123.jpeg';
const actual = Signal.Backup._getExportAttachmentFileName(message, index, attachment);
assert.strictEqual(actual, expected);
});
it('handles strange contentType', () => {
const message = {
body: 'something',
};
const index = 0;
const attachment = {
id: '123',
contentType: 'something',
};
const expected = '123.something';
const actual = Signal.Backup._getExportAttachmentFileName(message, index, attachment);
assert.strictEqual(actual, expected);
});
});
describe('_getAnonymousAttachmentFileName', () => {
it('uses message id', () => {
const message = {
id: 'id-45',
body: 'something',
};
const index = 0;
const attachment = {
fileName: 'blah.jpg',
};
const expected = 'id-45';
const actual = Signal.Backup._getAnonymousAttachmentFileName(message, index, attachment);
assert.strictEqual(actual, expected);
});
it('appends index if it is above zero', () => {
const message = {
id: 'id-45',
body: 'something',
};
const index = 1;
const attachment = {
fileName: 'blah.jpg',
};
const expected = 'id-45-1';
const actual = Signal.Backup._getAnonymousAttachmentFileName(message, index, attachment);
assert.strictEqual(actual, expected);
});
});
describe('_getConversationDirName', () => {
it('uses name if available', () => {
const conversation = {
active_at: 123,
name: '0123456789012345678901234567890123456789',
id: 'id',
};
const expected = '123 (012345678901234567890123456789 id)';
assert.strictEqual(Signal.Backup._getConversationDirName(conversation), expected);
});
it('uses just id if name is not available', () => {
const conversation = {
active_at: 123,
id: 'id',
};
const expected = '123 (id)';
assert.strictEqual(Signal.Backup._getConversationDirName(conversation), expected);
});
it('uses inactive for missing active_at', () => {
const conversation = {
name: 'name',
id: 'id',
};
const expected = 'inactive (name id)';
assert.strictEqual(Signal.Backup._getConversationDirName(conversation), expected);
});
});
describe('_getConversationLoggingName', () => {
it('uses plain id if conversation is private', () => {
const conversation = {
active_at: 123,
id: 'id',
type: 'private',
};
const expected = '123 (id)';
assert.strictEqual(Signal.Backup._getConversationLoggingName(conversation), expected);
});
it('uses just id if name is not available', () => {
const conversation = {
active_at: 123,
id: 'groupId',
type: 'group',
};
const expected = '123 ([REDACTED_GROUP]pId)';
assert.strictEqual(Signal.Backup._getConversationLoggingName(conversation), expected);
});
it('uses inactive for missing active_at', () => {
const conversation = {
id: 'id',
type: 'private',
};
const expected = 'inactive (id)';
assert.strictEqual(Signal.Backup._getConversationLoggingName(conversation), expected);
});
});
describe('end-to-end', () => {
it('exports then imports to produce the same data we started with', async function thisNeeded() {
this.timeout(6000);
const { attachmentsPath, fse, glob, path, tmp, isWindows } = window.test;
// Skip this test on windows
// because it always fails due to lstat permission error.
// Don't know how to fix it so this is a temp work around.
if (isWindows || !isWindows) {
console.log('Skipping exports then imports to produce the same data we started');
this.skip();
return;
}
const { upgradeMessageSchema, loadAttachmentData } = window.Signal.Migrations;
const staticKeyPair = await libsignal.KeyHelper.generateIdentityKeyPair();
const attachmentsPattern = path.join(attachmentsPath, '**');
const OUR_NUMBER = '+12025550000';
const CONTACT_ONE_NUMBER = '+12025550001';
const toArrayBuffer = nodeBuffer =>
nodeBuffer.buffer.slice(
nodeBuffer.byteOffset,
nodeBuffer.byteOffset + nodeBuffer.byteLength
);
const getFixture = target => toArrayBuffer(fse.readFileSync(target));
const FIXTURES = {
gif: getFixture('fixtures/giphy-7GFfijngKbeNy.gif'),
mp4: getFixture('fixtures/pixabay-Soap-Bubble-7141.mp4'),
jpg: getFixture('fixtures/koushik-chowdavarapu-105425-unsplash.jpg'),
mp3: getFixture('fixtures/incompetech-com-Agnus-Dei-X.mp3'),
txt: getFixture('fixtures/lorem-ipsum.txt'),
png: getFixture('fixtures/freepngs-2cd43b_bed7d1327e88454487397574d87b64dc_mv2.png'),
};
async function wrappedLoadAttachment(attachment) {
return _.omit(await loadAttachmentData(attachment), ['path']);
}
async function clearAllData() {
await window.Signal.Data.removeAll();
window.storage.reset();
await window.storage.fetch();
window.getConversationController().reset();
window.BlockedNumberController.reset();
await window.getConversationController().load();
await window.BlockedNumberController.load();
await fse.emptyDir(attachmentsPath);
}
function removeId(model) {
return _.omit(model, ['id']);
}
const getUndefinedKeys = object =>
Object.entries(object)
.filter(([, value]) => value === undefined)
.map(([name]) => name);
const omitUndefinedKeys = object => _.omit(object, getUndefinedKeys(object));
// We want to know which paths have two slashes, since that tells us which files
// in the attachment fan-out are files vs. directories.
const TWO_SLASHES = /[^/]*\/[^/]*\/[^/]*/;
// On windows, attachmentsPath has a normal windows path format (\ separators), but
// glob returns only /. We normalize to / separators for our manipulations.
const normalizedBase = attachmentsPath.replace(/\\/g, '/');
function removeDirs(dirs) {
return _.filter(dirs, fullDir => {
const dir = fullDir.replace(normalizedBase, '');
return TWO_SLASHES.test(dir);
});
}
function _mapQuotedAttachments(mapper) {
return async (message, context) => {
if (!message.quote) {
return message;
}
const wrappedMapper = async attachment => {
if (!attachment || !attachment.thumbnail) {
return attachment;
}
return Object.assign({}, attachment, {
thumbnail: await mapper(attachment.thumbnail, context),
});
};
const quotedAttachments = (message.quote && message.quote.attachments) || [];
return Object.assign({}, message, {
quote: Object.assign({}, message.quote, {
attachments: await Promise.all(quotedAttachments.map(wrappedMapper)),
}),
});
};
}
async function loadAllFilesFromDisk(message) {
const loadThumbnails = _mapQuotedAttachments(thumbnail => {
// we want to be bulletproof to thumbnails without data
if (!thumbnail.path) {
return thumbnail;
}
return wrappedLoadAttachment(thumbnail);
});
return Object.assign({}, await loadThumbnails(message), {
attachments: await Promise.all(
(message.attachments || []).map(async attachment => {
await wrappedLoadAttachment(attachment);
if (attachment.thumbnail) {
await wrappedLoadAttachment(attachment.thumbnail);
}
if (attachment.screenshot) {
await wrappedLoadAttachment(attachment.screenshot);
}
return attachment;
})
),
preview: await Promise.all(
(message.preview || []).map(async item => {
if (item.image) {
await wrappedLoadAttachment(item.image);
}
return item;
})
),
});
}
let backupDir;
try {
// Seven total:
// - Five from image/video attachments
// - One from embedded contact avatar
// - One from embedded quoted attachment thumbnail
// - One from a link preview image
const ATTACHMENT_COUNT = 8;
const MESSAGE_COUNT = 1;
const CONVERSATION_COUNT = 1;
const messageWithAttachments = {
conversationId: CONTACT_ONE_NUMBER,
body: 'Totally!',
source: OUR_NUMBER,
received_at: 1524185933350,
timestamp: 1524185933350,
errors: [],
attachments: [
// Note: generates two more files: screenshot and thumbnail
{
contentType: 'video/mp4',
fileName: 'video.mp4',
data: FIXTURES.mp4,
},
// Note: generates one more file: thumbnail
{
contentType: 'image/png',
fileName: 'landscape.png',
data: FIXTURES.png,
},
],
hasAttachments: 1,
hasVisualMediaAttachments: 1,
quote: {
text: "Isn't it cute?",
author: CONTACT_ONE_NUMBER,
id: 12345678,
attachments: [
{
contentType: 'audio/mp3',
fileName: 'song.mp3',
},
{
contentType: 'image/gif',
fileName: 'avatar.gif',
thumbnail: {
contentType: 'image/png',
data: FIXTURES.gif,
},
},
],
},
preview: [
{
url: 'https://www.instagram.com/p/BsOGulcndj-/',
title:
'EGG GANG 🌍 on Instagram: “Lets set a world record together and get the most liked post on Instagram. Beating the current world record held by Kylie Jenner (18…”',
image: {
contentType: 'image/jpeg',
data: FIXTURES.jpg,
},
},
],
};
console.log('Backup test: Clear all data');
await clearAllData();
console.log('Backup test: Create models, save to db/disk');
const message = await upgradeMessageSchema(messageWithAttachments);
await window.Signal.Data.saveMessage(message);
const conversation = {
active_at: 1524185933350,
color: 'orange',
expireTimer: 0,
id: CONTACT_ONE_NUMBER,
name: 'Someone Somewhere',
profileAvatar: {
contentType: 'image/jpeg',
data: FIXTURES.jpeg,
size: 64,
},
profileKey: 'BASE64KEY',
profileName: 'Someone! 🤔',
timestamp: 1524185933350,
type: 'private',
unreadCount: 0,
version: 2,
};
console.log({ conversation });
await window.Signal.Data.saveConversation(conversation);
console.log('Backup test: Ensure that all attachments were saved to disk');
const attachmentFiles = removeDirs(glob.sync(attachmentsPattern));
console.log({ attachmentFiles });
assert.strictEqual(ATTACHMENT_COUNT, attachmentFiles.length);
console.log('Backup test: Export!');
backupDir = tmp.dirSync().name;
console.log({ backupDir });
await Signal.Backup.exportToDirectory(backupDir, {
key: staticKeyPair.pubKey,
});
console.log('Backup test: Ensure that messages.tar.gz exists');
const archivePath = path.join(backupDir, 'messages.tar.gz');
const messageZipExists = fse.existsSync(archivePath);
assert.strictEqual(true, messageZipExists);
console.log('Backup test: Ensure that all attachments made it to backup dir');
const backupAttachmentPattern = path.join(backupDir, 'attachments/*');
const backupAttachments = glob.sync(backupAttachmentPattern);
console.log({ backupAttachments });
assert.strictEqual(ATTACHMENT_COUNT, backupAttachments.length);
console.log('Backup test: Clear all data');
await clearAllData();
console.log('Backup test: Import!');
await Signal.Backup.importFromDirectory(backupDir, {
key: staticKeyPair.privKey,
});
console.log('Backup test: Check conversations');
const conversationCollection = await window.Signal.Data.getAllConversations();
assert.strictEqual(conversationCollection.length, CONVERSATION_COUNT);
// We need to ommit any custom fields we have added
const ommited = [
'profileAvatar',
'swarmNodes',
'groupAdmins',
'isKickedFromGroup',
'unlockTimestamp',
'sessionResetStatus',
'isOnline',
];
const conversationFromDB = conversationCollection.at(0).attributes;
console.log({ conversationFromDB, conversation });
assert.deepEqual(_.omit(conversationFromDB, ommited), _.omit(conversation, ommited));
console.log('Backup test: Check messages');
const allMessages = await window.Signal.Data.getAllMessages();
assert.strictEqual(allMessages.length, MESSAGE_COUNT);
const messageFromDB = removeId(allMessages.at(0).attributes);
const expectedMessage = messageFromDB;
console.log({ messageFromDB, expectedMessage });
assert.deepEqual(messageFromDB, expectedMessage);
console.log('Backup test: ensure that all attachments were imported');
const recreatedAttachmentFiles = removeDirs(glob.sync(attachmentsPattern));
console.log({ recreatedAttachmentFiles });
assert.strictEqual(ATTACHMENT_COUNT, recreatedAttachmentFiles.length);
assert.deepEqual(attachmentFiles, recreatedAttachmentFiles);
console.log('Backup test: Check that all attachments were successfully imported');
const messageWithAttachmentsFromDB = await loadAllFilesFromDisk(messageFromDB);
const expectedMessageWithAttachments = await loadAllFilesFromDisk(
omitUndefinedKeys(message)
);
console.log({
messageWithAttachmentsFromDB,
expectedMessageWithAttachments,
});
assert.deepEqual(
_.omit(messageWithAttachmentsFromDB, ['sent']),
expectedMessageWithAttachments
);
console.log('Backup test: Clear all data');
await clearAllData();
console.log('Backup test: Complete!');
} finally {
if (backupDir) {
console.log({ backupDir });
console.log('Deleting', backupDir);
await fse.remove(backupDir);
}
}
});
});
});

@ -52,7 +52,6 @@
<script type="text/javascript" src="models/conversations_test.js"></script>
<script type="text/javascript" src="models/messages_test.js"></script>
<script type="text/javascript" src="backup_test.js"></script>
<script type="text/javascript" src="database_test.js"></script>
<script type="text/javascript" src="i18n_test.js"></script>

Loading…
Cancel
Save