Migrate attachments from IndexedDB to file system (#2129)
- [x] Generate random file names.
- [x] Generate random file paths that prevent too many files per folder using
fan-out.
- [x] Create attachment directory in user data folder.
- [x] Investigate operating system file indexing on:
- [x] Windows: Confirmed that `AppData` is not indexed by default.
- [x] macOS: Confirmed that `~/Library` files are not indexed by default.
Searching system files using Spotlight requires multi-step opt-in:
https://lifehacker.com/5711409/how-to-search-for-hidden-packaged-and-system-files-in-os-x.
More info https://apple.stackexchange.com/a/92785.
Added `.noindex` suffix to `attachments` folder.
- [x] Linux: n/a
- [x] Save incoming attachment files to disk
- [x] On received
- [x] On sync
- [x] Save outgoing attachments files to disk before sending
- [x] Display attachments either from disk or memory in attachment view.
Be robust to multiple render passes.
- [x] Test that missing attachment on disk doesn’t break app.
Behavior: Message is displayed without attachment.
- [x] Delete attachment files when message is deleted.
Relates to #1589.
pull/1/head
Daniel Gasienica
7 years ago
commit
2e9f3bcf8d
@ -0,0 +1,99 @@
|
|||||||
|
const crypto = require('crypto');
|
||||||
|
const fse = require('fs-extra');
|
||||||
|
const isArrayBuffer = require('lodash/isArrayBuffer');
|
||||||
|
const isString = require('lodash/isString');
|
||||||
|
const path = require('path');
|
||||||
|
const toArrayBuffer = require('to-arraybuffer');
|
||||||
|
|
||||||
|
|
||||||
|
const PATH = 'attachments.noindex';
|
||||||
|
|
||||||
|
// getPath :: AbsolutePath -> AbsolutePath
|
||||||
|
exports.getPath = (userDataPath) => {
|
||||||
|
if (!isString(userDataPath)) {
|
||||||
|
throw new TypeError('`userDataPath` must be a string');
|
||||||
|
}
|
||||||
|
return path.join(userDataPath, PATH);
|
||||||
|
};
|
||||||
|
|
||||||
|
// ensureDirectory :: AbsolutePath -> IO Unit
|
||||||
|
exports.ensureDirectory = async (userDataPath) => {
|
||||||
|
if (!isString(userDataPath)) {
|
||||||
|
throw new TypeError('`userDataPath` must be a string');
|
||||||
|
}
|
||||||
|
await fse.ensureDir(exports.getPath(userDataPath));
|
||||||
|
};
|
||||||
|
|
||||||
|
// readData :: AttachmentsPath ->
|
||||||
|
// RelativePath ->
|
||||||
|
// IO (Promise ArrayBuffer)
|
||||||
|
exports.readData = (root) => {
|
||||||
|
if (!isString(root)) {
|
||||||
|
throw new TypeError('`root` must be a path');
|
||||||
|
}
|
||||||
|
|
||||||
|
return async (relativePath) => {
|
||||||
|
if (!isString(relativePath)) {
|
||||||
|
throw new TypeError('`relativePath` must be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
const absolutePath = path.join(root, relativePath);
|
||||||
|
const buffer = await fse.readFile(absolutePath);
|
||||||
|
return toArrayBuffer(buffer);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// writeData :: AttachmentsPath ->
|
||||||
|
// ArrayBuffer ->
|
||||||
|
// IO (Promise RelativePath)
|
||||||
|
exports.writeData = (root) => {
|
||||||
|
if (!isString(root)) {
|
||||||
|
throw new TypeError('`root` must be a path');
|
||||||
|
}
|
||||||
|
|
||||||
|
return async (arrayBuffer) => {
|
||||||
|
if (!isArrayBuffer(arrayBuffer)) {
|
||||||
|
throw new TypeError('`arrayBuffer` must be an array buffer');
|
||||||
|
}
|
||||||
|
|
||||||
|
const buffer = Buffer.from(arrayBuffer);
|
||||||
|
const name = exports.createName();
|
||||||
|
const relativePath = exports.getRelativePath(name);
|
||||||
|
const absolutePath = path.join(root, relativePath);
|
||||||
|
await fse.ensureFile(absolutePath);
|
||||||
|
await fse.writeFile(absolutePath, buffer);
|
||||||
|
return relativePath;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// deleteData :: AttachmentsPath -> IO Unit
|
||||||
|
exports.deleteData = (root) => {
|
||||||
|
if (!isString(root)) {
|
||||||
|
throw new TypeError('`root` must be a path');
|
||||||
|
}
|
||||||
|
|
||||||
|
return async (relativePath) => {
|
||||||
|
if (!isString(relativePath)) {
|
||||||
|
throw new TypeError('`relativePath` must be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
const absolutePath = path.join(root, relativePath);
|
||||||
|
await fse.remove(absolutePath);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// createName :: Unit -> IO String
|
||||||
|
exports.createName = () => {
|
||||||
|
const buffer = crypto.randomBytes(32);
|
||||||
|
return buffer.toString('hex');
|
||||||
|
};
|
||||||
|
|
||||||
|
// getRelativePath :: String -> IO Path
|
||||||
|
exports.getRelativePath = (name) => {
|
||||||
|
if (!isString(name)) {
|
||||||
|
throw new TypeError('`name` must be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
const prefix = name.slice(0, 2);
|
||||||
|
return path.join(prefix, name);
|
||||||
|
};
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
exports.stringToArrayBuffer = (string) => {
|
||||||
|
if (typeof string !== 'string') {
|
||||||
|
throw new TypeError('"string" must be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
const array = new Uint8Array(string.length);
|
||||||
|
for (let i = 0; i < string.length; i += 1) {
|
||||||
|
array[i] = string.charCodeAt(i);
|
||||||
|
}
|
||||||
|
return array.buffer;
|
||||||
|
};
|
||||||
@ -0,0 +1,40 @@
|
|||||||
|
const isArrayBuffer = require('lodash/isArrayBuffer');
|
||||||
|
const isFunction = require('lodash/isFunction');
|
||||||
|
const isUndefined = require('lodash/isUndefined');
|
||||||
|
const omit = require('lodash/omit');
|
||||||
|
|
||||||
|
|
||||||
|
// type Context :: {
|
||||||
|
// writeAttachmentData :: ArrayBuffer -> Promise (IO Path)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// migrateDataToFileSystem :: Attachment ->
|
||||||
|
// Context ->
|
||||||
|
// Promise Attachment
|
||||||
|
exports.migrateDataToFileSystem = async (attachment, { writeAttachmentData } = {}) => {
|
||||||
|
if (!isFunction(writeAttachmentData)) {
|
||||||
|
throw new TypeError('"writeAttachmentData" must be a function');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { data } = attachment;
|
||||||
|
const hasData = !isUndefined(data);
|
||||||
|
const shouldSkipSchemaUpgrade = !hasData;
|
||||||
|
if (shouldSkipSchemaUpgrade) {
|
||||||
|
console.log('WARNING: `attachment.data` is `undefined`');
|
||||||
|
return attachment;
|
||||||
|
}
|
||||||
|
|
||||||
|
const isValidData = isArrayBuffer(data);
|
||||||
|
if (!isValidData) {
|
||||||
|
throw new TypeError('Expected `attachment.data` to be an array buffer;' +
|
||||||
|
` got: ${typeof attachment.data}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const path = await writeAttachmentData(data);
|
||||||
|
|
||||||
|
const attachmentWithoutData = omit(
|
||||||
|
Object.assign({}, attachment, { path }),
|
||||||
|
['data']
|
||||||
|
);
|
||||||
|
return attachmentWithoutData;
|
||||||
|
};
|
||||||
@ -0,0 +1,105 @@
|
|||||||
|
const fse = require('fs-extra');
|
||||||
|
const path = require('path');
|
||||||
|
const tmp = require('tmp');
|
||||||
|
const { assert } = require('chai');
|
||||||
|
|
||||||
|
const Attachments = require('../../app/attachments');
|
||||||
|
const { stringToArrayBuffer } = require('../../js/modules/string_to_array_buffer');
|
||||||
|
|
||||||
|
|
||||||
|
const PREFIX_LENGTH = 2;
|
||||||
|
const NUM_SEPARATORS = 1;
|
||||||
|
const NAME_LENGTH = 64;
|
||||||
|
const PATH_LENGTH = PREFIX_LENGTH + NUM_SEPARATORS + NAME_LENGTH;
|
||||||
|
|
||||||
|
describe('Attachments', () => {
|
||||||
|
describe('writeData', () => {
|
||||||
|
let tempRootDirectory = null;
|
||||||
|
before(() => {
|
||||||
|
tempRootDirectory = tmp.dirSync().name;
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await fse.remove(tempRootDirectory);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should write file to disk and return path', async () => {
|
||||||
|
const input = stringToArrayBuffer('test string');
|
||||||
|
const tempDirectory = path.join(tempRootDirectory, 'Attachments_writeData');
|
||||||
|
|
||||||
|
const outputPath = await Attachments.writeData(tempDirectory)(input);
|
||||||
|
const output = await fse.readFile(path.join(tempDirectory, outputPath));
|
||||||
|
|
||||||
|
assert.lengthOf(outputPath, PATH_LENGTH);
|
||||||
|
|
||||||
|
const inputBuffer = Buffer.from(input);
|
||||||
|
assert.deepEqual(inputBuffer, output);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('readData', () => {
|
||||||
|
let tempRootDirectory = null;
|
||||||
|
before(() => {
|
||||||
|
tempRootDirectory = tmp.dirSync().name;
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await fse.remove(tempRootDirectory);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should read file from disk', async () => {
|
||||||
|
const tempDirectory = path.join(tempRootDirectory, 'Attachments_readData');
|
||||||
|
|
||||||
|
const relativePath = Attachments.getRelativePath(Attachments.createName());
|
||||||
|
const fullPath = path.join(tempDirectory, relativePath);
|
||||||
|
const input = stringToArrayBuffer('test string');
|
||||||
|
|
||||||
|
const inputBuffer = Buffer.from(input);
|
||||||
|
await fse.ensureFile(fullPath);
|
||||||
|
await fse.writeFile(fullPath, inputBuffer);
|
||||||
|
const output = await Attachments.readData(tempDirectory)(relativePath);
|
||||||
|
|
||||||
|
assert.deepEqual(input, output);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('deleteData', () => {
|
||||||
|
let tempRootDirectory = null;
|
||||||
|
before(() => {
|
||||||
|
tempRootDirectory = tmp.dirSync().name;
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await fse.remove(tempRootDirectory);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete file from disk', async () => {
|
||||||
|
const tempDirectory = path.join(tempRootDirectory, 'Attachments_deleteData');
|
||||||
|
|
||||||
|
const relativePath = Attachments.getRelativePath(Attachments.createName());
|
||||||
|
const fullPath = path.join(tempDirectory, relativePath);
|
||||||
|
const input = stringToArrayBuffer('test string');
|
||||||
|
|
||||||
|
const inputBuffer = Buffer.from(input);
|
||||||
|
await fse.ensureFile(fullPath);
|
||||||
|
await fse.writeFile(fullPath, inputBuffer);
|
||||||
|
await Attachments.deleteData(tempDirectory)(relativePath);
|
||||||
|
|
||||||
|
const existsFile = await fse.exists(fullPath);
|
||||||
|
assert.isFalse(existsFile);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('createName', () => {
|
||||||
|
it('should return random file name with correct length', () => {
|
||||||
|
assert.lengthOf(Attachments.createName(), NAME_LENGTH);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getRelativePath', () => {
|
||||||
|
it('should return correct path', () => {
|
||||||
|
const name = '608ce3bc536edbf7637a6aeb6040bdfec49349140c0dd43e97c7ce263b15ff7e';
|
||||||
|
assert.lengthOf(Attachments.getRelativePath(name), PATH_LENGTH);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Reference in New Issue