keejef
/
session-ios
mirror of https://github.com/oxen-io/session-ios
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Batch backup exports.

Browse Source
pull/1/head
Matthew Chen 6 years ago
parent c1ac5c1872
commit 57205facbc
3 changed files with 144 additions and 346 deletions
Show Stats Download Patch File Download Diff File

8
Signal/src/ViewControllers/DebugUI/DebugUIBackup.m
Unescape Escape View File

@ -96,9 +96,11 @@ NS_ASSUME_NONNULL_BEGIN
OWSAssertDebug(success);
NSString *recipientId = self.tsAccountManager.localNumber;
[[self.backup ensureCloudKitAccess].then(^{
return
[OWSBackupAPI saveTestFileToCloudObjcWithRecipientId:recipientId fileUrl:[NSURL fileURLWithPath:filePath]];
NSString *recordName = [OWSBackupAPI recordNameForTestFileWithRecipientId:recipientId];
CKRecord *record = [OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:filePath] recordName:recordName];
[[self.backup ensureCloudKitAccess].thenInBackground(^{
return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]];
}) retainUntilComplete];
}

235
Signal/src/util/Backup/OWSBackupAPI.swift
Unescape Escape View File

@ -24,10 +24,6 @@ import PromiseKit
static let payloadKey = "payload"
static let maxRetries = 5
private class func recordIdForTest() -> String {
return "test-\(NSUUID().uuidString)"
}
private class func database() -> CKDatabase {
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
@ -43,18 +39,8 @@ import PromiseKit
// MARK: - Upload
@objc
public class func saveTestFileToCloudObjc(recipientId: String,
fileUrl: URL) -> AnyPromise {
return AnyPromise(saveTestFileToCloud(recipientId: recipientId,
fileUrl: fileUrl))
}
public class func saveTestFileToCloud(recipientId: String,
fileUrl: URL) -> Promise<String> {
let recordName = "\(recordNamePrefix(forRecipientId: recipientId))test-\(NSUUID().uuidString)"
return saveFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: signalBackupRecordType)
public class func recordNameForTestFile(recipientId: String) -> String {
return "\(recordNamePrefix(forRecipientId: recipientId))test-\(NSUUID().uuidString)"
}
// "Ephemeral" files are specific to this backup export and will always need to
@ -67,28 +53,6 @@ import PromiseKit
return "\(recordNamePrefix(forRecipientId: recipientId))ephemeral-\(label)-\(NSUUID().uuidString)"
}
// "Ephemeral" files are specific to this backup export and will always need to
// be saved. For example, a complete image of the database is exported each time.
// We wouldn't want to overwrite previous images until the entire backup export is
// complete.
@objc
public class func saveEphemeralFileToCloudObjc(recipientId: String,
label: String,
fileUrl: URL) -> AnyPromise {
return AnyPromise(saveEphemeralFileToCloud(recipientId: recipientId,
label: label,
fileUrl: fileUrl))
}
public class func saveEphemeralFileToCloud(recipientId: String,
label: String,
fileUrl: URL) -> Promise<String> {
let recordName = recordNameForEphemeralFile(recipientId: recipientId, label: label)
return saveFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: signalBackupRecordType)
}
// "Persistent" files may be shared between backup export; they should only be saved
// once. For example, attachment files should only be uploaded once. Subsequent
// backups can reuse the same record.
@ -152,127 +116,6 @@ import PromiseKit
return recipientIds
}
// "Persistent" files may be shared between backup export; they should only be saved
// once. For example, attachment files should only be uploaded once. Subsequent
// backups can reuse the same record.
@objc
public class func savePersistentFileOnceToCloudObjc(recipientId: String,
fileId: String,
fileUrlBlock: @escaping () -> URL?) -> AnyPromise {
return AnyPromise(savePersistentFileOnceToCloud(recipientId: recipientId,
fileId: fileId,
fileUrlBlock: fileUrlBlock))
}
public class func savePersistentFileOnceToCloud(recipientId: String,
fileId: String,
fileUrlBlock: @escaping () -> URL?) -> Promise<String> {
let recordName = recordNameForPersistentFile(recipientId: recipientId, fileId: fileId)
return saveFileOnceToCloud(recordName: recordName,
recordType: signalBackupRecordType,
fileUrlBlock: fileUrlBlock)
}
@objc
public class func upsertManifestFileToCloudObjc(recipientId: String,
fileUrl: URL) -> AnyPromise {
return AnyPromise(upsertManifestFileToCloud(recipientId: recipientId,
fileUrl: fileUrl))
}
public class func upsertManifestFileToCloud(recipientId: String,
fileUrl: URL) -> Promise<String> {
// We want to use a well-known record id and type for manifest files.
let recordName = recordNameForManifest(recipientId: recipientId)
return upsertFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: signalBackupRecordType)
}
@objc
public class func saveFileToCloudObjc(fileUrl: URL,
recordName: String) -> AnyPromise {
return AnyPromise(saveFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: signalBackupRecordType))
}
public class func saveFileToCloud(fileUrl: URL,
recordName: String,
recordType: String) -> Promise<String> {
let recordID = CKRecordID(recordName: recordName)
let record = CKRecord(recordType: recordType, recordID: recordID)
let asset = CKAsset(fileURL: fileUrl)
record[payloadKey] = asset
return saveRecordToCloud(record: record)
}
@objc
public class func saveRecordToCloudObjc(record: CKRecord) -> AnyPromise {
return AnyPromise(saveRecordToCloud(record: record))
}
public class func saveRecordToCloud(record: CKRecord) -> Promise<String> {
return saveRecordToCloud(record: record,
remainingRetries: maxRetries)
}
private class func saveRecordToCloud(record: CKRecord,
remainingRetries: Int) -> Promise<String> {
Logger.verbose("saveRecordToCloud \(record.recordID.recordName)")
return Promise { resolver in
let saveOperation = CKModifyRecordsOperation(recordsToSave: [record ], recordIDsToDelete: nil)
saveOperation.modifyRecordsCompletionBlock = { (_, _, error) in
let outcome = outcomeForCloudKitError(error: error,
remainingRetries: remainingRetries,
label: "Save Record")
switch outcome {
case .success:
let recordName = record.recordID.recordName
resolver.fulfill(recordName)
case .failureDoNotRetry(let outcomeError):
resolver.reject(outcomeError)
case .failureRetryAfterDelay(let retryDelay):
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
saveRecordToCloud(record: record,
remainingRetries: remainingRetries - 1)
.done { (recordName) in
resolver.fulfill(recordName)
}.catch { (error) in
resolver.reject(error)
}.retainUntilComplete()
})
case .failureRetryWithoutDelay:
DispatchQueue.global().async {
saveRecordToCloud(record: record,
remainingRetries: remainingRetries - 1)
.done { (recordName) in
resolver.fulfill(recordName)
}.catch { (error) in
resolver.reject(error)
}.retainUntilComplete()
}
case .unknownItem:
owsFailDebug("unexpected CloudKit response.")
resolver.reject(invalidServiceResponseError())
}
}
saveOperation.isAtomic = false
// These APIs are only available in iOS 9.3 and later.
if #available(iOS 9.3, *) {
saveOperation.isLongLived = true
saveOperation.qualityOfService = .background
}
database().add(saveOperation)
}
}
@objc
public class func record(forFileUrl fileUrl: URL,
recordName: String) -> CKRecord {
@ -366,80 +209,6 @@ import PromiseKit
}
}
// Compare:
// * An "upsert" creates a new record if none exists and
// or updates if there is an existing record.
// * A "save once" creates a new record if none exists and
// does nothing if there is an existing record.
@objc
public class func upsertFileToCloudObjc(fileUrl: URL,
recordName: String,
recordType: String) -> AnyPromise {
return AnyPromise(upsertFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: recordType))
}
public class func upsertFileToCloud(fileUrl: URL,
recordName: String,
recordType: String) -> Promise<String> {
return checkForFileInCloud(recordName: recordName,
remainingRetries: maxRetries)
.then { (record: CKRecord?) -> Promise<String> in
if let record = record {
// Record found, updating existing record.
let asset = CKAsset(fileURL: fileUrl)
record[payloadKey] = asset
return saveRecordToCloud(record: record)
}
// No record found, saving new record.
return saveFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: recordType)
}
}
// Compare:
// * An "upsert" creates a new record if none exists and
// or updates if there is an existing record.
// * A "save once" creates a new record if none exists and
// does nothing if there is an existing record.
@objc
public class func saveFileOnceToCloudObjc(recordName: String,
recordType: String,
fileUrlBlock: @escaping () -> URL?) -> AnyPromise {
return AnyPromise(saveFileOnceToCloud(recordName: recordName,
recordType: recordType,
fileUrlBlock: fileUrlBlock))
}
public class func saveFileOnceToCloud(recordName: String,
recordType: String,
fileUrlBlock: @escaping () -> URL?) -> Promise<String> {
return checkForFileInCloud(recordName: recordName,
remainingRetries: maxRetries)
.then { (record: CKRecord?) -> Promise<String> in
if record != nil {
// Record found, skipping save.
return Promise.value(recordName)
}
// No record found, saving new record.
guard let fileUrl = fileUrlBlock() else {
Logger.error("error preparing file for upload.")
return Promise(error: OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED",
comment: "Error indicating the backup export failed to save a file to the cloud.")))
}
return saveFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: recordType)
}
}
// MARK: - Delete
@objc

247
Signal/src/util/Backup/OWSBackupExportJob.m
Unescape Escape View File

@ -740,8 +740,6 @@ NS_ASSUME_NONNULL_BEGIN
// This method returns YES IFF "work was done and there might be more work to do".
- (AnyPromise *)saveDatabaseFilesToCloud
{
// AnyPromise *promise = [AnyPromise promiseWithValue:@(1)];
if (self.isComplete) {
return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")];
}
@ -759,7 +757,7 @@ NS_ASSUME_NONNULL_BEGIN
}
// TODO: Expose progress.
return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records].thenInBackground(^(NSString *recordName) {
return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records].thenInBackground(^{
OWSAssertDebug(items.count == records.count);
NSUInteger count = MIN(items.count, records.count);
for (NSUInteger i = 0; i < count; i++) {
@ -778,121 +776,152 @@ NS_ASSUME_NONNULL_BEGIN
// This method returns YES IFF "work was done and there might be more work to do".
- (AnyPromise *)saveAttachmentFilesToCloud
{
if (self.isComplete) {
return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")];
}
AnyPromise *promise = [AnyPromise promiseWithValue:@(1)];
NSMutableArray<OWSAttachmentExport *> *items = [NSMutableArray new];
NSMutableArray<CKRecord *> *records = [NSMutableArray new];
for (OWSAttachmentExport *attachmentExport in self.unsavedAttachmentExports) {
if ([self tryToSkipAttachmentUpload:attachmentExport]) {
continue;
}
promise = promise.thenInBackground(^{
if (self.isComplete) {
return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")];
@autoreleasepool {
// OWSAttachmentExport is used to lazily write an encrypted copy of the
// attachment to disk.
if (![attachmentExport prepareForUpload]) {
// Attachment files are non-critical so any error uploading them is recoverable.
return @(1);
}
OWSAssertDebug(attachmentExport.relativeFilePath.length > 0);
OWSAssertDebug(attachmentExport.encryptedItem);
}
NSURL *_Nullable fileUrl = ^{
if (attachmentExport.encryptedItem.filePath.length < 1) {
OWSLogError(@"attachment export missing temp file path");
return (NSURL *)nil;
}
if (attachmentExport.relativeFilePath.length < 1) {
OWSLogError(@"attachment export missing relative file path");
return (NSURL *)nil;
}
return [NSURL fileURLWithPath:attachmentExport.encryptedItem.filePath];
}();
if (!fileUrl) {
// Attachment files are non-critical so any error uploading them is recoverable.
return @(1);
}
return [self saveAttachmentFileToCloud:attachmentExport];
NSString *recordName =
[OWSBackupAPI recordNameForPersistentFileWithRecipientId:self.recipientId
fileId:attachmentExport.attachmentId];
CKRecord *record = [OWSBackupAPI recordForFileUrl:fileUrl recordName:recordName];
[records addObject:record];
[items addObject:attachmentExport];
return @(1);
});
}
[self.unsavedAttachmentExports removeAllObjects];
return promise;
// TODO: Expose progress.
dispatch_queue_t backgroundQueue = dispatch_get_global_queue(0, 0);
return promise
.thenInBackground(^{
return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records];
})
.ensureOn(backgroundQueue,
^{
for (OWSAttachmentExport *attachmentExport in items) {
if (![attachmentExport cleanUp]) {
OWSLogError(@"couldn't clean up attachment export.");
// Attachment files are non-critical so any error uploading them is recoverable.
}
}
})
.thenInBackground(^{
OWSAssertDebug(items.count == records.count);
NSUInteger count = MIN(items.count, records.count);
for (NSUInteger i = 0; i < count; i++) {
OWSAttachmentExport *attachmentExport = items[i];
CKRecord *record = records[i];
NSString *recordName = record.recordID.recordName;
OWSAssertDebug(recordName.length > 0);
OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
exportItem.encryptedItem = attachmentExport.encryptedItem;
exportItem.recordName = recordName;
exportItem.attachmentExport = attachmentExport;
[self.savedAttachmentItems addObject:exportItem];
// Immediately save the record metadata to facilitate export resume.
OWSBackupFragment *backupFragment = [[OWSBackupFragment alloc] initWithUniqueId:recordName];
backupFragment.recordName = recordName;
backupFragment.encryptionKey = exportItem.encryptedItem.encryptionKey;
backupFragment.relativeFilePath = attachmentExport.relativeFilePath;
backupFragment.attachmentId = attachmentExport.attachmentId;
backupFragment.uncompressedDataLength = exportItem.uncompressedDataLength;
[self.dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
[backupFragment saveWithTransaction:transaction];
}];
OWSLogVerbose(@"saved attachment: %@ as %@",
attachmentExport.attachmentFilePath,
attachmentExport.relativeFilePath);
}
})
.catchInBackground(^{
// Attachment files are non-critical so any error uploading them is recoverable.
return [AnyPromise promiseWithValue:@(1)];
});
}
- (AnyPromise *)saveAttachmentFileToCloud:(OWSAttachmentExport *)attachmentExport
- (BOOL)tryToSkipAttachmentUpload:(OWSAttachmentExport *)attachmentExport
{
if (self.lastValidRecordNames) {
// Wherever possible, we do incremental backups and re-use fragments of the last
// backup and/or restore.
// Recycling fragments doesn't just reduce redundant network activity,
// it allows us to skip the local export work, i.e. encryption.
// To do so, we must preserve the metadata for these fragments.
//
// We check two things:
//
// * That we already know the metadata for this fragment (from a previous backup
// or restore).
// * That this record does in fact exist in our CloudKit database.
NSString *lastRecordName =
[OWSBackupAPI recordNameForPersistentFileWithRecipientId:self.recipientId
fileId:attachmentExport.attachmentId];
OWSBackupFragment *_Nullable lastBackupFragment = [OWSBackupFragment fetchObjectWithUniqueID:lastRecordName];
if (lastBackupFragment && [self.lastValidRecordNames containsObject:lastRecordName]) {
OWSAssertDebug(lastBackupFragment.encryptionKey.length > 0);
OWSAssertDebug(lastBackupFragment.relativeFilePath.length > 0);
// Recycle the metadata from the last backup's manifest.
OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new];
encryptedItem.encryptionKey = lastBackupFragment.encryptionKey;
attachmentExport.encryptedItem = encryptedItem;
attachmentExport.relativeFilePath = lastBackupFragment.relativeFilePath;
OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
exportItem.encryptedItem = attachmentExport.encryptedItem;
exportItem.recordName = lastRecordName;
exportItem.attachmentExport = attachmentExport;
[self.savedAttachmentItems addObject:exportItem];
OWSLogVerbose(@"recycled attachment: %@ as %@",
attachmentExport.attachmentFilePath,
attachmentExport.relativeFilePath);
return [AnyPromise promiseWithValue:@(1)];
}
if (!self.lastValidRecordNames) {
return NO;
}
@autoreleasepool {
// OWSAttachmentExport is used to lazily write an encrypted copy of the
// attachment to disk.
if (![attachmentExport prepareForUpload]) {
// Attachment files are non-critical so any error uploading them is recoverable.
return [AnyPromise promiseWithValue:@(1)];
}
OWSAssertDebug(attachmentExport.relativeFilePath.length > 0);
OWSAssertDebug(attachmentExport.encryptedItem);
// Wherever possible, we do incremental backups and re-use fragments of the last
// backup and/or restore.
// Recycling fragments doesn't just reduce redundant network activity,
// it allows us to skip the local export work, i.e. encryption.
// To do so, we must preserve the metadata for these fragments.
//
// We check two things:
//
// * That we already know the metadata for this fragment (from a previous backup
// or restore).
// * That this record does in fact exist in our CloudKit database.
NSString *recordName =
[OWSBackupAPI recordNameForPersistentFileWithRecipientId:self.recipientId fileId:attachmentExport.attachmentId];
OWSBackupFragment *_Nullable lastBackupFragment = [OWSBackupFragment fetchObjectWithUniqueID:recordName];
if (!lastBackupFragment || ![self.lastValidRecordNames containsObject:recordName]) {
return NO;
}
return [OWSBackupAPI
savePersistentFileOnceToCloudObjcWithRecipientId:self.recipientId
fileId:attachmentExport.attachmentId
fileUrlBlock:^{
if (attachmentExport.encryptedItem.filePath.length < 1) {
OWSLogError(@"attachment export missing temp file path");
return (NSURL *)nil;
}
if (attachmentExport.relativeFilePath.length < 1) {
OWSLogError(@"attachment export missing relative file path");
return (NSURL *)nil;
}
return [NSURL fileURLWithPath:attachmentExport.encryptedItem.filePath];
}]
.thenInBackground(^(NSString *recordName) {
if (![attachmentExport cleanUp]) {
OWSLogError(@"couldn't clean up attachment export.");
// Attachment files are non-critical so any error uploading them is recoverable.
}
OWSAssertDebug(lastBackupFragment.encryptionKey.length > 0);
OWSAssertDebug(lastBackupFragment.relativeFilePath.length > 0);
OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
exportItem.encryptedItem = attachmentExport.encryptedItem;
exportItem.recordName = recordName;
exportItem.attachmentExport = attachmentExport;
[self.savedAttachmentItems addObject:exportItem];
// Immediately save the record metadata to facilitate export resume.
OWSBackupFragment *backupFragment = [[OWSBackupFragment alloc] initWithUniqueId:recordName];
backupFragment.recordName = recordName;
backupFragment.encryptionKey = exportItem.encryptedItem.encryptionKey;
backupFragment.relativeFilePath = attachmentExport.relativeFilePath;
backupFragment.attachmentId = attachmentExport.attachmentId;
backupFragment.uncompressedDataLength = exportItem.uncompressedDataLength;
[self.dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
[backupFragment saveWithTransaction:transaction];
}];
// Recycle the metadata from the last backup's manifest.
OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new];
encryptedItem.encryptionKey = lastBackupFragment.encryptionKey;
attachmentExport.encryptedItem = encryptedItem;
attachmentExport.relativeFilePath = lastBackupFragment.relativeFilePath;
OWSLogVerbose(
@"saved attachment: %@ as %@", attachmentExport.attachmentFilePath, attachmentExport.relativeFilePath);
})
.catchInBackground(^{
if (![attachmentExport cleanUp]) {
OWSLogError(@"couldn't clean up attachment export.");
// Attachment files are non-critical so any error uploading them is recoverable.
}
OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
exportItem.encryptedItem = attachmentExport.encryptedItem;
exportItem.recordName = recordName;
exportItem.attachmentExport = attachmentExport;
[self.savedAttachmentItems addObject:exportItem];
// Attachment files are non-critical so any error uploading them is recoverable.
return [AnyPromise promiseWithValue:@(1)];
});
OWSLogVerbose(
@"recycled attachment: %@ as %@", attachmentExport.attachmentFilePath, attachmentExport.relativeFilePath);
return YES;
}
- (AnyPromise *)saveLocalProfileAvatarToCloud
@ -922,8 +951,6 @@ NS_ASSUME_NONNULL_BEGIN
return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]].thenInBackground(^{
exportItem.recordName = recordName;
self.localProfileAvatarItem = exportItem;
return [AnyPromise promiseWithValue:@(1)];
});
}
@ -941,14 +968,14 @@ NS_ASSUME_NONNULL_BEGIN
OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
exportItem.encryptedItem = encryptedItem;
return [OWSBackupAPI upsertManifestFileToCloudObjcWithRecipientId:self.recipientId
fileUrl:[NSURL fileURLWithPath:encryptedItem.filePath]]
.thenInBackground(^(NSString *recordName) {
exportItem.recordName = recordName;
self.manifestItem = exportItem;
// All files have been saved to the cloud.
});
NSString *recordName = [OWSBackupAPI recordNameForManifestWithRecipientId:self.recipientId];
CKRecord *record =
[OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:encryptedItem.filePath] recordName:recordName];
return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]].thenInBackground(^{
exportItem.recordName = recordName;
self.manifestItem = exportItem;
});
}
- (nullable OWSBackupEncryptedItem *)writeManifestFile

Write Preview
Loading…
Cancel
Save