Complete minimal backup MVP.

pull/1/head
Matthew Chen 7 years ago committed by Matthew Chen
parent f46ea0e87d
commit 3f822e8ce3

@ -32,6 +32,10 @@ NS_ASSUME_NONNULL_BEGIN
actionBlock:^{
[DebugUIBackup checkForBackup];
}]];
[items addObject:[OWSTableItem itemWithTitle:@"Log CloudKit backup records"
actionBlock:^{
[DebugUIBackup logBackupRecords];
}]];
[items addObject:[OWSTableItem itemWithTitle:@"Try to restore CloudKit backup"
actionBlock:^{
[DebugUIBackup tryToImportBackup];
@ -75,6 +79,13 @@ NS_ASSUME_NONNULL_BEGIN
}];
}
+ (void)logBackupRecords
{
DDLogInfo(@"%@ logBackupRecords.", self.logTag);
[OWSBackup.sharedManager logBackupRecords];
}
+ (void)tryToImportBackup
{
DDLogInfo(@"%@ tryToImportBackup.", self.logTag);

@ -284,6 +284,10 @@ typedef NS_ENUM(NSInteger, CellState) { kArchiveState, kInboxState };
}
[self updateBarButtonItems];
dispatch_async(dispatch_get_main_queue(), ^{
[self settingsButtonPressed:nil];
});
}
- (void)viewDidAppear:(BOOL)animated

@ -63,6 +63,8 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) {
- (void)tryToImportBackup;
- (void)cancelImportBackup;
- (void)logBackupRecords;
@end
NS_ASSUME_NONNULL_END

@ -435,6 +435,23 @@ NS_ASSUME_NONNULL_BEGIN
}
}
- (void)logBackupRecords
{
OWSAssertIsOnMainThread();
DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
[OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray<NSString *> *recordNames) {
for (NSString *recordName in [recordNames sortedArrayUsingSelector:@selector(compare:)]) {
DDLogInfo(@"%@ \t %@", self.logTag, recordName);
}
DDLogInfo(@"%@ record count: %zd", self.logTag, recordNames.count);
}
failure:^(NSError *error) {
DDLogError(@"%@ Failed to retrieve backup records: %@", self.logTag, error);
}];
}
#pragma mark - Notifications
- (void)postDidChangeNotification

@ -39,7 +39,7 @@ import CloudKit
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveFileToCloud(fileUrl: fileUrl,
recordName: NSUUID().uuidString,
recordName: "ephemeralFile-\(NSUUID().uuidString)",
recordType: signalBackupRecordType,
success: success,
failure: failure)
@ -342,10 +342,14 @@ import CloudKit
success: { (asset) in
DispatchQueue.global().async {
do {
// TODO: delete
Logger.verbose("asset.fileURL: \(asset.fileURL.absoluteString)")
Logger.flush()
let data = try Data(contentsOf: asset.fileURL)
success(data)
} catch {
Logger.error("\(self.logTag) couldn't copy asset file.")
Logger.error("\(self.logTag) couldn't load asset file: \(error).")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED",
comment: "Error indicating the a backup import failed to download a file from the cloud.")))
@ -365,11 +369,15 @@ import CloudKit
success: { (asset) in
DispatchQueue.global().async {
do {
let fileManager = FileManager.default
try fileManager.copyItem(at: asset.fileURL, to: toFileUrl)
// TODO: delete
Logger.verbose("asset.fileURL: \(asset.fileURL.absoluteString)")
Logger.verbose("toFileUrl: \(toFileUrl.absoluteString)")
Logger.flush()
try FileManager.default.copyItem(at: asset.fileURL, to: toFileUrl)
success()
} catch {
Logger.error("\(self.logTag) couldn't copy asset file.")
Logger.error("\(self.logTag) couldn't copy asset file: \(error).")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED",
comment: "Error indicating the a backup import failed to download a file from the cloud.")))

@ -190,9 +190,9 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
return completion(NO);
}
NSString *exportDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"Database"];
if (![OWSFileSystem ensureDirectoryExists:exportDatabaseDirPath]) {
OWSProdLogAndFail(@"%@ Could not create exportDatabaseDirPath.", self.logTag);
NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"];
if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) {
OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag);
return completion(NO);
}
@ -205,7 +205,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
return databaseKeySpec;
};
self.backupStorage =
[[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:exportDatabaseDirPath keySpecBlock:keySpecBlock];
[[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock];
if (!self.backupStorage) {
OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag);
return completion(NO);
@ -265,60 +265,62 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
copiedEntities++;
}];
// Copy interactions.
// Copy attachments.
[srcTransaction
enumerateKeysAndObjectsInCollection:[TSInteraction collection]
enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
if (self.isComplete) {
*stop = YES;
return;
}
if (![object isKindOfClass:[TSInteraction class]]) {
if (![object isKindOfClass:[TSAttachment class]]) {
OWSProdLogAndFail(
@"%@ unexpected class: %@", self.logTag, [object class]);
return;
}
// Ignore disappearing messages.
if ([object isKindOfClass:[TSMessage class]]) {
TSMessage *message = object;
if (message.isExpiringMessage) {
return;
if ([object isKindOfClass:[TSAttachmentStream class]]) {
TSAttachmentStream *attachmentStream = object;
NSString *_Nullable filePath = attachmentStream.filePath;
if (filePath) {
OWSAssert(attachmentStream.uniqueId.length > 0);
self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath;
}
}
TSInteraction *interaction = object;
// Ignore dynamic interactions.
if (interaction.isDynamicInteraction) {
return;
}
[interaction saveWithTransaction:dstTransaction];
copiedInteractions++;
TSAttachment *attachment = object;
[attachment saveWithTransaction:dstTransaction];
copiedAttachments++;
copiedEntities++;
}];
// Copy attachments.
// Copy interactions.
//
// Interactions refer to threads and attachments, so copy the last.
[srcTransaction
enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection]
enumerateKeysAndObjectsInCollection:[TSInteraction collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
if (self.isComplete) {
*stop = YES;
return;
}
if (![object isKindOfClass:[TSAttachment class]]) {
if (![object isKindOfClass:[TSInteraction class]]) {
OWSProdLogAndFail(
@"%@ unexpected class: %@", self.logTag, [object class]);
return;
}
if ([object isKindOfClass:[TSAttachmentStream class]]) {
TSAttachmentStream *attachmentStream = object;
NSString *_Nullable filePath = attachmentStream.filePath;
if (filePath) {
OWSAssert(attachmentStream.uniqueId.length > 0);
self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath;
// Ignore disappearing messages.
if ([object isKindOfClass:[TSMessage class]]) {
TSMessage *message = object;
if (message.isExpiringMessage) {
return;
}
}
TSAttachment *attachment = object;
[attachment saveWithTransaction:dstTransaction];
copiedAttachments++;
TSInteraction *interaction = object;
// Ignore dynamic interactions.
if (interaction.isDynamicInteraction) {
return;
}
[interaction saveWithTransaction:dstTransaction];
copiedInteractions++;
copiedEntities++;
}];
}];
@ -373,109 +375,134 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
@"Indicates that the backup export data is being uploaded.")
progress:@(progress)];
__weak OWSBackupExportJob *weakSelf = self;
if (self.databaseFilePaths.count > 0) {
NSString *filePath = self.databaseFilePaths.lastObject;
[self.databaseFilePaths removeLastObject];
// Database files are encrypted and can be safely stored unencrypted in the cloud.
// TODO: Security review.
[OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath]
success:^(NSString *recordName) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExportJob *strongSelf = weakSelf;
if (!strongSelf) {
return;
}
strongSelf.databaseRecordMap[recordName] = [filePath lastPathComponent];
[strongSelf saveNextFileToCloud:completion];
});
}
failure:^(NSError *error) {
// Database files are critical so any error uploading them is unrecoverable.
completion(error);
}];
if ([self saveNextDatabaseFileToCloud:completion]) {
return;
}
if (self.attachmentFilePathMap.count > 0) {
NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject;
NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId];
[self.attachmentFilePathMap removeObjectForKey:attachmentId];
// OWSAttachmentExport is used to lazily write an encrypted copy of the
// attachment to disk.
OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new];
attachmentExport.delegate = self.delegate;
attachmentExport.jobTempDirPath = self.jobTempDirPath;
attachmentExport.attachmentId = attachmentId;
attachmentExport.attachmentFilePath = attachmentFilePath;
[OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId
fileUrlBlock:^{
[attachmentExport prepareForUpload];
if (attachmentExport.tempFilePath.length < 1) {
DDLogError(@"%@ attachment export missing temp file path", self.logTag);
return (NSURL *)nil;
}
if (attachmentExport.relativeFilePath.length < 1) {
DDLogError(@"%@ attachment export missing relative file path", self.logTag);
return (NSURL *)nil;
}
return [NSURL fileURLWithPath:attachmentExport.tempFilePath];
}
success:^(NSString *recordName) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExportJob *strongSelf = weakSelf;
if (!strongSelf) {
return;
}
strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath;
[strongSelf saveNextFileToCloud:completion];
});
}
failure:^(NSError *error) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Attachment files are non-critical so any error uploading them is recoverable.
[weakSelf saveNextFileToCloud:completion];
});
}];
if ([self saveNextAttachmentFileToCloud:completion]) {
return;
}
[self saveManifestFileToCloud:completion];
}
if (!self.manifestFilePath) {
if (![self writeManifestFile]) {
completion(OWSErrorWithCodeDescription(OWSErrorCodeExportBackupFailed,
NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
@"Error indicating the a backup export could not export the user's data.")));
return;
- (BOOL)saveNextDatabaseFileToCloud:(OWSBackupJobCompletion)completion
{
OWSAssert(completion);
__weak OWSBackupExportJob *weakSelf = self;
if (self.databaseFilePaths.count < 1) {
return NO;
}
NSString *filePath = self.databaseFilePaths.lastObject;
[self.databaseFilePaths removeLastObject];
// Database files are encrypted and can be safely stored unencrypted in the cloud.
// TODO: Security review.
[OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath]
success:^(NSString *recordName) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExportJob *strongSelf = weakSelf;
if (!strongSelf) {
return;
}
strongSelf.databaseRecordMap[recordName] = [filePath lastPathComponent];
[strongSelf saveNextFileToCloud:completion];
});
}
OWSAssert(self.manifestFilePath);
[OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:self.manifestFilePath]
success:^(NSString *recordName) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExportJob *strongSelf = weakSelf;
if (!strongSelf) {
return;
}
strongSelf.manifestRecordName = recordName;
[strongSelf saveNextFileToCloud:completion];
});
failure:^(NSError *error) {
// Database files are critical so any error uploading them is unrecoverable.
completion(error);
}];
return YES;
}
- (BOOL)saveNextAttachmentFileToCloud:(OWSBackupJobCompletion)completion
{
OWSAssert(completion);
__weak OWSBackupExportJob *weakSelf = self;
if (self.attachmentFilePathMap.count < 1) {
return NO;
}
NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject;
NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId];
[self.attachmentFilePathMap removeObjectForKey:attachmentId];
// OWSAttachmentExport is used to lazily write an encrypted copy of the
// attachment to disk.
OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new];
attachmentExport.delegate = self.delegate;
attachmentExport.jobTempDirPath = self.jobTempDirPath;
attachmentExport.attachmentId = attachmentId;
attachmentExport.attachmentFilePath = attachmentFilePath;
[OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId
fileUrlBlock:^{
[attachmentExport prepareForUpload];
if (attachmentExport.tempFilePath.length < 1) {
DDLogError(@"%@ attachment export missing temp file path", self.logTag);
return (NSURL *)nil;
}
failure:^(NSError *error) {
// The manifest file is critical so any error uploading them is unrecoverable.
completion(error);
}];
if (attachmentExport.relativeFilePath.length < 1) {
DDLogError(@"%@ attachment export missing relative file path", self.logTag);
return (NSURL *)nil;
}
return [NSURL fileURLWithPath:attachmentExport.tempFilePath];
}
success:^(NSString *recordName) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExportJob *strongSelf = weakSelf;
if (!strongSelf) {
return;
}
strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath;
[strongSelf saveNextFileToCloud:completion];
});
}
failure:^(NSError *error) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Attachment files are non-critical so any error uploading them is recoverable.
[weakSelf saveNextFileToCloud:completion];
});
}];
return YES;
}
- (void)saveManifestFileToCloud:(OWSBackupJobCompletion)completion
{
OWSAssert(completion);
if (![self writeManifestFile]) {
completion(OWSErrorWithCodeDescription(OWSErrorCodeExportBackupFailed,
NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
@"Error indicating the a backup export could not export the user's data.")));
return;
}
OWSAssert(self.manifestFilePath);
__weak OWSBackupExportJob *weakSelf = self;
[OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:self.manifestFilePath]
success:^(NSString *recordName) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExportJob *strongSelf = weakSelf;
if (!strongSelf) {
return;
}
strongSelf.manifestRecordName = recordName;
// All files have been saved to the cloud.
completion(nil);
// All files have been saved to the cloud.
completion(nil);
});
}
failure:^(NSError *error) {
// The manifest file is critical so any error uploading them is unrecoverable.
completion(error);
}];
}
- (BOOL)writeManifestFile
@ -497,6 +524,11 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
// JSON doesn't support byte arrays.
kOWSBackup_ManifestKey_DatabaseKeySpec : databaseKeySpec.base64EncodedString,
};
DDLogVerbose(@"%@ self.attachmentRecordMap: %@", self.logTag, self.attachmentRecordMap);
DDLogVerbose(@"%@ json: %@", self.logTag, json);
[DDLog flushLog];
NSError *error;
NSData *_Nullable jsonData =
[NSJSONSerialization dataWithJSONObject:json options:NSJSONWritingPrettyPrinted error:&error];

@ -25,65 +25,6 @@ NS_ASSUME_NONNULL_BEGIN
NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKeySpec";
//@interface OWSAttachmentImport : NSObject
//
//@property (nonatomic, weak) id<OWSBackupJobDelegate> delegate;
//@property (nonatomic) NSString *jobTempDirPath;
//@property (nonatomic) NSString *attachmentId;
//@property (nonatomic) NSString *attachmentFilePath;
//@property (nonatomic, nullable) NSString *tempFilePath;
//@property (nonatomic, nullable) NSString *relativeFilePath;
//
//@end
//
//#pragma mark -
//
//@implementation OWSAttachmentImport
//
//- (void)dealloc
//{
// // Surface memory leaks by logging the deallocation.
// DDLogVerbose(@"Dealloc: %@", self.class);
//
// // Delete temporary file ASAP.
// if (self.tempFilePath) {
// [OWSFileSystem deleteFileIfExists:self.tempFilePath];
// }
//}
//
//// On success, tempFilePath will be non-nil.
//- (void)prepareForUpload
//{
// OWSAssert(self.jobTempDirPath.length > 0);
// OWSAssert(self.attachmentId.length > 0);
// OWSAssert(self.attachmentFilePath.length > 0);
//
// NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder];
// if (![self.attachmentFilePath hasPrefix:attachmentsDirPath]) {
// DDLogError(@"%@ attachment has unexpected path.", self.logTag);
// OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, self.attachmentFilePath);
// return;
// }
// NSString *relativeFilePath = [self.attachmentFilePath substringFromIndex:attachmentsDirPath.length];
// NSString *pathSeparator = @"/";
// if ([relativeFilePath hasPrefix:pathSeparator]) {
// relativeFilePath = [relativeFilePath substringFromIndex:pathSeparator.length];
// }
// self.relativeFilePath = relativeFilePath;
//
// NSString *_Nullable tempFilePath = [OWSBackupImportJob encryptFileAsTempFile:self.attachmentFilePath
// jobTempDirPath:self.jobTempDirPath
// delegate:self.delegate];
// if (!tempFilePath) {
// DDLogError(@"%@ attachment could not be encrypted.", self.logTag);
// OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath);
// return;
// }
// self.tempFilePath = tempFilePath;
//}
//
//@end
#pragma mark -
@interface OWSBackupImportJob () <SSZipArchiveDelegate>
@ -141,53 +82,66 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
progress:nil];
__weak OWSBackupImportJob *weakSelf = self;
[self configureImport:^(BOOL success) {
if (!success) {
[self failWithErrorDescription:
NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
@"Error indicating the a backup import could not import the user's data.")];
[self configureImport:^(BOOL configureSuccess) {
if (!configureSuccess) {
[weakSelf failWithErrorDescription:
NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
@"Error indicating the a backup import could not import the user's data.")];
return;
}
if (self.isComplete) {
if (weakSelf.isComplete) {
return;
}
[self downloadAndProcessManifest:^(NSError *_Nullable manifestError) {
[weakSelf downloadAndProcessManifest:^(NSError *_Nullable manifestError) {
if (manifestError) {
[self failWithError:manifestError];
[weakSelf failWithError:manifestError];
return;
}
if (self.isComplete) {
if (weakSelf.isComplete) {
return;
}
NSMutableArray<NSString *> *allRecordNames = [NSMutableArray new];
[allRecordNames addObjectsFromArray:self.databaseRecordMap.allKeys];
[allRecordNames addObjectsFromArray:self.attachmentRecordMap.allKeys];
[self downloadFilesFromCloud:allRecordNames
completion:^(NSError *_Nullable fileDownloadError) {
if (fileDownloadError) {
[self failWithError:fileDownloadError];
return;
}
if (self.isComplete) {
return;
}
[self restoreAttachmentFiles:^(NSError *_Nullable restoreAttachmentError) {
if (restoreAttachmentError) {
[self failWithError:restoreAttachmentError];
return;
}
if (self.isComplete) {
return;
}
}];
}];
[allRecordNames addObjectsFromArray:weakSelf.databaseRecordMap.allKeys];
// TODO: We could skip attachments that have already been restored
// by previous "backup import" attempts.
[allRecordNames addObjectsFromArray:weakSelf.attachmentRecordMap.allKeys];
[weakSelf
downloadFilesFromCloud:allRecordNames
completion:^(NSError *_Nullable fileDownloadError) {
if (fileDownloadError) {
[weakSelf failWithError:fileDownloadError];
return;
}
if (weakSelf.isComplete) {
return;
}
[weakSelf restoreAttachmentFiles];
if (weakSelf.isComplete) {
return;
}
[weakSelf restoreDatabase:^(BOOL restoreDatabaseSuccess) {
if (!restoreDatabaseSuccess) {
[weakSelf failWithErrorDescription:NSLocalizedString(
@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
@"Error indicating the a backup import "
@"could not import the user's data.")];
return;
}
if (weakSelf.isComplete) {
return;
}
[weakSelf succeed];
}];
}];
// TODO:
}];
@ -311,6 +265,10 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag);
return completion(NO);
}
DDLogVerbose(@"%@ json: %@", self.logTag, json);
[DDLog flushLog];
NSDictionary<NSString *, NSString *> *_Nullable databaseRecordMap = json[kOWSBackup_ManifestKey_DatabaseFiles];
NSDictionary<NSString *, NSString *> *_Nullable attachmentRecordMap = json[kOWSBackup_ManifestKey_AttachmentFiles];
NSString *_Nullable databaseKeySpecBase64 = json[kOWSBackup_ManifestKey_DatabaseKeySpec];
@ -330,6 +288,9 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
return completion(NO);
}
DDLogVerbose(@"%@ attachmentRecordMap: %@", self.logTag, attachmentRecordMap);
[DDLog flushLog];
self.databaseRecordMap = [databaseRecordMap mutableCopy];
self.attachmentRecordMap = [attachmentRecordMap mutableCopy];
@ -426,6 +387,11 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
OWSAssert(recordNames);
OWSAssert(completion);
if (self.isComplete) {
// Job was aborted.
return completion(nil);
}
if (recordNames.count < 1) {
// All downloads are complete; exit.
return completion(nil);
@ -434,15 +400,26 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
[recordNames removeLastObject];
if (![recordName isKindOfClass:[NSString class]]) {
DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]);
// Invalid record name in the manifest. This may be recoverable.
// Ignore this for now and proceed with the other downloads.
return [self downloadNextFileFromCloud:recordNames completion:completion];
}
NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString];
// Use a predictable file path so that multiple "import backup" attempts
// will leverage successful file downloads from previous attempts.
NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:recordName];
// Skip redundant file download.
if ([NSFileManager.defaultManager fileExistsAtPath:tempFilePath]) {
[OWSFileSystem protectFileOrFolderAtPath:tempFilePath];
self.downloadedFileMap[recordName] = tempFilePath;
[self downloadNextFileFromCloud:recordNames completion:completion];
return;
}
[OWSBackupAPI downloadFileFromCloudWithRecordName:recordName
toFileUrl:[NSURL URLWithString:tempFilePath]
toFileUrl:[NSURL fileURLWithPath:tempFilePath]
success:^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[OWSFileSystem protectFileOrFolderAtPath:tempFilePath];
@ -455,145 +432,236 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
}];
}
- (void)restoreAttachmentFiles:(OWSBackupJobCompletion)completion
- (void)restoreAttachmentFiles
{
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder];
DDLogVerbose(@"%@ self.attachmentRecordMap: %@", self.logTag, self.attachmentRecordMap);
[DDLog flushLog];
for (NSString *recordName in self.attachmentRecordMap) {
if (self.isComplete) {
return;
}
NSString *dstRelativePath = self.attachmentRecordMap[recordName];
if (!
[self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:attachmentsDirPath]) {
// Attachment-related errors are recoverable and can be ignored.
continue;
}
}
}
- (void)restoreDatabase:(OWSBackupJobBoolCompletion)completion
{
OWSAssert(completion);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
// // A map of "record name"-to-"downloaded file path".
// self.downloadedFileMap = [NSMutableDictionary new];
//
// [self downloadNextFileFromCloud:recordNames
// completion:completion];
NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"];
if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) {
OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag);
return completion(NO);
}
for (NSString *recordName in self.databaseRecordMap) {
if (self.isComplete) {
return completion(NO);
}
NSString *dstRelativePath = self.databaseRecordMap[recordName];
if (!
[self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:jobDatabaseDirPath]) {
// Database-related errors are unrecoverable.
return completion(NO);
}
}
BackupStorageKeySpecBlock keySpecBlock = ^{
NSData *_Nullable databaseKeySpec =
[OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ImportDatabaseKeySpec];
if (!databaseKeySpec) {
OWSProdLogAndFail(@"%@ Could not load database keyspec for import.", self.logTag);
}
return databaseKeySpec;
};
OWSBackupStorage *_Nullable backupStorage =
[[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock];
if (!backupStorage) {
OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag);
return completion(NO);
}
// TODO: Do we really need to run these registrations on the main thread?
__weak OWSBackupImportJob *weakSelf = self;
dispatch_async(dispatch_get_main_queue(), ^{
[backupStorage runSyncRegistrations];
[backupStorage runAsyncRegistrationsWithCompletion:^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[weakSelf restoreDatabaseContents:backupStorage completion:completion];
completion(YES);
});
}];
});
}
- (void)restoreDatabaseContents:(OWSBackupStorage *)backupStorage completion:(OWSBackupJobBoolCompletion)completion
{
OWSAssert(backupStorage);
OWSAssert(completion);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
if (self.isComplete) {
return completion(NO);
}
YapDatabaseConnection *_Nullable tempDBConnection = backupStorage.newDatabaseConnection;
if (!tempDBConnection) {
OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag);
return completion(NO);
}
YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection;
if (!primaryDBConnection) {
OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag);
return completion(NO);
}
__block unsigned long long copiedThreads = 0;
__block unsigned long long copiedInteractions = 0;
__block unsigned long long copiedEntities = 0;
__block unsigned long long copiedAttachments = 0;
[tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *srcTransaction) {
[primaryDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) {
// Copy threads.
[srcTransaction
enumerateKeysAndObjectsInCollection:[TSThread collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
if (self.isComplete) {
*stop = YES;
return;
}
if (![object isKindOfClass:[TSThread class]]) {
OWSProdLogAndFail(
@"%@ unexpected class: %@", self.logTag, [object class]);
return;
}
TSThread *thread = object;
[thread saveWithTransaction:dstTransaction];
copiedThreads++;
copiedEntities++;
}];
// Copy attachments.
[srcTransaction
enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
if (self.isComplete) {
*stop = YES;
return;
}
if (![object isKindOfClass:[TSAttachment class]]) {
OWSProdLogAndFail(
@"%@ unexpected class: %@", self.logTag, [object class]);
return;
}
TSAttachment *attachment = object;
[attachment saveWithTransaction:dstTransaction];
copiedAttachments++;
copiedEntities++;
}];
// Copy interactions.
//
// Interactions refer to threads and attachments, so copy the last.
[srcTransaction
enumerateKeysAndObjectsInCollection:[TSInteraction collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
if (self.isComplete) {
*stop = YES;
return;
}
if (![object isKindOfClass:[TSInteraction class]]) {
OWSProdLogAndFail(
@"%@ unexpected class: %@", self.logTag, [object class]);
return;
}
// Ignore disappearing messages.
if ([object isKindOfClass:[TSMessage class]]) {
TSMessage *message = object;
if (message.isExpiringMessage) {
return;
}
}
TSInteraction *interaction = object;
// Ignore dynamic interactions.
if (interaction.isDynamicInteraction) {
return;
}
[interaction saveWithTransaction:dstTransaction];
copiedInteractions++;
copiedEntities++;
}];
}];
}];
DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads);
DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions);
DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities);
DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments);
[backupStorage logFileSizes];
// Close the database.
tempDBConnection = nil;
backupStorage = nil;
return completion(YES);
}
- (BOOL)restoreFileWithRecordName:(NSString *)recordName
dstRelativePath:(NSString *)dstRelativePath
dstDirPath:(NSString *)dstDirPath
{
OWSAssert(recordName);
OWSAssert(dstDirPath.length > 0);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
if (![recordName isKindOfClass:[NSString class]]) {
DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]);
return NO;
}
if (![dstRelativePath isKindOfClass:[NSString class]]) {
DDLogError(@"%@ invalid dstRelativePath in manifest: %@", self.logTag, [recordName class]);
return NO;
}
NSString *dstFilePath = [dstDirPath stringByAppendingPathComponent:dstRelativePath];
if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) {
DDLogError(@"%@ skipping redundant file restore.", self.logTag);
return NO;
}
NSString *downloadedFilePath = self.downloadedFileMap[recordName];
if (![NSFileManager.defaultManager fileExistsAtPath:downloadedFilePath]) {
DDLogError(@"%@ missing downloaded attachment file.", self.logTag);
return NO;
}
NSError *error;
BOOL success = [NSFileManager.defaultManager moveItemAtPath:downloadedFilePath toPath:dstFilePath error:&error];
if (!success || error) {
DDLogError(@"%@ could not restore attachment file.", self.logTag);
return NO;
}
// TODO: Remove
DDLogVerbose(@"%@ Restored attachment file: %@ -> %@", self.logTag, downloadedFilePath, dstFilePath);
return YES;
}
//- (BOOL)importDatabase
//{
// DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
//
// YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection;
// if (!tempDBConnection) {
// OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag);
// return NO;
// }
// YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection;
// if (!primaryDBConnection) {
// OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag);
// return NO;
// }
//
// __block unsigned long long copiedThreads = 0;
// __block unsigned long long copiedInteractions = 0;
// __block unsigned long long copiedEntities = 0;
// __block unsigned long long copiedAttachments = 0;
//
// self.attachmentFilePathMap = [NSMutableDictionary new];
//
// [primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) {
// [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) {
// // Copy threads.
// [srcTransaction
// enumerateKeysAndObjectsInCollection:[TSThread collection]
// usingBlock:^(NSString *key, id object, BOOL *stop) {
// if (self.isComplete) {
// *stop = YES;
// return;
// }
// if (![object isKindOfClass:[TSThread class]]) {
// OWSProdLogAndFail(
// @"%@ unexpected class: %@", self.logTag, [object class]);
// return;
// }
// TSThread *thread = object;
// [thread saveWithTransaction:dstTransaction];
// copiedThreads++;
// copiedEntities++;
// }];
//
// // Copy interactions.
// [srcTransaction
// enumerateKeysAndObjectsInCollection:[TSInteraction collection]
// usingBlock:^(NSString *key, id object, BOOL *stop) {
// if (self.isComplete) {
// *stop = YES;
// return;
// }
// if (![object isKindOfClass:[TSInteraction class]]) {
// OWSProdLogAndFail(
// @"%@ unexpected class: %@", self.logTag, [object class]);
// return;
// }
// // Ignore disappearing messages.
// if ([object isKindOfClass:[TSMessage class]]) {
// TSMessage *message = object;
// if (message.isExpiringMessage) {
// return;
// }
// }
// TSInteraction *interaction = object;
// // Ignore dynamic interactions.
// if (interaction.isDynamicInteraction) {
// return;
// }
// [interaction saveWithTransaction:dstTransaction];
// copiedInteractions++;
// copiedEntities++;
// }];
//
// // Copy attachments.
// [srcTransaction
// enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection]
// usingBlock:^(NSString *key, id object, BOOL *stop) {
// if (self.isComplete) {
// *stop = YES;
// return;
// }
// if (![object isKindOfClass:[TSAttachment class]]) {
// OWSProdLogAndFail(
// @"%@ unexpected class: %@", self.logTag, [object class]);
// return;
// }
// if ([object isKindOfClass:[TSAttachmentStream class]]) {
// TSAttachmentStream *attachmentStream = object;
// NSString *_Nullable filePath = attachmentStream.filePath;
// if (filePath) {
// OWSAssert(attachmentStream.uniqueId.length > 0);
// self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath;
// }
// }
// TSAttachment *attachment = object;
// [attachment saveWithTransaction:dstTransaction];
// copiedAttachments++;
// copiedEntities++;
// }];
// }];
// }];
//
// // TODO: Should we do a database checkpoint?
//
// DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads);
// DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions);
// DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities);
// DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments);
//
// [self.backupStorage logFileSizes];
//
// // Capture the list of files to save.
// self.databaseFilePaths = [@[
// self.backupStorage.databaseFilePath,
// self.backupStorage.databaseFilePath_WAL,
// self.backupStorage.databaseFilePath_SHM,
// ] mutableCopy];
//
// // Close the database.
// tempDBConnection = nil;
// self.backupStorage = nil;
//
// return YES;
//}
//
//- (void)saveToCloud:(OWSBackupJobCompletion)completion
//{
// OWSAssert(completion);

@ -101,6 +101,9 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
{
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
// TODO: Exports should use a new directory each time, but imports
// might want to use a predictable directory so that repeated
// import attempts can reuse downloads from previous attempts.
NSString *temporaryDirectory = NSTemporaryDirectory();
self.jobTempDirPath = [temporaryDirectory stringByAppendingString:[NSUUID UUID].UUIDString];

@ -23,16 +23,10 @@ typedef NSData *_Nullable (^BackupStorageKeySpecBlock)(void);
- (YapDatabaseConnection *)dbConnection;
- (void)logFileSizes;
- (void)runSyncRegistrations;
- (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion;
- (BOOL)areAllRegistrationsComplete;
- (NSString *)databaseFilePath;
- (NSString *)databaseFilePath_SHM;
- (NSString *)databaseFilePath_WAL;
@end
NS_ASSUME_NONNULL_END

@ -93,13 +93,6 @@ NS_ASSUME_NONNULL_BEGIN
}];
}
- (void)logFileSizes
{
DDLogInfo(@"%@ Database file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath]);
DDLogInfo(@"%@ \t SHM file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM]);
DDLogInfo(@"%@ \t WAL file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL]);
}
- (void)protectFiles
{
[self logFileSizes];

@ -309,11 +309,31 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage)
return self.sharedDataDatabaseFilePath;
}
+ (NSString *)databaseFilePath_SHM
{
return self.sharedDataDatabaseFilePath_SHM;
}
+ (NSString *)databaseFilePath_WAL
{
return self.sharedDataDatabaseFilePath_WAL;
}
- (NSString *)databaseFilePath
{
return OWSPrimaryStorage.databaseFilePath;
}
- (NSString *)databaseFilename_SHM
{
return OWSPrimaryStorage.databaseFilename_SHM;
}
- (NSString *)databaseFilename_WAL
{
return OWSPrimaryStorage.databaseFilename_WAL;
}
+ (YapDatabaseConnection *)dbReadConnection
{
return OWSPrimaryStorage.sharedManager.dbReadConnection;

@ -17,6 +17,8 @@ NS_ASSUME_NONNULL_BEGIN
- (BOOL)areSyncRegistrationsComplete;
- (NSString *)databaseFilePath;
- (NSString *)databaseFilePath_SHM;
- (NSString *)databaseFilePath_WAL;
- (void)resetStorage;

@ -79,6 +79,7 @@ extern NSString *const StorageIsReadyNotification;
+ (void)storeDatabaseCipherKeySpec:(NSData *)cipherKeySpecData;
- (void)logFileSizes;
@end

@ -557,6 +557,20 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
return @"";
}
- (NSString *)databaseFilePath_SHM
{
OWS_ABSTRACT_METHOD();
return @"";
}
- (NSString *)databaseFilePath_WAL
{
OWS_ABSTRACT_METHOD();
return @"";
}
#pragma mark - Keychain
+ (BOOL)isDatabasePasswordAccessible
@ -725,6 +739,13 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
}
}
- (void)logFileSizes
{
DDLogInfo(@"%@ Database file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath]);
DDLogInfo(@"%@ \t SHM file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM]);
DDLogInfo(@"%@ \t WAL file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL]);
}
@end
NS_ASSUME_NONNULL_END

Loading…
Cancel
Save