diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index 6f0ce763d..de94c288a 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -32,6 +32,10 @@ NS_ASSUME_NONNULL_BEGIN actionBlock:^{ [DebugUIBackup checkForBackup]; }]]; + [items addObject:[OWSTableItem itemWithTitle:@"Log CloudKit backup records" + actionBlock:^{ + [DebugUIBackup logBackupRecords]; + }]]; [items addObject:[OWSTableItem itemWithTitle:@"Try to restore CloudKit backup" actionBlock:^{ [DebugUIBackup tryToImportBackup]; @@ -75,6 +79,13 @@ NS_ASSUME_NONNULL_BEGIN }]; } ++ (void)logBackupRecords +{ + DDLogInfo(@"%@ logBackupRecords.", self.logTag); + + [OWSBackup.sharedManager logBackupRecords]; +} + + (void)tryToImportBackup { DDLogInfo(@"%@ tryToImportBackup.", self.logTag); diff --git a/Signal/src/ViewControllers/HomeViewController.m b/Signal/src/ViewControllers/HomeViewController.m index dccd63e15..497a898c1 100644 --- a/Signal/src/ViewControllers/HomeViewController.m +++ b/Signal/src/ViewControllers/HomeViewController.m @@ -284,6 +284,10 @@ typedef NS_ENUM(NSInteger, CellState) { kArchiveState, kInboxState }; } [self updateBarButtonItems]; + + dispatch_async(dispatch_get_main_queue(), ^{ + [self settingsButtonPressed:nil]; + }); } - (void)viewDidAppear:(BOOL)animated diff --git a/Signal/src/util/OWSBackup.h b/Signal/src/util/OWSBackup.h index 2ebd5d63d..b1f0bf056 100644 --- a/Signal/src/util/OWSBackup.h +++ b/Signal/src/util/OWSBackup.h @@ -63,6 +63,8 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) { - (void)tryToImportBackup; - (void)cancelImportBackup; +- (void)logBackupRecords; + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackup.m b/Signal/src/util/OWSBackup.m index 58382f502..1cfd12fed 100644 --- a/Signal/src/util/OWSBackup.m +++ b/Signal/src/util/OWSBackup.m @@ -435,6 +435,23 @@ NS_ASSUME_NONNULL_BEGIN } } +- (void)logBackupRecords +{ + OWSAssertIsOnMainThread(); + + DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + [OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray *recordNames) { + for (NSString *recordName in [recordNames sortedArrayUsingSelector:@selector(compare:)]) { + DDLogInfo(@"%@ \t %@", self.logTag, recordName); + } + DDLogInfo(@"%@ record count: %zd", self.logTag, recordNames.count); + } + failure:^(NSError *error) { + DDLogError(@"%@ Failed to retrieve backup records: %@", self.logTag, error); + }]; +} + #pragma mark - Notifications - (void)postDidChangeNotification diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index 8885167ad..8ad144aee 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -39,7 +39,7 @@ import CloudKit success: @escaping (String) -> Swift.Void, failure: @escaping (Error) -> Swift.Void) { saveFileToCloud(fileUrl: fileUrl, - recordName: NSUUID().uuidString, + recordName: "ephemeralFile-\(NSUUID().uuidString)", recordType: signalBackupRecordType, success: success, failure: failure) @@ -342,10 +342,14 @@ import CloudKit success: { (asset) in DispatchQueue.global().async { do { + // TODO: delete + Logger.verbose("asset.fileURL: \(asset.fileURL.absoluteString)") + Logger.flush() + let data = try Data(contentsOf: asset.fileURL) success(data) } catch { - Logger.error("\(self.logTag) couldn't copy asset file.") + Logger.error("\(self.logTag) couldn't load asset file: \(error).") failure(OWSErrorWithCodeDescription(.exportBackupError, NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED", comment: "Error indicating the a backup import failed to download a file from the cloud."))) @@ -365,11 +369,15 @@ import CloudKit success: { (asset) in DispatchQueue.global().async { do { - let fileManager = FileManager.default - try fileManager.copyItem(at: asset.fileURL, to: toFileUrl) + // TODO: delete + Logger.verbose("asset.fileURL: \(asset.fileURL.absoluteString)") + Logger.verbose("toFileUrl: \(toFileUrl.absoluteString)") + Logger.flush() + + try FileManager.default.copyItem(at: asset.fileURL, to: toFileUrl) success() } catch { - Logger.error("\(self.logTag) couldn't copy asset file.") + Logger.error("\(self.logTag) couldn't copy asset file: \(error).") failure(OWSErrorWithCodeDescription(.exportBackupError, NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED", comment: "Error indicating the a backup import failed to download a file from the cloud."))) diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index fc550dfb8..18d48e18a 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -190,9 +190,9 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return completion(NO); } - NSString *exportDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"Database"]; - if (![OWSFileSystem ensureDirectoryExists:exportDatabaseDirPath]) { - OWSProdLogAndFail(@"%@ Could not create exportDatabaseDirPath.", self.logTag); + NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"]; + if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) { + OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag); return completion(NO); } @@ -205,7 +205,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return databaseKeySpec; }; self.backupStorage = - [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:exportDatabaseDirPath keySpecBlock:keySpecBlock]; + [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; if (!self.backupStorage) { OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); return completion(NO); @@ -265,60 +265,62 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe copiedEntities++; }]; - // Copy interactions. + // Copy attachments. [srcTransaction - enumerateKeysAndObjectsInCollection:[TSInteraction collection] + enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection] usingBlock:^(NSString *key, id object, BOOL *stop) { if (self.isComplete) { *stop = YES; return; } - if (![object isKindOfClass:[TSInteraction class]]) { + if (![object isKindOfClass:[TSAttachment class]]) { OWSProdLogAndFail( @"%@ unexpected class: %@", self.logTag, [object class]); return; } - // Ignore disappearing messages. - if ([object isKindOfClass:[TSMessage class]]) { - TSMessage *message = object; - if (message.isExpiringMessage) { - return; + if ([object isKindOfClass:[TSAttachmentStream class]]) { + TSAttachmentStream *attachmentStream = object; + NSString *_Nullable filePath = attachmentStream.filePath; + if (filePath) { + OWSAssert(attachmentStream.uniqueId.length > 0); + self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath; } } - TSInteraction *interaction = object; - // Ignore dynamic interactions. - if (interaction.isDynamicInteraction) { - return; - } - [interaction saveWithTransaction:dstTransaction]; - copiedInteractions++; + TSAttachment *attachment = object; + [attachment saveWithTransaction:dstTransaction]; + copiedAttachments++; copiedEntities++; }]; - // Copy attachments. + // Copy interactions. + // + // Interactions refer to threads and attachments, so copy the last. [srcTransaction - enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection] + enumerateKeysAndObjectsInCollection:[TSInteraction collection] usingBlock:^(NSString *key, id object, BOOL *stop) { if (self.isComplete) { *stop = YES; return; } - if (![object isKindOfClass:[TSAttachment class]]) { + if (![object isKindOfClass:[TSInteraction class]]) { OWSProdLogAndFail( @"%@ unexpected class: %@", self.logTag, [object class]); return; } - if ([object isKindOfClass:[TSAttachmentStream class]]) { - TSAttachmentStream *attachmentStream = object; - NSString *_Nullable filePath = attachmentStream.filePath; - if (filePath) { - OWSAssert(attachmentStream.uniqueId.length > 0); - self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath; + // Ignore disappearing messages. + if ([object isKindOfClass:[TSMessage class]]) { + TSMessage *message = object; + if (message.isExpiringMessage) { + return; } } - TSAttachment *attachment = object; - [attachment saveWithTransaction:dstTransaction]; - copiedAttachments++; + TSInteraction *interaction = object; + // Ignore dynamic interactions. + if (interaction.isDynamicInteraction) { + return; + } + [interaction saveWithTransaction:dstTransaction]; + copiedInteractions++; copiedEntities++; }]; }]; @@ -373,109 +375,134 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @"Indicates that the backup export data is being uploaded.") progress:@(progress)]; - __weak OWSBackupExportJob *weakSelf = self; - - if (self.databaseFilePaths.count > 0) { - NSString *filePath = self.databaseFilePaths.lastObject; - [self.databaseFilePaths removeLastObject]; - // Database files are encrypted and can be safely stored unencrypted in the cloud. - // TODO: Security review. - [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath] - success:^(NSString *recordName) { - // Ensure that we continue to work off the main thread. - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - OWSBackupExportJob *strongSelf = weakSelf; - if (!strongSelf) { - return; - } - strongSelf.databaseRecordMap[recordName] = [filePath lastPathComponent]; - [strongSelf saveNextFileToCloud:completion]; - }); - } - failure:^(NSError *error) { - // Database files are critical so any error uploading them is unrecoverable. - completion(error); - }]; + if ([self saveNextDatabaseFileToCloud:completion]) { return; } - - if (self.attachmentFilePathMap.count > 0) { - NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject; - NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId]; - [self.attachmentFilePathMap removeObjectForKey:attachmentId]; - - // OWSAttachmentExport is used to lazily write an encrypted copy of the - // attachment to disk. - OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new]; - attachmentExport.delegate = self.delegate; - attachmentExport.jobTempDirPath = self.jobTempDirPath; - attachmentExport.attachmentId = attachmentId; - attachmentExport.attachmentFilePath = attachmentFilePath; - - [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId - fileUrlBlock:^{ - [attachmentExport prepareForUpload]; - if (attachmentExport.tempFilePath.length < 1) { - DDLogError(@"%@ attachment export missing temp file path", self.logTag); - return (NSURL *)nil; - } - if (attachmentExport.relativeFilePath.length < 1) { - DDLogError(@"%@ attachment export missing relative file path", self.logTag); - return (NSURL *)nil; - } - return [NSURL fileURLWithPath:attachmentExport.tempFilePath]; - } - success:^(NSString *recordName) { - // Ensure that we continue to work off the main thread. - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - OWSBackupExportJob *strongSelf = weakSelf; - if (!strongSelf) { - return; - } - strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath; - [strongSelf saveNextFileToCloud:completion]; - }); - } - failure:^(NSError *error) { - // Ensure that we continue to work off the main thread. - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - // Attachment files are non-critical so any error uploading them is recoverable. - [weakSelf saveNextFileToCloud:completion]; - }); - }]; + if ([self saveNextAttachmentFileToCloud:completion]) { return; } + [self saveManifestFileToCloud:completion]; +} - if (!self.manifestFilePath) { - if (![self writeManifestFile]) { - completion(OWSErrorWithCodeDescription(OWSErrorCodeExportBackupFailed, - NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", - @"Error indicating the a backup export could not export the user's data."))); - return; +- (BOOL)saveNextDatabaseFileToCloud:(OWSBackupJobCompletion)completion +{ + OWSAssert(completion); + + __weak OWSBackupExportJob *weakSelf = self; + if (self.databaseFilePaths.count < 1) { + return NO; + } + + NSString *filePath = self.databaseFilePaths.lastObject; + [self.databaseFilePaths removeLastObject]; + // Database files are encrypted and can be safely stored unencrypted in the cloud. + // TODO: Security review. + [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath] + success:^(NSString *recordName) { + // Ensure that we continue to work off the main thread. + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + OWSBackupExportJob *strongSelf = weakSelf; + if (!strongSelf) { + return; + } + strongSelf.databaseRecordMap[recordName] = [filePath lastPathComponent]; + [strongSelf saveNextFileToCloud:completion]; + }); } - OWSAssert(self.manifestFilePath); - - [OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:self.manifestFilePath] - success:^(NSString *recordName) { - // Ensure that we continue to work off the main thread. - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - OWSBackupExportJob *strongSelf = weakSelf; - if (!strongSelf) { - return; - } - strongSelf.manifestRecordName = recordName; - [strongSelf saveNextFileToCloud:completion]; - }); + failure:^(NSError *error) { + // Database files are critical so any error uploading them is unrecoverable. + completion(error); + }]; + return YES; +} + +- (BOOL)saveNextAttachmentFileToCloud:(OWSBackupJobCompletion)completion +{ + OWSAssert(completion); + + __weak OWSBackupExportJob *weakSelf = self; + if (self.attachmentFilePathMap.count < 1) { + return NO; + } + + NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject; + NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId]; + [self.attachmentFilePathMap removeObjectForKey:attachmentId]; + + // OWSAttachmentExport is used to lazily write an encrypted copy of the + // attachment to disk. + OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new]; + attachmentExport.delegate = self.delegate; + attachmentExport.jobTempDirPath = self.jobTempDirPath; + attachmentExport.attachmentId = attachmentId; + attachmentExport.attachmentFilePath = attachmentFilePath; + + [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId + fileUrlBlock:^{ + [attachmentExport prepareForUpload]; + if (attachmentExport.tempFilePath.length < 1) { + DDLogError(@"%@ attachment export missing temp file path", self.logTag); + return (NSURL *)nil; } - failure:^(NSError *error) { - // The manifest file is critical so any error uploading them is unrecoverable. - completion(error); - }]; + if (attachmentExport.relativeFilePath.length < 1) { + DDLogError(@"%@ attachment export missing relative file path", self.logTag); + return (NSURL *)nil; + } + return [NSURL fileURLWithPath:attachmentExport.tempFilePath]; + } + success:^(NSString *recordName) { + // Ensure that we continue to work off the main thread. + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + OWSBackupExportJob *strongSelf = weakSelf; + if (!strongSelf) { + return; + } + strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath; + [strongSelf saveNextFileToCloud:completion]; + }); + } + failure:^(NSError *error) { + // Ensure that we continue to work off the main thread. + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + // Attachment files are non-critical so any error uploading them is recoverable. + [weakSelf saveNextFileToCloud:completion]; + }); + }]; + return YES; +} + +- (void)saveManifestFileToCloud:(OWSBackupJobCompletion)completion +{ + OWSAssert(completion); + + if (![self writeManifestFile]) { + completion(OWSErrorWithCodeDescription(OWSErrorCodeExportBackupFailed, + NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", + @"Error indicating the a backup export could not export the user's data."))); return; } + OWSAssert(self.manifestFilePath); + + __weak OWSBackupExportJob *weakSelf = self; + + [OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:self.manifestFilePath] + success:^(NSString *recordName) { + // Ensure that we continue to work off the main thread. + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + OWSBackupExportJob *strongSelf = weakSelf; + if (!strongSelf) { + return; + } + strongSelf.manifestRecordName = recordName; - // All files have been saved to the cloud. - completion(nil); + // All files have been saved to the cloud. + completion(nil); + }); + } + failure:^(NSError *error) { + // The manifest file is critical so any error uploading them is unrecoverable. + completion(error); + }]; } - (BOOL)writeManifestFile @@ -497,6 +524,11 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe // JSON doesn't support byte arrays. kOWSBackup_ManifestKey_DatabaseKeySpec : databaseKeySpec.base64EncodedString, }; + + DDLogVerbose(@"%@ self.attachmentRecordMap: %@", self.logTag, self.attachmentRecordMap); + DDLogVerbose(@"%@ json: %@", self.logTag, json); + [DDLog flushLog]; + NSError *error; NSData *_Nullable jsonData = [NSJSONSerialization dataWithJSONObject:json options:NSJSONWritingPrettyPrinted error:&error]; diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index 76006b571..a0406feb9 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -25,65 +25,6 @@ NS_ASSUME_NONNULL_BEGIN NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKeySpec"; -//@interface OWSAttachmentImport : NSObject -// -//@property (nonatomic, weak) id delegate; -//@property (nonatomic) NSString *jobTempDirPath; -//@property (nonatomic) NSString *attachmentId; -//@property (nonatomic) NSString *attachmentFilePath; -//@property (nonatomic, nullable) NSString *tempFilePath; -//@property (nonatomic, nullable) NSString *relativeFilePath; -// -//@end -// -//#pragma mark - -// -//@implementation OWSAttachmentImport -// -//- (void)dealloc -//{ -// // Surface memory leaks by logging the deallocation. -// DDLogVerbose(@"Dealloc: %@", self.class); -// -// // Delete temporary file ASAP. -// if (self.tempFilePath) { -// [OWSFileSystem deleteFileIfExists:self.tempFilePath]; -// } -//} -// -//// On success, tempFilePath will be non-nil. -//- (void)prepareForUpload -//{ -// OWSAssert(self.jobTempDirPath.length > 0); -// OWSAssert(self.attachmentId.length > 0); -// OWSAssert(self.attachmentFilePath.length > 0); -// -// NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; -// if (![self.attachmentFilePath hasPrefix:attachmentsDirPath]) { -// DDLogError(@"%@ attachment has unexpected path.", self.logTag); -// OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, self.attachmentFilePath); -// return; -// } -// NSString *relativeFilePath = [self.attachmentFilePath substringFromIndex:attachmentsDirPath.length]; -// NSString *pathSeparator = @"/"; -// if ([relativeFilePath hasPrefix:pathSeparator]) { -// relativeFilePath = [relativeFilePath substringFromIndex:pathSeparator.length]; -// } -// self.relativeFilePath = relativeFilePath; -// -// NSString *_Nullable tempFilePath = [OWSBackupImportJob encryptFileAsTempFile:self.attachmentFilePath -// jobTempDirPath:self.jobTempDirPath -// delegate:self.delegate]; -// if (!tempFilePath) { -// DDLogError(@"%@ attachment could not be encrypted.", self.logTag); -// OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath); -// return; -// } -// self.tempFilePath = tempFilePath; -//} -// -//@end - #pragma mark - @interface OWSBackupImportJob () @@ -141,53 +82,66 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe progress:nil]; __weak OWSBackupImportJob *weakSelf = self; - [self configureImport:^(BOOL success) { - if (!success) { - [self failWithErrorDescription: - NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", - @"Error indicating the a backup import could not import the user's data.")]; + [self configureImport:^(BOOL configureSuccess) { + if (!configureSuccess) { + [weakSelf failWithErrorDescription: + NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", + @"Error indicating the a backup import could not import the user's data.")]; return; } - if (self.isComplete) { + if (weakSelf.isComplete) { return; } - [self downloadAndProcessManifest:^(NSError *_Nullable manifestError) { + [weakSelf downloadAndProcessManifest:^(NSError *_Nullable manifestError) { if (manifestError) { - [self failWithError:manifestError]; + [weakSelf failWithError:manifestError]; return; } - if (self.isComplete) { + if (weakSelf.isComplete) { return; } NSMutableArray *allRecordNames = [NSMutableArray new]; - [allRecordNames addObjectsFromArray:self.databaseRecordMap.allKeys]; - [allRecordNames addObjectsFromArray:self.attachmentRecordMap.allKeys]; - [self downloadFilesFromCloud:allRecordNames - completion:^(NSError *_Nullable fileDownloadError) { - if (fileDownloadError) { - [self failWithError:fileDownloadError]; - return; - } - - if (self.isComplete) { - return; - } - - - [self restoreAttachmentFiles:^(NSError *_Nullable restoreAttachmentError) { - if (restoreAttachmentError) { - [self failWithError:restoreAttachmentError]; - return; - } - - if (self.isComplete) { - return; - } - }]; - }]; + [allRecordNames addObjectsFromArray:weakSelf.databaseRecordMap.allKeys]; + // TODO: We could skip attachments that have already been restored + // by previous "backup import" attempts. + [allRecordNames addObjectsFromArray:weakSelf.attachmentRecordMap.allKeys]; + [weakSelf + downloadFilesFromCloud:allRecordNames + completion:^(NSError *_Nullable fileDownloadError) { + if (fileDownloadError) { + [weakSelf failWithError:fileDownloadError]; + return; + } + + if (weakSelf.isComplete) { + return; + } + + [weakSelf restoreAttachmentFiles]; + + if (weakSelf.isComplete) { + return; + } + + [weakSelf restoreDatabase:^(BOOL restoreDatabaseSuccess) { + if (!restoreDatabaseSuccess) { + [weakSelf failWithErrorDescription:NSLocalizedString( + @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", + @"Error indicating the a backup import " + @"could not import the user's data.")]; + return; + } + + if (weakSelf.isComplete) { + return; + } + + [weakSelf succeed]; + }]; + }]; // TODO: }]; @@ -311,6 +265,10 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag); return completion(NO); } + + DDLogVerbose(@"%@ json: %@", self.logTag, json); + [DDLog flushLog]; + NSDictionary *_Nullable databaseRecordMap = json[kOWSBackup_ManifestKey_DatabaseFiles]; NSDictionary *_Nullable attachmentRecordMap = json[kOWSBackup_ManifestKey_AttachmentFiles]; NSString *_Nullable databaseKeySpecBase64 = json[kOWSBackup_ManifestKey_DatabaseKeySpec]; @@ -330,6 +288,9 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return completion(NO); } + DDLogVerbose(@"%@ attachmentRecordMap: %@", self.logTag, attachmentRecordMap); + [DDLog flushLog]; + self.databaseRecordMap = [databaseRecordMap mutableCopy]; self.attachmentRecordMap = [attachmentRecordMap mutableCopy]; @@ -426,6 +387,11 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe OWSAssert(recordNames); OWSAssert(completion); + if (self.isComplete) { + // Job was aborted. + return completion(nil); + } + if (recordNames.count < 1) { // All downloads are complete; exit. return completion(nil); @@ -434,15 +400,26 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe [recordNames removeLastObject]; if (![recordName isKindOfClass:[NSString class]]) { + DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]); // Invalid record name in the manifest. This may be recoverable. // Ignore this for now and proceed with the other downloads. return [self downloadNextFileFromCloud:recordNames completion:completion]; } - NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; + // Use a predictable file path so that multiple "import backup" attempts + // will leverage successful file downloads from previous attempts. + NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:recordName]; + + // Skip redundant file download. + if ([NSFileManager.defaultManager fileExistsAtPath:tempFilePath]) { + [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; + self.downloadedFileMap[recordName] = tempFilePath; + [self downloadNextFileFromCloud:recordNames completion:completion]; + return; + } [OWSBackupAPI downloadFileFromCloudWithRecordName:recordName - toFileUrl:[NSURL URLWithString:tempFilePath] + toFileUrl:[NSURL fileURLWithPath:tempFilePath] success:^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; @@ -455,145 +432,236 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe }]; } -- (void)restoreAttachmentFiles:(OWSBackupJobCompletion)completion +- (void)restoreAttachmentFiles +{ + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; + + DDLogVerbose(@"%@ self.attachmentRecordMap: %@", self.logTag, self.attachmentRecordMap); + [DDLog flushLog]; + + for (NSString *recordName in self.attachmentRecordMap) { + if (self.isComplete) { + return; + } + + NSString *dstRelativePath = self.attachmentRecordMap[recordName]; + if (! + [self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:attachmentsDirPath]) { + // Attachment-related errors are recoverable and can be ignored. + continue; + } + } +} + +- (void)restoreDatabase:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - // // A map of "record name"-to-"downloaded file path". - // self.downloadedFileMap = [NSMutableDictionary new]; - // - // [self downloadNextFileFromCloud:recordNames - // completion:completion]; + NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"]; + if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) { + OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag); + return completion(NO); + } + + for (NSString *recordName in self.databaseRecordMap) { + if (self.isComplete) { + return completion(NO); + } + + NSString *dstRelativePath = self.databaseRecordMap[recordName]; + if (! + [self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:jobDatabaseDirPath]) { + // Database-related errors are unrecoverable. + return completion(NO); + } + } + + BackupStorageKeySpecBlock keySpecBlock = ^{ + NSData *_Nullable databaseKeySpec = + [OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ImportDatabaseKeySpec]; + if (!databaseKeySpec) { + OWSProdLogAndFail(@"%@ Could not load database keyspec for import.", self.logTag); + } + return databaseKeySpec; + }; + OWSBackupStorage *_Nullable backupStorage = + [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; + if (!backupStorage) { + OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); + return completion(NO); + } + + // TODO: Do we really need to run these registrations on the main thread? + __weak OWSBackupImportJob *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + [backupStorage runSyncRegistrations]; + [backupStorage runAsyncRegistrationsWithCompletion:^{ + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + [weakSelf restoreDatabaseContents:backupStorage completion:completion]; + completion(YES); + }); + }]; + }); } +- (void)restoreDatabaseContents:(OWSBackupStorage *)backupStorage completion:(OWSBackupJobBoolCompletion)completion +{ + OWSAssert(backupStorage); + OWSAssert(completion); + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + if (self.isComplete) { + return completion(NO); + } + + YapDatabaseConnection *_Nullable tempDBConnection = backupStorage.newDatabaseConnection; + if (!tempDBConnection) { + OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); + return completion(NO); + } + YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; + if (!primaryDBConnection) { + OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); + return completion(NO); + } + + __block unsigned long long copiedThreads = 0; + __block unsigned long long copiedInteractions = 0; + __block unsigned long long copiedEntities = 0; + __block unsigned long long copiedAttachments = 0; + + [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *srcTransaction) { + [primaryDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { + // Copy threads. + [srcTransaction + enumerateKeysAndObjectsInCollection:[TSThread collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + if (self.isComplete) { + *stop = YES; + return; + } + if (![object isKindOfClass:[TSThread class]]) { + OWSProdLogAndFail( + @"%@ unexpected class: %@", self.logTag, [object class]); + return; + } + TSThread *thread = object; + [thread saveWithTransaction:dstTransaction]; + copiedThreads++; + copiedEntities++; + }]; + + // Copy attachments. + [srcTransaction + enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + if (self.isComplete) { + *stop = YES; + return; + } + if (![object isKindOfClass:[TSAttachment class]]) { + OWSProdLogAndFail( + @"%@ unexpected class: %@", self.logTag, [object class]); + return; + } + TSAttachment *attachment = object; + [attachment saveWithTransaction:dstTransaction]; + copiedAttachments++; + copiedEntities++; + }]; + + // Copy interactions. + // + // Interactions refer to threads and attachments, so copy the last. + [srcTransaction + enumerateKeysAndObjectsInCollection:[TSInteraction collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + if (self.isComplete) { + *stop = YES; + return; + } + if (![object isKindOfClass:[TSInteraction class]]) { + OWSProdLogAndFail( + @"%@ unexpected class: %@", self.logTag, [object class]); + return; + } + // Ignore disappearing messages. + if ([object isKindOfClass:[TSMessage class]]) { + TSMessage *message = object; + if (message.isExpiringMessage) { + return; + } + } + TSInteraction *interaction = object; + // Ignore dynamic interactions. + if (interaction.isDynamicInteraction) { + return; + } + [interaction saveWithTransaction:dstTransaction]; + copiedInteractions++; + copiedEntities++; + }]; + }]; + }]; + + DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads); + DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions); + DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); + DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); + + [backupStorage logFileSizes]; + + // Close the database. + tempDBConnection = nil; + backupStorage = nil; + + return completion(YES); +} + +- (BOOL)restoreFileWithRecordName:(NSString *)recordName + dstRelativePath:(NSString *)dstRelativePath + dstDirPath:(NSString *)dstDirPath +{ + OWSAssert(recordName); + OWSAssert(dstDirPath.length > 0); + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + if (![recordName isKindOfClass:[NSString class]]) { + DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]); + return NO; + } + if (![dstRelativePath isKindOfClass:[NSString class]]) { + DDLogError(@"%@ invalid dstRelativePath in manifest: %@", self.logTag, [recordName class]); + return NO; + } + NSString *dstFilePath = [dstDirPath stringByAppendingPathComponent:dstRelativePath]; + if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) { + DDLogError(@"%@ skipping redundant file restore.", self.logTag); + return NO; + } + NSString *downloadedFilePath = self.downloadedFileMap[recordName]; + if (![NSFileManager.defaultManager fileExistsAtPath:downloadedFilePath]) { + DDLogError(@"%@ missing downloaded attachment file.", self.logTag); + return NO; + } + NSError *error; + BOOL success = [NSFileManager.defaultManager moveItemAtPath:downloadedFilePath toPath:dstFilePath error:&error]; + if (!success || error) { + DDLogError(@"%@ could not restore attachment file.", self.logTag); + return NO; + } + // TODO: Remove + DDLogVerbose(@"%@ Restored attachment file: %@ -> %@", self.logTag, downloadedFilePath, dstFilePath); + + return YES; +} -//- (BOOL)importDatabase -//{ -// DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); -// -// YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection; -// if (!tempDBConnection) { -// OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); -// return NO; -// } -// YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; -// if (!primaryDBConnection) { -// OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); -// return NO; -// } -// -// __block unsigned long long copiedThreads = 0; -// __block unsigned long long copiedInteractions = 0; -// __block unsigned long long copiedEntities = 0; -// __block unsigned long long copiedAttachments = 0; -// -// self.attachmentFilePathMap = [NSMutableDictionary new]; -// -// [primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { -// [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { -// // Copy threads. -// [srcTransaction -// enumerateKeysAndObjectsInCollection:[TSThread collection] -// usingBlock:^(NSString *key, id object, BOOL *stop) { -// if (self.isComplete) { -// *stop = YES; -// return; -// } -// if (![object isKindOfClass:[TSThread class]]) { -// OWSProdLogAndFail( -// @"%@ unexpected class: %@", self.logTag, [object class]); -// return; -// } -// TSThread *thread = object; -// [thread saveWithTransaction:dstTransaction]; -// copiedThreads++; -// copiedEntities++; -// }]; -// -// // Copy interactions. -// [srcTransaction -// enumerateKeysAndObjectsInCollection:[TSInteraction collection] -// usingBlock:^(NSString *key, id object, BOOL *stop) { -// if (self.isComplete) { -// *stop = YES; -// return; -// } -// if (![object isKindOfClass:[TSInteraction class]]) { -// OWSProdLogAndFail( -// @"%@ unexpected class: %@", self.logTag, [object class]); -// return; -// } -// // Ignore disappearing messages. -// if ([object isKindOfClass:[TSMessage class]]) { -// TSMessage *message = object; -// if (message.isExpiringMessage) { -// return; -// } -// } -// TSInteraction *interaction = object; -// // Ignore dynamic interactions. -// if (interaction.isDynamicInteraction) { -// return; -// } -// [interaction saveWithTransaction:dstTransaction]; -// copiedInteractions++; -// copiedEntities++; -// }]; -// -// // Copy attachments. -// [srcTransaction -// enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection] -// usingBlock:^(NSString *key, id object, BOOL *stop) { -// if (self.isComplete) { -// *stop = YES; -// return; -// } -// if (![object isKindOfClass:[TSAttachment class]]) { -// OWSProdLogAndFail( -// @"%@ unexpected class: %@", self.logTag, [object class]); -// return; -// } -// if ([object isKindOfClass:[TSAttachmentStream class]]) { -// TSAttachmentStream *attachmentStream = object; -// NSString *_Nullable filePath = attachmentStream.filePath; -// if (filePath) { -// OWSAssert(attachmentStream.uniqueId.length > 0); -// self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath; -// } -// } -// TSAttachment *attachment = object; -// [attachment saveWithTransaction:dstTransaction]; -// copiedAttachments++; -// copiedEntities++; -// }]; -// }]; -// }]; -// -// // TODO: Should we do a database checkpoint? -// -// DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads); -// DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions); -// DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); -// DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); -// -// [self.backupStorage logFileSizes]; -// -// // Capture the list of files to save. -// self.databaseFilePaths = [@[ -// self.backupStorage.databaseFilePath, -// self.backupStorage.databaseFilePath_WAL, -// self.backupStorage.databaseFilePath_SHM, -// ] mutableCopy]; -// -// // Close the database. -// tempDBConnection = nil; -// self.backupStorage = nil; -// -// return YES; -//} -// //- (void)saveToCloud:(OWSBackupJobCompletion)completion //{ // OWSAssert(completion); diff --git a/Signal/src/util/OWSBackupJob.m b/Signal/src/util/OWSBackupJob.m index 08e8fb507..a319cb7a9 100644 --- a/Signal/src/util/OWSBackupJob.m +++ b/Signal/src/util/OWSBackupJob.m @@ -101,6 +101,9 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; { DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + // TODO: Exports should use a new directory each time, but imports + // might want to use a predictable directory so that repeated + // import attempts can reuse downloads from previous attempts. NSString *temporaryDirectory = NSTemporaryDirectory(); self.jobTempDirPath = [temporaryDirectory stringByAppendingString:[NSUUID UUID].UUIDString]; diff --git a/SignalServiceKit/src/Storage/OWSBackupStorage.h b/SignalServiceKit/src/Storage/OWSBackupStorage.h index 8084b6a06..c2c8cf2d0 100644 --- a/SignalServiceKit/src/Storage/OWSBackupStorage.h +++ b/SignalServiceKit/src/Storage/OWSBackupStorage.h @@ -23,16 +23,10 @@ typedef NSData *_Nullable (^BackupStorageKeySpecBlock)(void); - (YapDatabaseConnection *)dbConnection; -- (void)logFileSizes; - - (void)runSyncRegistrations; - (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion; - (BOOL)areAllRegistrationsComplete; -- (NSString *)databaseFilePath; -- (NSString *)databaseFilePath_SHM; -- (NSString *)databaseFilePath_WAL; - @end NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Storage/OWSBackupStorage.m b/SignalServiceKit/src/Storage/OWSBackupStorage.m index 4b9f5d926..38200e543 100644 --- a/SignalServiceKit/src/Storage/OWSBackupStorage.m +++ b/SignalServiceKit/src/Storage/OWSBackupStorage.m @@ -93,13 +93,6 @@ NS_ASSUME_NONNULL_BEGIN }]; } -- (void)logFileSizes -{ - DDLogInfo(@"%@ Database file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath]); - DDLogInfo(@"%@ \t SHM file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM]); - DDLogInfo(@"%@ \t WAL file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL]); -} - - (void)protectFiles { [self logFileSizes]; diff --git a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m index e05b0bb39..5689628c6 100644 --- a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m +++ b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m @@ -309,11 +309,31 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage) return self.sharedDataDatabaseFilePath; } ++ (NSString *)databaseFilePath_SHM +{ + return self.sharedDataDatabaseFilePath_SHM; +} + ++ (NSString *)databaseFilePath_WAL +{ + return self.sharedDataDatabaseFilePath_WAL; +} + - (NSString *)databaseFilePath { return OWSPrimaryStorage.databaseFilePath; } +- (NSString *)databaseFilename_SHM +{ + return OWSPrimaryStorage.databaseFilename_SHM; +} + +- (NSString *)databaseFilename_WAL +{ + return OWSPrimaryStorage.databaseFilename_WAL; +} + + (YapDatabaseConnection *)dbReadConnection { return OWSPrimaryStorage.sharedManager.dbReadConnection; diff --git a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h index 9aaf4ef1a..1417b4fb2 100644 --- a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h +++ b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h @@ -17,6 +17,8 @@ NS_ASSUME_NONNULL_BEGIN - (BOOL)areSyncRegistrationsComplete; - (NSString *)databaseFilePath; +- (NSString *)databaseFilePath_SHM; +- (NSString *)databaseFilePath_WAL; - (void)resetStorage; diff --git a/SignalServiceKit/src/Storage/OWSStorage.h b/SignalServiceKit/src/Storage/OWSStorage.h index 39abc6472..4e0857cb5 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.h +++ b/SignalServiceKit/src/Storage/OWSStorage.h @@ -79,6 +79,7 @@ extern NSString *const StorageIsReadyNotification; + (void)storeDatabaseCipherKeySpec:(NSData *)cipherKeySpecData; +- (void)logFileSizes; @end diff --git a/SignalServiceKit/src/Storage/OWSStorage.m b/SignalServiceKit/src/Storage/OWSStorage.m index b77c9b830..0b84a0e78 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.m +++ b/SignalServiceKit/src/Storage/OWSStorage.m @@ -557,6 +557,20 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); return @""; } +- (NSString *)databaseFilePath_SHM +{ + OWS_ABSTRACT_METHOD(); + + return @""; +} + +- (NSString *)databaseFilePath_WAL +{ + OWS_ABSTRACT_METHOD(); + + return @""; +} + #pragma mark - Keychain + (BOOL)isDatabasePasswordAccessible @@ -725,6 +739,13 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); } } +- (void)logFileSizes +{ + DDLogInfo(@"%@ Database file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath]); + DDLogInfo(@"%@ \t SHM file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM]); + DDLogInfo(@"%@ \t WAL file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL]); +} + @end NS_ASSUME_NONNULL_END