Merge branch 'charlesmchen/incrementalBackup5'

Signal/src/ViewControllers/DebugUI/DebugUIBackup.m

@@ -44,6 +44,10 @@ NS_ASSUME_NONNULL_BEGIN
                                      actionBlock:^{
                                          [DebugUIBackup logDatabaseSizeStats];
                                      }]];
+    [items addObject:[OWSTableItem itemWithTitle:@"Clear All CloudKit Records"
+                                     actionBlock:^{
+                                         [DebugUIBackup clearAllCloudKitRecords];
+                                     }]];
 
     return [OWSTableSection sectionWithTitle:self.name items:items];
 }
@@ -150,6 +154,13 @@ NS_ASSUME_NONNULL_BEGIN
     }
 }
 
++ (void)clearAllCloudKitRecords
+{
+    DDLogInfo(@"%@ clearAllCloudKitRecords.", self.logTag);
+
+    [OWSBackup.sharedManager clearAllCloudKitRecords];
+}
+
 @end
 
 NS_ASSUME_NONNULL_END

Signal/src/util/OWSBackup.h

@@ -69,6 +69,7 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) {
 - (void)cancelImportBackup;
 
 - (void)logBackupRecords;
+- (void)clearAllCloudKitRecords;
 
 @end
 

Signal/src/util/OWSBackup.m

@@ -459,6 +459,30 @@ NS_ASSUME_NONNULL_BEGIN
         }];
 }
 
+- (void)clearAllCloudKitRecords
+{
+    OWSAssertIsOnMainThread();
+
+    DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
+
+    [OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray<NSString *> *recordNames) {
+        if (recordNames.count < 1) {
+            DDLogInfo(@"%@ No CloudKit records found to clear.", self.logTag);
+            return;
+        }
+        [OWSBackupAPI deleteRecordsFromCloudWithRecordNames:recordNames
+            success:^{
+                DDLogInfo(@"%@ Clear all CloudKit records succeeded.", self.logTag);
+            }
+            failure:^(NSError *error) {
+                DDLogError(@"%@ Clear all CloudKit records failed: %@.", self.logTag, error);
+            }];
+    }
+        failure:^(NSError *error) {
+            DDLogError(@"%@ Failed to retrieve CloudKit records: %@", self.logTag, error);
+        }];
+}
+
 #pragma mark - Notifications
 
 - (void)postDidChangeNotification

Signal/src/util/OWSBackupAPI.swift

@@ -59,6 +59,14 @@ import CloudKit
             failure: failure)
     }
 
+    // "Persistent" files may be shared between backup export; they should only be saved
+    // once.  For example, attachment files should only be uploaded once.  Subsequent
+    // backups can reuse the same record.
+    @objc
+    public class func recordNameForPersistentFile(fileId: String) -> String {
+        return "persistentFile-\(fileId)"
+    }
+
     // "Persistent" files may be shared between backup export; they should only be saved
     // once.  For example, attachment files should only be uploaded once.  Subsequent
     // backups can reuse the same record.
@@ -67,7 +75,7 @@ import CloudKit
                                                     fileUrlBlock: @escaping (()) -> URL?,
                                                     success: @escaping (String) -> Void,
                                                     failure: @escaping (Error) -> Void) {
-        saveFileOnceToCloud(recordName: "persistentFile-\(fileId)",
+        saveFileOnceToCloud(recordName: recordNameForPersistentFile(fileId: fileId),
             recordType: signalBackupRecordType,
             fileUrlBlock: fileUrlBlock,
             success: success,

Signal/src/util/OWSBackupExportJob.m

@@ -294,6 +294,10 @@ NS_ASSUME_NONNULL_BEGIN
 
 @property (nonatomic, nullable) OWSBackupExportItem *manifestItem;
 
+// If we are replacing an existing backup, we use some of its contents for continuity.
+@property (nonatomic, nullable) NSDictionary<NSString *, OWSBackupManifestItem *> *lastManifestItemMap;
+@property (nonatomic, nullable) NSSet<NSString *> *lastRecordNames;
+
 @end
 
 #pragma mark -
@@ -312,11 +316,15 @@ NS_ASSUME_NONNULL_BEGIN
 
     __weak OWSBackupExportJob *weakSelf = self;
     [OWSBackupAPI checkCloudKitAccessWithCompletion:^(BOOL hasAccess) {
-        if (hasAccess) {
+        dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
-            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+            if (hasAccess) {
                 [weakSelf start];
-            });
+            } else {
-        }
+                [weakSelf failWithErrorDescription:
+                              NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
+                                  @"Error indicating the a backup export could not export the user's data.")];
+            }
+        });
     }];
 }
 
@@ -338,29 +346,41 @@ NS_ASSUME_NONNULL_BEGIN
         if (self.isComplete) {
             return;
         }
-        [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_EXPORT",
+        [self tryToFetchManifestWithCompletion:^(BOOL tryToFetchManifestSuccess) {
-                                                @"Indicates that the backup export data is being exported.")
+            if (!tryToFetchManifestSuccess) {
-                                   progress:nil];
+                [self failWithErrorDescription:
-        if (![self exportDatabase]) {
+                          NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
-            [self failWithErrorDescription:
+                              @"Error indicating the a backup export could not export the user's data.")];
-                      NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
+                return;
-                          @"Error indicating the a backup export could not export the user's data.")];
+            }
-            return;
+
-        }
+            if (self.isComplete) {
-        if (self.isComplete) {
-            return;
-        }
-        [self saveToCloudWithCompletion:^(NSError *_Nullable saveError) {
-            if (saveError) {
-                [weakSelf failWithError:saveError];
                 return;
             }
-            [self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) {
+            [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_EXPORT",
-                if (cleanUpError) {
+                                                    @"Indicates that the backup export data is being exported.")
-                    [weakSelf failWithError:cleanUpError];
+                                       progress:nil];
+            if (![self exportDatabase]) {
+                [self failWithErrorDescription:
+                          NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
+                              @"Error indicating the a backup export could not export the user's data.")];
+                return;
+            }
+            if (self.isComplete) {
+                return;
+            }
+            [self saveToCloudWithCompletion:^(NSError *_Nullable saveError) {
+                if (saveError) {
+                    [weakSelf failWithError:saveError];
                     return;
                 }
-                [weakSelf succeed];
+                [self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) {
+                    if (cleanUpError) {
+                        [weakSelf failWithError:cleanUpError];
+                        return;
+                    }
+                    [weakSelf succeed];
+                }];
             }];
         }];
     }];
@@ -402,6 +422,111 @@ NS_ASSUME_NONNULL_BEGIN
         }];
 }
 
+- (void)tryToFetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion
+{
+    OWSAssert(completion);
+
+    if (self.isComplete) {
+        return;
+    }
+
+    DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
+
+    [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_CHECK_BACKUP",
+                                            @"Indicates that the backup import is checking for an existing backup.")
+                               progress:nil];
+
+    __weak OWSBackupExportJob *weakSelf = self;
+
+    [OWSBackupAPI checkForManifestInCloudWithSuccess:^(BOOL value) {
+        dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+            if (value) {
+                [weakSelf fetchManifestWithCompletion:completion];
+            } else {
+                // There is no existing manifest; continue.
+                completion(YES);
+            }
+        });
+    }
+        failure:^(NSError *error) {
+            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+                completion(NO);
+            });
+        }];
+}
+
+- (void)fetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion
+{
+    OWSAssert(completion);
+
+    if (self.isComplete) {
+        return;
+    }
+
+    DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
+
+    __weak OWSBackupExportJob *weakSelf = self;
+    [weakSelf downloadAndProcessManifestWithSuccess:^(OWSBackupManifestContents *manifest) {
+        OWSBackupExportJob *strongSelf = weakSelf;
+        if (!strongSelf) {
+            return;
+        }
+        if (strongSelf.isComplete) {
+            return;
+        }
+        OWSCAssert(manifest.databaseItems.count > 0);
+        OWSCAssert(manifest.attachmentsItems);
+        [strongSelf processLastManifest:manifest];
+        [strongSelf fetchAllRecordsWithCompletion:completion];
+    }
+        failure:^(NSError *manifestError) {
+            completion(NO);
+        }
+        backupIO:self.backupIO];
+}
+
+- (void)fetchAllRecordsWithCompletion:(OWSBackupJobBoolCompletion)completion
+{
+    OWSAssert(completion);
+
+    if (self.isComplete) {
+        return;
+    }
+
+    DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
+
+    __weak OWSBackupExportJob *weakSelf = self;
+    [OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray<NSString *> *recordNames) {
+        dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+            OWSBackupExportJob *strongSelf = weakSelf;
+            if (!strongSelf) {
+                return;
+            }
+            if (strongSelf.isComplete) {
+                return;
+            }
+            strongSelf.lastRecordNames = [NSSet setWithArray:recordNames];
+            completion(YES);
+        });
+    }
+        failure:^(NSError *error) {
+            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+                completion(NO);
+            });
+        }];
+}
+
+- (void)processLastManifest:(OWSBackupManifestContents *)manifest
+{
+    OWSAssert(manifest);
+
+    NSMutableDictionary<NSString *, OWSBackupManifestItem *> *lastManifestItemMap = [NSMutableDictionary new];
+    for (OWSBackupManifestItem *manifestItem in manifest.attachmentsItems) {
+        lastManifestItemMap[manifestItem.recordName] = manifestItem;
+    }
+    self.lastManifestItemMap = [lastManifestItemMap copy];
+}
+
 - (BOOL)exportDatabase
 {
     OWSAssert(self.backupIO);
@@ -657,9 +782,12 @@ NS_ASSUME_NONNULL_BEGIN
             });
         }
         failure:^(NSError *error) {
-            // Database files are critical so any error uploading them is unrecoverable.
+            // Ensure that we continue to work off the main thread.
-            DDLogVerbose(@"%@ error while saving file: %@", weakSelf.logTag, item.encryptedItem.filePath);
+            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
-            completion(error);
+                // Database files are critical so any error uploading them is unrecoverable.
+                DDLogVerbose(@"%@ error while saving file: %@", weakSelf.logTag, item.encryptedItem.filePath);
+                completion(error);
+            });
         }];
     return YES;
 }
@@ -678,6 +806,44 @@ NS_ASSUME_NONNULL_BEGIN
     OWSAttachmentExport *attachmentExport = self.unsavedAttachmentExports.lastObject;
     [self.unsavedAttachmentExports removeLastObject];
 
+    if (self.lastManifestItemMap && self.lastRecordNames) {
+        // Wherever possible, we do incremental backups and re-use fragments of the last backup.
+        // Recycling fragments doesn't just reduce redundant network activity,
+        // it allows us to skip the local export work, i.e. encryption.
+        // To do so, we must preserve the metadata for these fragments.
+        //
+        // We check two things:
+        //
+        // * That the "last known backup manifest" contains an item from which we can recover
+        //   this record's metadata.
+        // * That this record does in fact exist in our CloudKit database.
+        NSString *lastRecordName = [OWSBackupAPI recordNameForPersistentFileWithFileId:attachmentExport.attachmentId];
+        OWSBackupManifestItem *_Nullable lastManifestItem = self.lastManifestItemMap[lastRecordName];
+        if (lastManifestItem && [self.lastRecordNames containsObject:lastRecordName]) {
+            OWSAssert(lastManifestItem.encryptionKey.length > 0);
+            OWSAssert(lastManifestItem.relativeFilePath.length > 0);
+
+            // Recycle the metadata from the last backup's manifest.
+            OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new];
+            encryptedItem.encryptionKey = lastManifestItem.encryptionKey;
+            attachmentExport.encryptedItem = encryptedItem;
+            attachmentExport.relativeFilePath = lastManifestItem.relativeFilePath;
+
+            OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
+            exportItem.encryptedItem = attachmentExport.encryptedItem;
+            exportItem.recordName = lastRecordName;
+            exportItem.attachmentExport = attachmentExport;
+            [self.savedAttachmentItems addObject:exportItem];
+
+            DDLogVerbose(@"%@ recycled attachment: %@ as %@",
+                self.logTag,
+                attachmentExport.attachmentFilePath,
+                attachmentExport.relativeFilePath);
+            [self saveNextFileToCloudWithCompletion:completion];
+            return YES;
+        }
+    }
+
     @autoreleasepool {
         // OWSAttachmentExport is used to lazily write an encrypted copy of the
         // attachment to disk.
@@ -778,8 +944,11 @@ NS_ASSUME_NONNULL_BEGIN
             });
         }
         failure:^(NSError *error) {
-            // The manifest file is critical so any error uploading them is unrecoverable.
+            // Ensure that we continue to work off the main thread.
-            completion(error);
+            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+                // The manifest file is critical so any error uploading them is unrecoverable.
+                completion(error);
+            });
         }];
 }
 
@@ -815,7 +984,6 @@ NS_ASSUME_NONNULL_BEGIN
         OWSAssert(item.recordName.length > 0);
 
         itemJson[kOWSBackup_ManifestKey_RecordName] = item.recordName;
-        OWSAssert(item.encryptedItem.filePath.length > 0);
         OWSAssert(item.encryptedItem.encryptionKey.length > 0);
         itemJson[kOWSBackup_ManifestKey_EncryptionKey] = item.encryptedItem.encryptionKey.base64EncodedString;
         if (item.attachmentExport) {

Signal/src/util/OWSBackupImportJob.m

@@ -20,36 +20,14 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
 
 #pragma mark -
 
-@interface OWSBackupImportItem : NSObject
-
-@property (nonatomic) NSString *recordName;
-
-@property (nonatomic) NSData *encryptionKey;
-
-@property (nonatomic, nullable) NSString *relativeFilePath;
-
-@property (nonatomic, nullable) NSString *downloadFilePath;
-
-@property (nonatomic, nullable) NSNumber *uncompressedDataLength;
-
-@end
-
-#pragma mark -
-
-@implementation OWSBackupImportItem
-
-@end
-
-#pragma mark -
-
 @interface OWSBackupImportJob ()
 
 @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask;
 
 @property (nonatomic) OWSBackupIO *backupIO;
 
-@property (nonatomic) NSArray<OWSBackupImportItem *> *databaseItems;
+@property (nonatomic) NSArray<OWSBackupManifestItem *> *databaseItems;
-@property (nonatomic) NSArray<OWSBackupImportItem *> *attachmentsItems;
+@property (nonatomic) NSArray<OWSBackupManifestItem *> *attachmentsItems;
 
 @end
 
@@ -69,11 +47,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
 
     __weak OWSBackupImportJob *weakSelf = self;
     [OWSBackupAPI checkCloudKitAccessWithCompletion:^(BOOL hasAccess) {
-        if (hasAccess) {
+        dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
-            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+            if (hasAccess) {
                 [weakSelf start];
-            });
+            } else {
-        }
+                [weakSelf failWithErrorDescription:
+                              NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
+                                  @"Error indicating the a backup import could not import the user's data.")];
+            }
+        });
     }];
 }
 
@@ -98,41 +80,69 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
                                progress:nil];
 
     __weak OWSBackupImportJob *weakSelf = self;
-    [weakSelf downloadAndProcessManifestWithCompletion:^(NSError *_Nullable manifestError) {
+    [weakSelf downloadAndProcessManifestWithSuccess:^(OWSBackupManifestContents *manifest) {
-        if (manifestError) {
+        OWSBackupImportJob *strongSelf = weakSelf;
-            [weakSelf failWithError:manifestError];
+        if (!strongSelf) {
             return;
         }
-
+        if (self.isComplete) {
-        if (weakSelf.isComplete) {
             return;
         }
+        OWSCAssert(manifest.databaseItems.count > 0);
+        OWSCAssert(manifest.attachmentsItems);
+        strongSelf.databaseItems = manifest.databaseItems;
+        strongSelf.attachmentsItems = manifest.attachmentsItems;
+        [strongSelf downloadAndProcessImport];
+    }
+        failure:^(NSError *manifestError) {
+            [weakSelf failWithError:manifestError];
+        }
+        backupIO:self.backupIO];
+}
 
-        OWSAssert(self.databaseItems);
+- (void)downloadAndProcessImport
-        OWSAssert(self.attachmentsItems);
+{
-        NSMutableArray<OWSBackupImportItem *> *allItems = [NSMutableArray new];
+    OWSAssert(self.databaseItems);
-        [allItems addObjectsFromArray:self.databaseItems];
+    OWSAssert(self.attachmentsItems);
-        [allItems addObjectsFromArray:self.attachmentsItems];
-        [weakSelf
-            downloadFilesFromCloud:allItems
-                        completion:^(NSError *_Nullable fileDownloadError) {
-                            if (fileDownloadError) {
-                                [weakSelf failWithError:fileDownloadError];
-                                return;
-                            }
 
-                            if (weakSelf.isComplete) {
+    NSMutableArray<OWSBackupManifestItem *> *allItems = [NSMutableArray new];
+    [allItems addObjectsFromArray:self.databaseItems];
+    [allItems addObjectsFromArray:self.attachmentsItems];
+
+    __weak OWSBackupImportJob *weakSelf = self;
+    [weakSelf
+        downloadFilesFromCloud:allItems
+                    completion:^(NSError *_Nullable fileDownloadError) {
+                        if (fileDownloadError) {
+                            [weakSelf failWithError:fileDownloadError];
+                            return;
+                        }
+
+                        if (weakSelf.isComplete) {
+                            return;
+                        }
+
+                        [weakSelf restoreAttachmentFiles];
+
+                        if (weakSelf.isComplete) {
+                            return;
+                        }
+
+                        [weakSelf restoreDatabaseWithCompletion:^(BOOL restoreDatabaseSuccess) {
+                            if (!restoreDatabaseSuccess) {
+                                [weakSelf
+                                    failWithErrorDescription:NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
+                                                                 @"Error indicating the a backup import "
+                                                                 @"could not import the user's data.")];
                                 return;
                             }
 
-                            [weakSelf restoreAttachmentFiles];
-
                             if (weakSelf.isComplete) {
                                 return;
                             }
 
-                            [weakSelf restoreDatabaseWithCompletion:^(BOOL restoreDatabaseSuccess) {
+                            [weakSelf ensureMigrationsWithCompletion:^(BOOL ensureMigrationsSuccess) {
-                                if (!restoreDatabaseSuccess) {
+                                if (!ensureMigrationsSuccess) {
                                     [weakSelf failWithErrorDescription:NSLocalizedString(
                                                                            @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
                                                                            @"Error indicating the a backup import "
@@ -144,24 +154,10 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
                                     return;
                                 }
 
-                                [weakSelf ensureMigrationsWithCompletion:^(BOOL ensureMigrationsSuccess) {
+                                [weakSelf succeed];
-                                    if (!ensureMigrationsSuccess) {
-                                        [weakSelf failWithErrorDescription:NSLocalizedString(
-                                                                               @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
-                                                                               @"Error indicating the a backup import "
-                                                                               @"could not import the user's data.")];
-                                        return;
-                                    }
-
-                                    if (weakSelf.isComplete) {
-                                        return;
-                                    }
-
-                                    [weakSelf succeed];
-                                }];
                             }];
                         }];
-    }];
+                    }];
 }
 
 - (BOOL)configureImport
@@ -178,138 +174,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
     return YES;
 }
 
-- (void)downloadAndProcessManifestWithCompletion:(OWSBackupJobCompletion)completion
+- (void)downloadFilesFromCloud:(NSMutableArray<OWSBackupManifestItem *> *)items
-{
-    OWSAssert(completion);
-
-    DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
-
-    __weak OWSBackupImportJob *weakSelf = self;
-    [OWSBackupAPI downloadManifestFromCloudWithSuccess:^(NSData *data) {
-        dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
-            [weakSelf processManifest:data
-                           completion:^(BOOL success) {
-                               if (success) {
-                                   completion(nil);
-                               } else {
-                                   completion(OWSErrorWithCodeDescription(OWSErrorCodeImportBackupFailed,
-                                       NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
-                                           @"Error indicating the a backup import could not import the user's data.")));
-                               }
-                           }];
-        });
-    }
-        failure:^(NSError *error) {
-            // The manifest file is critical so any error downloading it is unrecoverable.
-            OWSProdLogAndFail(@"%@ Could not download manifest.", weakSelf.logTag);
-            completion(error);
-        }];
-}
-
-- (void)processManifest:(NSData *)manifestDataEncrypted completion:(OWSBackupJobBoolCompletion)completion
-{
-    OWSAssert(completion);
-    OWSAssert(self.backupIO);
-
-    if (self.isComplete) {
-        return;
-    }
-
-    DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
-
-    NSData *_Nullable manifestDataDecrypted =
-        [self.backupIO decryptDataAsData:manifestDataEncrypted encryptionKey:self.delegate.backupEncryptionKey];
-    if (!manifestDataDecrypted) {
-        OWSProdLogAndFail(@"%@ Could not decrypt manifest.", self.logTag);
-        return completion(NO);
-    }
-
-    NSError *error;
-    NSDictionary<NSString *, id> *_Nullable json =
-        [NSJSONSerialization JSONObjectWithData:manifestDataDecrypted options:0 error:&error];
-    if (![json isKindOfClass:[NSDictionary class]]) {
-        OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag);
-        return completion(NO);
-    }
-
-    DDLogVerbose(@"%@ json: %@", self.logTag, json);
-
-    NSArray<OWSBackupImportItem *> *_Nullable databaseItems =
-        [self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles];
-    if (!databaseItems) {
-        return completion(NO);
-    }
-    NSArray<OWSBackupImportItem *> *_Nullable attachmentsItems =
-        [self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles];
-    if (!attachmentsItems) {
-        return completion(NO);
-    }
-
-    self.databaseItems = databaseItems;
-    self.attachmentsItems = attachmentsItems;
-
-    return completion(YES);
-}
-
-- (nullable NSArray<OWSBackupImportItem *> *)parseItems:(id)json key:(NSString *)key
-{
-    OWSAssert(json);
-    OWSAssert(key.length);
-
-    if (![json isKindOfClass:[NSDictionary class]]) {
-        OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key);
-        return nil;
-    }
-    NSArray *itemMaps = json[key];
-    if (![itemMaps isKindOfClass:[NSArray class]]) {
-        OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key);
-        return nil;
-    }
-    NSMutableArray<OWSBackupImportItem *> *items = [NSMutableArray new];
-    for (NSDictionary *itemMap in itemMaps) {
-        if (![itemMap isKindOfClass:[NSDictionary class]]) {
-            OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key);
-            return nil;
-        }
-        NSString *_Nullable recordName = itemMap[kOWSBackup_ManifestKey_RecordName];
-        NSString *_Nullable encryptionKeyString = itemMap[kOWSBackup_ManifestKey_EncryptionKey];
-        NSString *_Nullable relativeFilePath = itemMap[kOWSBackup_ManifestKey_RelativeFilePath];
-        NSNumber *_Nullable uncompressedDataLength = itemMap[kOWSBackup_ManifestKey_DataSize];
-        if (![recordName isKindOfClass:[NSString class]]) {
-            OWSProdLogAndFail(@"%@ manifest has invalid recordName: %@.", self.logTag, key);
-            return nil;
-        }
-        if (![encryptionKeyString isKindOfClass:[NSString class]]) {
-            OWSProdLogAndFail(@"%@ manifest has invalid encryptionKey: %@.", self.logTag, key);
-            return nil;
-        }
-        // relativeFilePath is an optional field.
-        if (relativeFilePath && ![relativeFilePath isKindOfClass:[NSString class]]) {
-            OWSProdLogAndFail(@"%@ manifest has invalid relativeFilePath: %@.", self.logTag, key);
-            return nil;
-        }
-        NSData *_Nullable encryptionKey = [NSData dataFromBase64String:encryptionKeyString];
-        if (!encryptionKey) {
-            OWSProdLogAndFail(@"%@ manifest has corrupt encryptionKey: %@.", self.logTag, key);
-            return nil;
-        }
-        // uncompressedDataLength is an optional field.
-        if (uncompressedDataLength && ![uncompressedDataLength isKindOfClass:[NSNumber class]]) {
-            OWSProdLogAndFail(@"%@ manifest has invalid uncompressedDataLength: %@.", self.logTag, key);
-            return nil;
-        }
-
-        OWSBackupImportItem *item = [OWSBackupImportItem new];
-        item.recordName = recordName;
-        item.encryptionKey = encryptionKey;
-        item.relativeFilePath = relativeFilePath;
-        item.uncompressedDataLength = uncompressedDataLength;
-        [items addObject:item];
-    }
-    return items;
-}
-
-- (void)downloadFilesFromCloud:(NSMutableArray<OWSBackupImportItem *> *)items
                     completion:(OWSBackupJobCompletion)completion
 {
     OWSAssert(items.count > 0);
@@ -320,7 +185,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
     [self downloadNextItemFromCloud:items recordCount:items.count completion:completion];
 }
 
-- (void)downloadNextItemFromCloud:(NSMutableArray<OWSBackupImportItem *> *)items
+- (void)downloadNextItemFromCloud:(NSMutableArray<OWSBackupManifestItem *> *)items
                       recordCount:(NSUInteger)recordCount
                        completion:(OWSBackupJobCompletion)completion
 {
@@ -336,7 +201,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
         // All downloads are complete; exit.
         return completion(nil);
     }
-    OWSBackupImportItem *item = items.lastObject;
+    OWSBackupManifestItem *item = items.lastObject;
     [items removeLastObject];
 
     CGFloat progress = (recordCount > 0 ? ((recordCount - items.count) / (CGFloat)recordCount) : 0.f);
@@ -372,7 +237,10 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
             });
         }
         failure:^(NSError *error) {
-            completion(error);
+            // Ensure that we continue to work off the main thread.
+            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+                completion(error);
+            });
         }];
 }
 
@@ -383,7 +251,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
     NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder];
 
     NSUInteger count = 0;
-    for (OWSBackupImportItem *item in self.attachmentsItems) {
+    for (OWSBackupManifestItem *item in self.attachmentsItems) {
         if (self.isComplete) {
             return;
         }
@@ -408,6 +276,13 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
             DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath);
             continue;
         }
+        NSString *dstDirPath = [dstFilePath stringByDeletingLastPathComponent];
+        if (![NSFileManager.defaultManager fileExistsAtPath:dstDirPath]) {
+            if (![OWSFileSystem ensureDirectoryExists:dstDirPath]) {
+                DDLogError(@"%@ couldn't create directory for file restore: %@.", self.logTag, dstFilePath);
+                continue;
+            }
+        }
         @autoreleasepool {
             if (![self.backupIO decryptFileAsFile:item.downloadFilePath
                                       dstFilePath:dstFilePath
@@ -475,7 +350,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
         }
 
         NSUInteger count = 0;
-        for (OWSBackupImportItem *item in self.databaseItems) {
+        for (OWSBackupManifestItem *item in self.databaseItems) {
             if (self.isComplete) {
                 return;
             }

Signal/src/util/OWSBackupJob.h

@@ -11,10 +11,44 @@ extern NSString *const kOWSBackup_ManifestKey_EncryptionKey;
 extern NSString *const kOWSBackup_ManifestKey_RelativeFilePath;
 extern NSString *const kOWSBackup_ManifestKey_DataSize;
 
+@class OWSBackupIO;
+@class OWSBackupJob;
+@class OWSBackupManifestContents;
+
 typedef void (^OWSBackupJobBoolCompletion)(BOOL success);
 typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error);
+typedef void (^OWSBackupJobManifestSuccess)(OWSBackupManifestContents *manifest);
+typedef void (^OWSBackupJobManifestFailure)(NSError *error);
 
-@class OWSBackupJob;
+@interface OWSBackupManifestItem : NSObject
+
+@property (nonatomic) NSString *recordName;
+
+@property (nonatomic) NSData *encryptionKey;
+
+// This property is only set for certain types of manifest item,
+// namely attachments where we need to know where the attachment's
+// file should reside relative to the attachments folder.
+@property (nonatomic, nullable) NSString *relativeFilePath;
+
+// This property is only set if the manifest item is downloaded.
+@property (nonatomic, nullable) NSString *downloadFilePath;
+
+// This property is only set if the manifest item is compressed.
+@property (nonatomic, nullable) NSNumber *uncompressedDataLength;
+
+@end
+
+#pragma mark -
+
+@interface OWSBackupManifestContents : NSObject
+
+@property (nonatomic) NSArray<OWSBackupManifestItem *> *databaseItems;
+@property (nonatomic) NSArray<OWSBackupManifestItem *> *attachmentsItems;
+
+@end
+
+#pragma mark -
 
 @protocol OWSBackupJobDelegate <NSObject>
 
@@ -63,6 +97,12 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error);
 - (void)failWithError:(NSError *)error;
 - (void)updateProgressWithDescription:(nullable NSString *)description progress:(nullable NSNumber *)progress;
 
+#pragma mark - Manifest
+
+- (void)downloadAndProcessManifestWithSuccess:(OWSBackupJobManifestSuccess)success
+                                      failure:(OWSBackupJobManifestFailure)failure
+                                     backupIO:(OWSBackupIO *)backupIO;
+
 @end
 
 NS_ASSUME_NONNULL_END

Signal/src/util/OWSBackupJob.m

@@ -3,6 +3,7 @@
 //
 
 #import "OWSBackupJob.h"
+#import "OWSBackupIO.h"
 #import "Signal-Swift.h"
 #import <Curve25519Kit/Randomness.h>
 #import <SAMKeychain/SAMKeychain.h>
@@ -19,6 +20,18 @@ NSString *const kOWSBackup_ManifestKey_DataSize = @"data_size";
 
 NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
 
+@implementation OWSBackupManifestItem
+
+@end
+
+#pragma mark -
+
+@implementation OWSBackupManifestContents
+
+@end
+
+#pragma mark -
+
 @interface OWSBackupJob ()
 
 @property (nonatomic, weak) id<OWSBackupJobDelegate> delegate;
@@ -146,6 +159,149 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
     });
 }
 
+#pragma mark - Manifest
+
+- (void)downloadAndProcessManifestWithSuccess:(OWSBackupJobManifestSuccess)success
+                                      failure:(OWSBackupJobManifestFailure)failure
+                                     backupIO:(OWSBackupIO *)backupIO
+{
+    OWSAssert(success);
+    OWSAssert(failure);
+    OWSAssert(backupIO);
+
+    DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
+
+    __weak OWSBackupJob *weakSelf = self;
+    [OWSBackupAPI downloadManifestFromCloudWithSuccess:^(NSData *data) {
+        dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+            [weakSelf processManifest:data
+                              success:success
+                              failure:^{
+                                  failure(OWSErrorWithCodeDescription(OWSErrorCodeImportBackupFailed,
+                                      NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT",
+                                          @"Error indicating the a backup import could not import the user's data.")));
+                              }
+                             backupIO:backupIO];
+        });
+    }
+        failure:^(NSError *error) {
+            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+                // The manifest file is critical so any error downloading it is unrecoverable.
+                OWSProdLogAndFail(@"%@ Could not download manifest.", weakSelf.logTag);
+                failure(error);
+            });
+        }];
+}
+
+- (void)processManifest:(NSData *)manifestDataEncrypted
+                success:(OWSBackupJobManifestSuccess)success
+                failure:(dispatch_block_t)failure
+               backupIO:(OWSBackupIO *)backupIO
+{
+    OWSAssert(manifestDataEncrypted.length > 0);
+    OWSAssert(success);
+    OWSAssert(failure);
+    OWSAssert(backupIO);
+
+    if (self.isComplete) {
+        return;
+    }
+
+    DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
+
+    NSData *_Nullable manifestDataDecrypted =
+        [backupIO decryptDataAsData:manifestDataEncrypted encryptionKey:self.delegate.backupEncryptionKey];
+    if (!manifestDataDecrypted) {
+        OWSProdLogAndFail(@"%@ Could not decrypt manifest.", self.logTag);
+        return failure();
+    }
+
+    NSError *error;
+    NSDictionary<NSString *, id> *_Nullable json =
+        [NSJSONSerialization JSONObjectWithData:manifestDataDecrypted options:0 error:&error];
+    if (![json isKindOfClass:[NSDictionary class]]) {
+        OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag);
+        return failure();
+    }
+
+    DDLogVerbose(@"%@ json: %@", self.logTag, json);
+
+    NSArray<OWSBackupManifestItem *> *_Nullable databaseItems =
+        [self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles];
+    if (!databaseItems) {
+        return failure();
+    }
+    NSArray<OWSBackupManifestItem *> *_Nullable attachmentsItems =
+        [self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles];
+    if (!attachmentsItems) {
+        return failure();
+    }
+
+    OWSBackupManifestContents *contents = [OWSBackupManifestContents new];
+    contents.databaseItems = databaseItems;
+    contents.attachmentsItems = attachmentsItems;
+
+    return success(contents);
+}
+
+- (nullable NSArray<OWSBackupManifestItem *> *)parseItems:(id)json key:(NSString *)key
+{
+    OWSAssert(json);
+    OWSAssert(key.length);
+
+    if (![json isKindOfClass:[NSDictionary class]]) {
+        OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key);
+        return nil;
+    }
+    NSArray *itemMaps = json[key];
+    if (![itemMaps isKindOfClass:[NSArray class]]) {
+        OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key);
+        return nil;
+    }
+    NSMutableArray<OWSBackupManifestItem *> *items = [NSMutableArray new];
+    for (NSDictionary *itemMap in itemMaps) {
+        if (![itemMap isKindOfClass:[NSDictionary class]]) {
+            OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key);
+            return nil;
+        }
+        NSString *_Nullable recordName = itemMap[kOWSBackup_ManifestKey_RecordName];
+        NSString *_Nullable encryptionKeyString = itemMap[kOWSBackup_ManifestKey_EncryptionKey];
+        NSString *_Nullable relativeFilePath = itemMap[kOWSBackup_ManifestKey_RelativeFilePath];
+        NSNumber *_Nullable uncompressedDataLength = itemMap[kOWSBackup_ManifestKey_DataSize];
+        if (![recordName isKindOfClass:[NSString class]]) {
+            OWSProdLogAndFail(@"%@ manifest has invalid recordName: %@.", self.logTag, key);
+            return nil;
+        }
+        if (![encryptionKeyString isKindOfClass:[NSString class]]) {
+            OWSProdLogAndFail(@"%@ manifest has invalid encryptionKey: %@.", self.logTag, key);
+            return nil;
+        }
+        // relativeFilePath is an optional field.
+        if (relativeFilePath && ![relativeFilePath isKindOfClass:[NSString class]]) {
+            OWSProdLogAndFail(@"%@ manifest has invalid relativeFilePath: %@.", self.logTag, key);
+            return nil;
+        }
+        NSData *_Nullable encryptionKey = [NSData dataFromBase64String:encryptionKeyString];
+        if (!encryptionKey) {
+            OWSProdLogAndFail(@"%@ manifest has corrupt encryptionKey: %@.", self.logTag, key);
+            return nil;
+        }
+        // uncompressedDataLength is an optional field.
+        if (uncompressedDataLength && ![uncompressedDataLength isKindOfClass:[NSNumber class]]) {
+            OWSProdLogAndFail(@"%@ manifest has invalid uncompressedDataLength: %@.", self.logTag, key);
+            return nil;
+        }
+
+        OWSBackupManifestItem *item = [OWSBackupManifestItem new];
+        item.recordName = recordName;
+        item.encryptionKey = encryptionKey;
+        item.relativeFilePath = relativeFilePath;
+        item.uncompressedDataLength = uncompressedDataLength;
+        [items addObject:item];
+    }
+    return items;
+}
+
 @end
 
 NS_ASSUME_NONNULL_END

Signal/translations/en.lproj/Localizable.strings

@@ -184,6 +184,9 @@
 /* Error indicating the a backup import could not import the user's data. */
 "BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT" = "Backup could not be imported.";
 
+/* Indicates that the backup import is checking for an existing backup. */
+"BACKUP_IMPORT_PHASE_CHECK_BACKUP" = "Checking Backup State";
+
 /* Indicates that the backup import is being configured. */
 "BACKUP_IMPORT_PHASE_CONFIGURATION" = "Configuring Backup";