Merge branch 'charlesmchen/incrementalBackup8'

pull/1/head
Matthew Chen 7 years ago
commit 972cbaebfb

@ -1 +1 @@
Subproject commit 1d47ca77ea929a2fd76b2b3410487b61f18f5b54
Subproject commit 10be6cb3689bd1815ffcecc2f9500b2b55f60962

@ -205,6 +205,7 @@
34D1F0C01F8EC1760066283D /* MessageRecipientStatusUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D1F0BF1F8EC1760066283D /* MessageRecipientStatusUtils.swift */; };
34D2CCD4206294B900CB1A14 /* OWSScreenLock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D2CCD3206294B900CB1A14 /* OWSScreenLock.swift */; };
34D2CCDA2062E7D000CB1A14 /* OWSScreenLockUI.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D2CCD92062E7D000CB1A14 /* OWSScreenLockUI.m */; };
34D2CCD220618B3000CB1A14 /* OWSBackupLazyRestoreJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D2CCD120618B2F00CB1A14 /* OWSBackupLazyRestoreJob.swift */; };
34D5CCA91EAE3D30005515DB /* AvatarViewHelper.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D5CCA81EAE3D30005515DB /* AvatarViewHelper.m */; };
34D8C0271ED3673300188D7C /* DebugUIMessages.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C0241ED3673300188D7C /* DebugUIMessages.m */; };
34D8C0281ED3673300188D7C /* DebugUITableViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C0261ED3673300188D7C /* DebugUITableViewController.m */; };
@ -812,6 +813,7 @@
34D2CCD3206294B900CB1A14 /* OWSScreenLock.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OWSScreenLock.swift; sourceTree = "<group>"; };
34D2CCD82062E7D000CB1A14 /* OWSScreenLockUI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSScreenLockUI.h; sourceTree = "<group>"; };
34D2CCD92062E7D000CB1A14 /* OWSScreenLockUI.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSScreenLockUI.m; sourceTree = "<group>"; };
34D2CCD120618B2F00CB1A14 /* OWSBackupLazyRestoreJob.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OWSBackupLazyRestoreJob.swift; sourceTree = "<group>"; };
34D5CCA71EAE3D30005515DB /* AvatarViewHelper.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AvatarViewHelper.h; sourceTree = "<group>"; };
34D5CCA81EAE3D30005515DB /* AvatarViewHelper.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AvatarViewHelper.m; sourceTree = "<group>"; };
34D8C0231ED3673300188D7C /* DebugUIMessages.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DebugUIMessages.h; sourceTree = "<group>"; };
@ -1974,6 +1976,7 @@
34D2CCD3206294B900CB1A14 /* OWSScreenLock.swift */,
34D2CCD82062E7D000CB1A14 /* OWSScreenLockUI.h */,
34D2CCD92062E7D000CB1A14 /* OWSScreenLockUI.m */,
34D2CCD120618B2F00CB1A14 /* OWSBackupLazyRestoreJob.swift */,
4579431C1E7C8CE9008ED0C0 /* Pastelog.h */,
4579431D1E7C8CE9008ED0C0 /* Pastelog.m */,
450DF2041E0D74AC003D14BE /* Platform.swift */,
@ -3143,6 +3146,7 @@
458DE9D91DEE7B360071BB03 /* OWSWebRTCDataProtos.pb.m in Sources */,
76EB063C18170B33006006FC /* NumberUtil.m in Sources */,
451166C01FD86B98000739BA /* AccountManager.swift in Sources */,
34D2CCD220618B3000CB1A14 /* OWSBackupLazyRestoreJob.swift in Sources */,
3430FE181F7751D4000EC51B /* GiphyAPI.swift in Sources */,
34A55F3720485465002CC6DE /* OWS2FARegistrationViewController.m in Sources */,
340FC8AD204DAC8D007AEB0F /* OWSLinkedDevicesTableViewController.m in Sources */,

@ -418,8 +418,8 @@ static NSString *const kURLHostVerifyPrefix = @"verify";
// Every time we change or add a database view in such a way that
// might cause a delay on launch, we need to bump this constant.
//
// We upgraded YapDatabase in v2.20.0 and need to regenerate all database views.
NSString *kLastVersionWithDatabaseViewChange = @"2.20.0";
// We added a database view in v2.23.0.
NSString *kLastVersionWithDatabaseViewChange = @"2.23.0";
BOOL mayNeedUpgrade = ([TSAccountManager isRegistered] && lastLaunchedAppVersion
&& (!lastCompletedLaunchAppVersion ||
[VersionMigrations isVersion:lastCompletedLaunchAppVersion
@ -1134,6 +1134,9 @@ static NSString *const kURLHostVerifyPrefix = @"verify";
[self ensureRootViewController];
[OWSBackup.sharedManager setup];
// Resume lazy restore.
[OWSBackupLazyRestoreJob runAsync];
}
- (void)registrationStateDidChange

@ -19,6 +19,8 @@
#import "NotificationsManager.h"
#import "OWSAnyTouchGestureRecognizer.h"
#import "OWSAudioPlayer.h"
#import "OWSBackup.h"
#import "OWSBackupIO.h"
#import "OWSBezierPathView.h"
#import "OWSCallNotificationsAdaptee.h"
#import "OWSDatabaseMigration.h"

@ -48,6 +48,10 @@ NS_ASSUME_NONNULL_BEGIN
actionBlock:^{
[DebugUIBackup clearAllCloudKitRecords];
}]];
[items addObject:[OWSTableItem itemWithTitle:@"Clear Backup Metadata Cache"
actionBlock:^{
[DebugUIBackup clearBackupMetadataCache];
}]];
return [OWSTableSection sectionWithTitle:self.name items:items];
}
@ -161,6 +165,16 @@ NS_ASSUME_NONNULL_BEGIN
[OWSBackup.sharedManager clearAllCloudKitRecords];
}
+ (void)clearBackupMetadataCache
{
DDLogInfo(@"%@ ClearBackupMetadataCache.", self.logTag);
[OWSPrimaryStorage.sharedManager.newDatabaseConnection
readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
[transaction removeAllObjectsInCollection:[OWSBackupFragment collection]];
}];
}
@end
NS_ASSUME_NONNULL_END

@ -20,13 +20,15 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) {
OWSBackupState_Succeeded,
};
@class OWSBackupIO;
@class TSAttachmentStream;
@class TSThread;
@interface OWSBackup : NSObject
- (instancetype)init NS_UNAVAILABLE;
+ (instancetype)sharedManager;
+ (instancetype)sharedManager NS_SWIFT_NAME(shared());
- (void)setup;
@ -71,6 +73,16 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) {
- (void)logBackupRecords;
- (void)clearAllCloudKitRecords;
#pragma mark - Lazy Restore
- (NSArray<NSString *> *)attachmentRecordNamesForLazyRestore;
- (NSArray<NSString *> *)attachmentIdsForLazyRestore;
- (void)lazyRestoreAttachment:(TSAttachmentStream *)attachment
backupIO:(OWSBackupIO *)backupIO
completion:(OWSBackupBoolBlock)completion;
@end
NS_ASSUME_NONNULL_END

@ -4,6 +4,7 @@
#import "OWSBackup.h"
#import "OWSBackupExportJob.h"
#import "OWSBackupIO.h"
#import "OWSBackupImportJob.h"
#import "Signal-Swift.h"
#import <Curve25519Kit/Randomness.h>
@ -483,6 +484,164 @@ NS_ASSUME_NONNULL_BEGIN
}];
}
#pragma mark - Lazy Restore
- (NSArray<NSString *> *)attachmentRecordNamesForLazyRestore
{
NSMutableArray<NSString *> *recordNames = [NSMutableArray new];
[self.dbConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
id ext = [transaction ext:TSLazyRestoreAttachmentsDatabaseViewExtensionName];
if (!ext) {
OWSProdLogAndFail(@"%@ Could not load database view.", self.logTag);
return;
}
[ext enumerateKeysAndObjectsInGroup:TSLazyRestoreAttachmentsGroup
usingBlock:^(
NSString *collection, NSString *key, id object, NSUInteger index, BOOL *stop) {
if (![object isKindOfClass:[TSAttachmentStream class]]) {
OWSProdLogAndFail(@"%@ Unexpected object: %@ in collection:%@",
self.logTag,
[object class],
collection);
return;
}
TSAttachmentStream *attachmentStream = object;
if (!attachmentStream.lazyRestoreFragment) {
OWSProdLogAndFail(@"%@ Invalid object: %@ in collection:%@",
self.logTag,
[object class],
collection);
return;
}
[recordNames addObject:attachmentStream.lazyRestoreFragment.recordName];
}];
}];
return recordNames;
}
- (NSArray<NSString *> *)attachmentIdsForLazyRestore
{
NSMutableArray<NSString *> *attachmentIds = [NSMutableArray new];
[self.dbConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
id ext = [transaction ext:TSLazyRestoreAttachmentsDatabaseViewExtensionName];
if (!ext) {
OWSProdLogAndFail(@"%@ Could not load database view.", self.logTag);
return;
}
[ext enumerateKeysInGroup:TSLazyRestoreAttachmentsGroup
usingBlock:^(NSString *collection, NSString *key, NSUInteger index, BOOL *stop) {
[attachmentIds addObject:key];
}];
}];
return attachmentIds;
}
- (void)lazyRestoreAttachment:(TSAttachmentStream *)attachment
backupIO:(OWSBackupIO *)backupIO
completion:(OWSBackupBoolBlock)completion
{
OWSAssert(attachment);
OWSAssert(backupIO);
OWSAssert(completion);
NSString *_Nullable attachmentFilePath = [attachment filePath];
if (attachmentFilePath.length < 1) {
DDLogError(@"%@ Attachment has invalid file path.", self.logTag);
return completion(NO);
}
if ([NSFileManager.defaultManager fileExistsAtPath:attachmentFilePath]) {
DDLogError(@"%@ Attachment already has file.", self.logTag);
return completion(NO);
}
OWSBackupFragment *_Nullable lazyRestoreFragment = attachment.lazyRestoreFragment;
if (!lazyRestoreFragment) {
DDLogWarn(@"%@ Attachment missing lazy restore metadata.", self.logTag);
return completion(NO);
}
if (lazyRestoreFragment.recordName.length < 1 || lazyRestoreFragment.encryptionKey.length < 1) {
DDLogError(@"%@ Incomplete lazy restore metadata.", self.logTag);
return completion(NO);
}
// Use a predictable file path so that multiple "import backup" attempts
// will leverage successful file downloads from previous attempts.
//
// TODO: This will also require imports using a predictable jobTempDirPath.
NSString *tempFilePath = [backupIO generateTempFilePath];
[OWSBackupAPI downloadFileFromCloudWithRecordName:lazyRestoreFragment.recordName
toFileUrl:[NSURL fileURLWithPath:tempFilePath]
success:^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[self lazyRestoreAttachment:attachment
backupIO:backupIO
encryptedFilePath:tempFilePath
encryptionKey:lazyRestoreFragment.encryptionKey
completion:completion];
});
}
failure:^(NSError *error) {
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
completion(NO);
});
}];
}
- (void)lazyRestoreAttachment:(TSAttachmentStream *)attachment
backupIO:(OWSBackupIO *)backupIO
encryptedFilePath:(NSString *)encryptedFilePath
encryptionKey:(NSData *)encryptionKey
completion:(OWSBackupBoolBlock)completion
{
OWSAssert(attachment);
OWSAssert(backupIO);
OWSAssert(encryptedFilePath.length > 0);
OWSAssert(encryptionKey.length > 0);
OWSAssert(completion);
NSData *_Nullable data = [NSData dataWithContentsOfFile:encryptedFilePath];
if (!data) {
DDLogError(@"%@ Could not load encrypted file.", self.logTag);
return completion(NO);
}
NSString *decryptedFilePath = [backupIO generateTempFilePath];
@autoreleasepool {
if (![backupIO decryptFileAsFile:encryptedFilePath dstFilePath:decryptedFilePath encryptionKey:encryptionKey]) {
DDLogError(@"%@ Could not load decrypt file.", self.logTag);
return completion(NO);
}
}
NSString *_Nullable attachmentFilePath = [attachment filePath];
if (attachmentFilePath.length < 1) {
DDLogError(@"%@ Attachment has invalid file path.", self.logTag);
return completion(NO);
}
NSString *attachmentDirPath = [attachmentFilePath stringByDeletingLastPathComponent];
if (![OWSFileSystem ensureDirectoryExists:attachmentDirPath]) {
DDLogError(@"%@ Couldn't create directory for attachment file.", self.logTag);
return completion(NO);
}
NSError *error;
BOOL success =
[NSFileManager.defaultManager moveItemAtPath:decryptedFilePath toPath:attachmentFilePath error:&error];
if (!success || error) {
DDLogError(@"%@ Attachment file could not be restored: %@.", self.logTag, error);
return completion(NO);
}
[attachment updateWithLazyRestoreComplete];
completion(YES);
}
#pragma mark - Notifications
- (void)postDidChangeNotification

@ -6,6 +6,12 @@ import Foundation
import SignalServiceKit
import CloudKit
// We don't worry about atomic writes. Each backup export
// will diff against last successful backup.
//
// Note that all of our CloudKit records are immutable.
// "Persistent" records are only uploaded once.
// "Ephemeral" records are always uploaded to a new record name.
@objc public class OWSBackupAPI: NSObject {
// If we change the record types, we need to ensure indices

@ -295,8 +295,7 @@ NS_ASSUME_NONNULL_BEGIN
@property (nonatomic, nullable) OWSBackupExportItem *manifestItem;
// If we are replacing an existing backup, we use some of its contents for continuity.
@property (nonatomic, nullable) NSDictionary<NSString *, OWSBackupManifestItem *> *lastManifestItemMap;
@property (nonatomic, nullable) NSSet<NSString *> *lastRecordNames;
@property (nonatomic, nullable) NSSet<NSString *> *lastValidRecordNames;
@end
@ -346,7 +345,7 @@ NS_ASSUME_NONNULL_BEGIN
if (self.isComplete) {
return;
}
[self tryToFetchManifestWithCompletion:^(BOOL tryToFetchManifestSuccess) {
[self fetchAllRecordsWithCompletion:^(BOOL tryToFetchManifestSuccess) {
if (!tryToFetchManifestSuccess) {
[self failWithErrorDescription:
NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
@ -374,7 +373,7 @@ NS_ASSUME_NONNULL_BEGIN
[weakSelf failWithError:saveError];
return;
}
[self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) {
[self cleanUpWithCompletion:^(NSError *_Nullable cleanUpError) {
if (cleanUpError) {
[weakSelf failWithError:cleanUpError];
return;
@ -422,69 +421,6 @@ NS_ASSUME_NONNULL_BEGIN
}];
}
- (void)tryToFetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion
{
OWSAssert(completion);
if (self.isComplete) {
return;
}
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
[self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_CHECK_BACKUP",
@"Indicates that the backup import is checking for an existing backup.")
progress:nil];
__weak OWSBackupExportJob *weakSelf = self;
[OWSBackupAPI checkForManifestInCloudWithSuccess:^(BOOL value) {
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
if (value) {
[weakSelf fetchManifestWithCompletion:completion];
} else {
// There is no existing manifest; continue.
completion(YES);
}
});
}
failure:^(NSError *error) {
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
completion(NO);
});
}];
}
- (void)fetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion
{
OWSAssert(completion);
if (self.isComplete) {
return;
}
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
__weak OWSBackupExportJob *weakSelf = self;
[weakSelf downloadAndProcessManifestWithSuccess:^(OWSBackupManifestContents *manifest) {
OWSBackupExportJob *strongSelf = weakSelf;
if (!strongSelf) {
return;
}
if (strongSelf.isComplete) {
return;
}
OWSCAssert(manifest.databaseItems.count > 0);
OWSCAssert(manifest.attachmentsItems);
[strongSelf processLastManifest:manifest];
[strongSelf fetchAllRecordsWithCompletion:completion];
}
failure:^(NSError *manifestError) {
completion(NO);
}
backupIO:self.backupIO];
}
- (void)fetchAllRecordsWithCompletion:(OWSBackupJobBoolCompletion)completion
{
OWSAssert(completion);
@ -505,7 +441,7 @@ NS_ASSUME_NONNULL_BEGIN
if (strongSelf.isComplete) {
return;
}
strongSelf.lastRecordNames = [NSSet setWithArray:recordNames];
strongSelf.lastValidRecordNames = [NSSet setWithArray:recordNames];
completion(YES);
});
}
@ -516,17 +452,6 @@ NS_ASSUME_NONNULL_BEGIN
}];
}
- (void)processLastManifest:(OWSBackupManifestContents *)manifest
{
OWSAssert(manifest);
NSMutableDictionary<NSString *, OWSBackupManifestItem *> *lastManifestItemMap = [NSMutableDictionary new];
for (OWSBackupManifestItem *manifestItem in manifest.attachmentsItems) {
lastManifestItemMap[manifestItem.recordName] = manifestItem;
}
self.lastManifestItemMap = [lastManifestItemMap copy];
}
- (BOOL)exportDatabase
{
OWSAssert(self.backupIO);
@ -806,28 +731,29 @@ NS_ASSUME_NONNULL_BEGIN
OWSAttachmentExport *attachmentExport = self.unsavedAttachmentExports.lastObject;
[self.unsavedAttachmentExports removeLastObject];
if (self.lastManifestItemMap && self.lastRecordNames) {
// Wherever possible, we do incremental backups and re-use fragments of the last backup.
if (self.lastValidRecordNames) {
// Wherever possible, we do incremental backups and re-use fragments of the last
// backup and/or restore.
// Recycling fragments doesn't just reduce redundant network activity,
// it allows us to skip the local export work, i.e. encryption.
// To do so, we must preserve the metadata for these fragments.
//
// We check two things:
//
// * That the "last known backup manifest" contains an item from which we can recover
// this record's metadata.
// * That we already know the metadata for this fragment (from a previous backup
// or restore).
// * That this record does in fact exist in our CloudKit database.
NSString *lastRecordName = [OWSBackupAPI recordNameForPersistentFileWithFileId:attachmentExport.attachmentId];
OWSBackupManifestItem *_Nullable lastManifestItem = self.lastManifestItemMap[lastRecordName];
if (lastManifestItem && [self.lastRecordNames containsObject:lastRecordName]) {
OWSAssert(lastManifestItem.encryptionKey.length > 0);
OWSAssert(lastManifestItem.relativeFilePath.length > 0);
OWSBackupFragment *_Nullable lastBackupFragment = [OWSBackupFragment fetchObjectWithUniqueID:lastRecordName];
if (lastBackupFragment && [self.lastValidRecordNames containsObject:lastRecordName]) {
OWSAssert(lastBackupFragment.encryptionKey.length > 0);
OWSAssert(lastBackupFragment.relativeFilePath.length > 0);
// Recycle the metadata from the last backup's manifest.
OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new];
encryptedItem.encryptionKey = lastManifestItem.encryptionKey;
encryptedItem.encryptionKey = lastBackupFragment.encryptionKey;
attachmentExport.encryptedItem = encryptedItem;
attachmentExport.relativeFilePath = lastManifestItem.relativeFilePath;
attachmentExport.relativeFilePath = lastBackupFragment.relativeFilePath;
OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
exportItem.encryptedItem = attachmentExport.encryptedItem;
@ -887,6 +813,15 @@ NS_ASSUME_NONNULL_BEGIN
exportItem.attachmentExport = attachmentExport;
[strongSelf.savedAttachmentItems addObject:exportItem];
// Immediately save the record metadata to facilitate export resume.
OWSBackupFragment *backupFragment = [OWSBackupFragment new];
backupFragment.recordName = recordName;
backupFragment.encryptionKey = exportItem.encryptedItem.encryptionKey;
backupFragment.relativeFilePath = attachmentExport.relativeFilePath;
backupFragment.attachmentId = attachmentExport.attachmentId;
backupFragment.uncompressedDataLength = exportItem.uncompressedDataLength;
[backupFragment save];
DDLogVerbose(@"%@ saved attachment: %@ as %@",
self.logTag,
attachmentExport.attachmentFilePath,
@ -990,6 +925,10 @@ NS_ASSUME_NONNULL_BEGIN
OWSAssert(item.attachmentExport.relativeFilePath.length > 0);
itemJson[kOWSBackup_ManifestKey_RelativeFilePath] = item.attachmentExport.relativeFilePath;
}
if (item.attachmentExport.attachmentId) {
OWSAssert(item.attachmentExport.attachmentId.length > 0);
itemJson[kOWSBackup_ManifestKey_AttachmentId] = item.attachmentExport.attachmentId;
}
if (item.uncompressedDataLength) {
itemJson[kOWSBackup_ManifestKey_DataSize] = item.uncompressedDataLength;
}
@ -999,10 +938,15 @@ NS_ASSUME_NONNULL_BEGIN
return result;
}
- (void)cleanUpCloudWithCompletion:(OWSBackupJobCompletion)completion
- (void)cleanUpWithCompletion:(OWSBackupJobCompletion)completion
{
OWSAssert(completion);
if (self.isComplete) {
// Job was aborted.
return completion(nil);
}
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
[self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_CLEAN_UP",
@ -1029,9 +973,47 @@ NS_ASSUME_NONNULL_BEGIN
OWSAssert(![activeRecordNames containsObject:self.manifestItem.recordName]);
[activeRecordNames addObject:self.manifestItem.recordName];
// TODO: If we implement "lazy restores" where attachments (etc.) are
// restored lazily, we need to include the record names for all
// Because we do "lazy attachment restores", we need to include the record names for all
// records that haven't been restored yet.
NSArray<NSString *> *restoringRecordNames = [OWSBackup.sharedManager attachmentRecordNamesForLazyRestore];
[activeRecordNames addObjectsFromArray:restoringRecordNames];
[self cleanUpMetadataCacheWithActiveRecordNames:activeRecordNames];
[self cleanUpCloudWithActiveRecordNames:activeRecordNames completion:completion];
}
- (void)cleanUpMetadataCacheWithActiveRecordNames:(NSSet<NSString *> *)activeRecordNames
{
OWSAssert(activeRecordNames.count > 0);
if (self.isComplete) {
// Job was aborted.
return;
}
// After every successful backup export, we can (and should) cull metadata
// for any backup fragment (i.e. CloudKit record) that wasn't involved in
// the latest backup export.
[self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
NSMutableSet<NSString *> *obsoleteRecordNames = [NSMutableSet new];
[obsoleteRecordNames addObjectsFromArray:[transaction allKeysInCollection:[OWSBackupFragment collection]]];
[obsoleteRecordNames minusSet:activeRecordNames];
[transaction removeObjectsForKeys:obsoleteRecordNames.allObjects inCollection:[OWSBackupFragment collection]];
}];
}
- (void)cleanUpCloudWithActiveRecordNames:(NSSet<NSString *> *)activeRecordNames
completion:(OWSBackupJobCompletion)completion
{
OWSAssert(activeRecordNames.count > 0);
OWSAssert(completion);
if (self.isComplete) {
// Job was aborted.
return completion(nil);
}
__weak OWSBackupExportJob *weakSelf = self;
[OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray<NSString *> *recordNames) {

@ -20,6 +20,10 @@ NS_ASSUME_NONNULL_BEGIN
- (instancetype)initWithJobTempDirPath:(NSString *)jobTempDirPath;
- (NSString *)generateTempFilePath;
- (nullable NSString *)createTempFile;
#pragma mark - Encrypt
- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath;

@ -44,6 +44,21 @@ static const compression_algorithm SignalCompressionAlgorithm = COMPRESSION_LZMA
return self;
}
- (NSString *)generateTempFilePath
{
return [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString];
}
- (nullable NSString *)createTempFile
{
NSString *filePath = [self generateTempFilePath];
if (![OWSFileSystem ensureFileExists:filePath]) {
OWSProdLogAndFail(@"%@ could not create temp file.", self.logTag);
return nil;
}
return filePath;
}
#pragma mark - Encrypt
- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath
@ -92,7 +107,10 @@ static const compression_algorithm SignalCompressionAlgorithm = COMPRESSION_LZMA
// TODO: Encrypt the data using key;
NSData *encryptedData = unencryptedData;
NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString];
NSString *_Nullable dstFilePath = [self createTempFile];
if (!dstFilePath) {
return nil;
}
NSError *error;
BOOL success = [encryptedData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error];
if (!success || error) {

@ -26,8 +26,8 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
@property (nonatomic) OWSBackupIO *backupIO;
@property (nonatomic) NSArray<OWSBackupManifestItem *> *databaseItems;
@property (nonatomic) NSArray<OWSBackupManifestItem *> *attachmentsItems;
@property (nonatomic) NSArray<OWSBackupFragment *> *databaseItems;
@property (nonatomic) NSArray<OWSBackupFragment *> *attachmentsItems;
@end
@ -105,10 +105,17 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
OWSAssert(self.databaseItems);
OWSAssert(self.attachmentsItems);
NSMutableArray<OWSBackupManifestItem *> *allItems = [NSMutableArray new];
NSMutableArray<OWSBackupFragment *> *allItems = [NSMutableArray new];
[allItems addObjectsFromArray:self.databaseItems];
[allItems addObjectsFromArray:self.attachmentsItems];
// Record metadata for all items, so that we can re-use them in incremental backups after the restore.
[self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
for (OWSBackupFragment *item in allItems) {
[item saveWithTransaction:transaction];
}
}];
__weak OWSBackupImportJob *weakSelf = self;
[weakSelf
downloadFilesFromCloud:allItems
@ -122,12 +129,6 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
return;
}
[weakSelf restoreAttachmentFiles];
if (weakSelf.isComplete) {
return;
}
[weakSelf restoreDatabaseWithCompletion:^(BOOL restoreDatabaseSuccess) {
if (!restoreDatabaseSuccess) {
[weakSelf
@ -154,6 +155,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
return;
}
[weakSelf restoreAttachmentFiles];
if (weakSelf.isComplete) {
return;
}
// Kick off lazy restore.
[OWSBackupLazyRestoreJob runAsync];
[weakSelf succeed];
}];
}];
@ -174,7 +184,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
return YES;
}
- (void)downloadFilesFromCloud:(NSMutableArray<OWSBackupManifestItem *> *)items
- (void)downloadFilesFromCloud:(NSMutableArray<OWSBackupFragment *> *)items
completion:(OWSBackupJobCompletion)completion
{
OWSAssert(items.count > 0);
@ -185,7 +195,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
[self downloadNextItemFromCloud:items recordCount:items.count completion:completion];
}
- (void)downloadNextItemFromCloud:(NSMutableArray<OWSBackupManifestItem *> *)items
- (void)downloadNextItemFromCloud:(NSMutableArray<OWSBackupFragment *> *)items
recordCount:(NSUInteger)recordCount
completion:(OWSBackupJobCompletion)completion
{
@ -201,7 +211,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
// All downloads are complete; exit.
return completion(nil);
}
OWSBackupManifestItem *item = items.lastObject;
OWSBackupFragment *item = items.lastObject;
[items removeLastObject];
CGFloat progress = (recordCount > 0 ? ((recordCount - items.count) / (CGFloat)recordCount) : 0.f);
@ -209,7 +219,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
@"Indicates that the backup import data is being downloaded.")
progress:@(progress)];
// Use a predictable file path so that multiple "import backup" attempts
// TODO: Use a predictable file path so that multiple "import backup" attempts
// will leverage successful file downloads from previous attempts.
//
// TODO: This will also require imports using a predictable jobTempDirPath.
@ -248,53 +258,44 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
{
DDLogVerbose(@"%@ %s: %zd", self.logTag, __PRETTY_FUNCTION__, self.attachmentsItems.count);
NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder];
NSUInteger count = 0;
for (OWSBackupManifestItem *item in self.attachmentsItems) {
if (self.isComplete) {
return;
}
if (item.recordName.length < 1) {
DDLogError(@"%@ attachment was not downloaded.", self.logTag);
// Attachment-related errors are recoverable and can be ignored.
continue;
}
if (item.relativeFilePath.length < 1) {
DDLogError(@"%@ attachment missing relative file path.", self.logTag);
// Attachment-related errors are recoverable and can be ignored.
continue;
}
count++;
[self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_FILES",
@"Indicates that the backup import data is being restored.")
progress:@(count / (CGFloat)self.attachmentsItems.count)];
NSString *dstFilePath = [attachmentsDirPath stringByAppendingPathComponent:item.relativeFilePath];
if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) {
DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath);
continue;
}
NSString *dstDirPath = [dstFilePath stringByDeletingLastPathComponent];
if (![NSFileManager.defaultManager fileExistsAtPath:dstDirPath]) {
if (![OWSFileSystem ensureDirectoryExists:dstDirPath]) {
DDLogError(@"%@ couldn't create directory for file restore: %@.", self.logTag, dstFilePath);
__block NSUInteger count = 0;
YapDatabaseConnection *dbConnection = self.primaryStorage.newDatabaseConnection;
[dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
for (OWSBackupFragment *item in self.attachmentsItems) {
if (self.isComplete) {
return;
}
if (item.recordName.length < 1) {
DDLogError(@"%@ attachment was not downloaded.", self.logTag);
// Attachment-related errors are recoverable and can be ignored.
continue;
}
}
@autoreleasepool {
if (![self.backupIO decryptFileAsFile:item.downloadFilePath
dstFilePath:dstFilePath
encryptionKey:item.encryptionKey]) {
DDLogError(@"%@ attachment could not be restored.", self.logTag);
if (item.attachmentId.length < 1) {
DDLogError(@"%@ attachment missing attachment id.", self.logTag);
// Attachment-related errors are recoverable and can be ignored.
continue;
}
if (item.relativeFilePath.length < 1) {
DDLogError(@"%@ attachment missing relative file path.", self.logTag);
// Attachment-related errors are recoverable and can be ignored.
continue;
}
TSAttachmentStream *_Nullable attachment =
[TSAttachmentStream fetchObjectWithUniqueID:item.attachmentId transaction:transaction];
if (!attachment) {
DDLogError(@"%@ attachment to restore could not be found.", self.logTag);
// Attachment-related errors are recoverable and can be ignored.
continue;
}
[attachment markForLazyRestoreWithFragment:item transaction:transaction];
count++;
[self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_FILES",
@"Indicates that the backup import data is being restored.")
progress:@(count / (CGFloat)self.attachmentsItems.count)];
}
}];
DDLogError(@"%@ restored file: %@.", self.logTag, item.relativeFilePath);
}
DDLogError(@"%@ enqueued lazy restore of %zd files.", self.logTag, count);
}
- (void)restoreDatabaseWithCompletion:(OWSBackupJobBoolCompletion)completion
@ -350,7 +351,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
}
NSUInteger count = 0;
for (OWSBackupManifestItem *item in self.databaseItems) {
for (OWSBackupFragment *item in self.databaseItems) {
if (self.isComplete) {
return;
}

@ -2,6 +2,9 @@
// Copyright (c) 2018 Open Whisper Systems. All rights reserved.
//
#import "TSYapDatabaseObject.h"
#import <SignalServiceKit/OWSBackupFragment.h>
NS_ASSUME_NONNULL_BEGIN
extern NSString *const kOWSBackup_ManifestKey_DatabaseFiles;
@ -9,6 +12,7 @@ extern NSString *const kOWSBackup_ManifestKey_AttachmentFiles;
extern NSString *const kOWSBackup_ManifestKey_RecordName;
extern NSString *const kOWSBackup_ManifestKey_EncryptionKey;
extern NSString *const kOWSBackup_ManifestKey_RelativeFilePath;
extern NSString *const kOWSBackup_ManifestKey_AttachmentId;
extern NSString *const kOWSBackup_ManifestKey_DataSize;
@class OWSBackupIO;
@ -20,31 +24,10 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error);
typedef void (^OWSBackupJobManifestSuccess)(OWSBackupManifestContents *manifest);
typedef void (^OWSBackupJobManifestFailure)(NSError *error);
@interface OWSBackupManifestItem : NSObject
@property (nonatomic) NSString *recordName;
@property (nonatomic) NSData *encryptionKey;
// This property is only set for certain types of manifest item,
// namely attachments where we need to know where the attachment's
// file should reside relative to the attachments folder.
@property (nonatomic, nullable) NSString *relativeFilePath;
// This property is only set if the manifest item is downloaded.
@property (nonatomic, nullable) NSString *downloadFilePath;
// This property is only set if the manifest item is compressed.
@property (nonatomic, nullable) NSNumber *uncompressedDataLength;
@end
#pragma mark -
@interface OWSBackupManifestContents : NSObject
@property (nonatomic) NSArray<OWSBackupManifestItem *> *databaseItems;
@property (nonatomic) NSArray<OWSBackupManifestItem *> *attachmentsItems;
@property (nonatomic) NSArray<OWSBackupFragment *> *databaseItems;
@property (nonatomic) NSArray<OWSBackupFragment *> *attachmentsItems;
@end

@ -16,16 +16,11 @@ NSString *const kOWSBackup_ManifestKey_AttachmentFiles = @"attachment_files";
NSString *const kOWSBackup_ManifestKey_RecordName = @"record_name";
NSString *const kOWSBackup_ManifestKey_EncryptionKey = @"encryption_key";
NSString *const kOWSBackup_ManifestKey_RelativeFilePath = @"relative_file_path";
NSString *const kOWSBackup_ManifestKey_AttachmentId = @"attachment_id";
NSString *const kOWSBackup_ManifestKey_DataSize = @"data_size";
NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
@implementation OWSBackupManifestItem
@end
#pragma mark -
@implementation OWSBackupManifestContents
@end
@ -86,16 +81,12 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
// might want to use a predictable directory so that repeated
// import attempts can reuse downloads from previous attempts.
NSString *temporaryDirectory = NSTemporaryDirectory();
self.jobTempDirPath = [temporaryDirectory stringByAppendingString:[NSUUID UUID].UUIDString];
self.jobTempDirPath = [temporaryDirectory stringByAppendingPathComponent:[NSUUID UUID].UUIDString];
if (![OWSFileSystem ensureDirectoryExists:self.jobTempDirPath]) {
OWSProdLogAndFail(@"%@ Could not create jobTempDirPath.", self.logTag);
return NO;
}
if (![OWSFileSystem protectFileOrFolderAtPath:self.jobTempDirPath]) {
OWSProdLogAndFail(@"%@ Could not protect jobTempDirPath.", self.logTag);
return NO;
}
return YES;
}
@ -226,12 +217,12 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
DDLogVerbose(@"%@ json: %@", self.logTag, json);
NSArray<OWSBackupManifestItem *> *_Nullable databaseItems =
NSArray<OWSBackupFragment *> *_Nullable databaseItems =
[self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles];
if (!databaseItems) {
return failure();
}
NSArray<OWSBackupManifestItem *> *_Nullable attachmentsItems =
NSArray<OWSBackupFragment *> *_Nullable attachmentsItems =
[self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles];
if (!attachmentsItems) {
return failure();
@ -244,7 +235,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
return success(contents);
}
- (nullable NSArray<OWSBackupManifestItem *> *)parseItems:(id)json key:(NSString *)key
- (nullable NSArray<OWSBackupFragment *> *)parseItems:(id)json key:(NSString *)key
{
OWSAssert(json);
OWSAssert(key.length);
@ -258,7 +249,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key);
return nil;
}
NSMutableArray<OWSBackupManifestItem *> *items = [NSMutableArray new];
NSMutableArray<OWSBackupFragment *> *items = [NSMutableArray new];
for (NSDictionary *itemMap in itemMaps) {
if (![itemMap isKindOfClass:[NSDictionary class]]) {
OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key);
@ -267,6 +258,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
NSString *_Nullable recordName = itemMap[kOWSBackup_ManifestKey_RecordName];
NSString *_Nullable encryptionKeyString = itemMap[kOWSBackup_ManifestKey_EncryptionKey];
NSString *_Nullable relativeFilePath = itemMap[kOWSBackup_ManifestKey_RelativeFilePath];
NSString *_Nullable attachmentId = itemMap[kOWSBackup_ManifestKey_AttachmentId];
NSNumber *_Nullable uncompressedDataLength = itemMap[kOWSBackup_ManifestKey_DataSize];
if (![recordName isKindOfClass:[NSString class]]) {
OWSProdLogAndFail(@"%@ manifest has invalid recordName: %@.", self.logTag, key);
@ -281,6 +273,11 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
OWSProdLogAndFail(@"%@ manifest has invalid relativeFilePath: %@.", self.logTag, key);
return nil;
}
// attachmentId is an optional field.
if (attachmentId && ![attachmentId isKindOfClass:[NSString class]]) {
OWSProdLogAndFail(@"%@ manifest has invalid attachmentId: %@.", self.logTag, key);
return nil;
}
NSData *_Nullable encryptionKey = [NSData dataFromBase64String:encryptionKeyString];
if (!encryptionKey) {
OWSProdLogAndFail(@"%@ manifest has corrupt encryptionKey: %@.", self.logTag, key);
@ -292,10 +289,11 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
return nil;
}
OWSBackupManifestItem *item = [OWSBackupManifestItem new];
OWSBackupFragment *item = [OWSBackupFragment new];
item.recordName = recordName;
item.encryptionKey = encryptionKey;
item.relativeFilePath = relativeFilePath;
item.attachmentId = attachmentId;
item.uncompressedDataLength = uncompressedDataLength;
[items addObject:item];
}

@ -0,0 +1,92 @@
//
// Copyright (c) 2018 Open Whisper Systems. All rights reserved.
//
import Foundation
import PromiseKit
import SignalServiceKit
@objc
public class OWSBackupLazyRestoreJob: NSObject {
let primaryStorage: OWSPrimaryStorage
private var jobTempDirPath: String?
deinit {
if let jobTempDirPath = self.jobTempDirPath {
DispatchQueue.global().async {
OWSFileSystem.deleteFile(jobTempDirPath)
}
}
}
@objc
public class func runAsync() {
OWSBackupLazyRestoreJob().runAsync()
}
public override init() {
self.primaryStorage = OWSPrimaryStorage.shared()
}
private func runAsync() {
AssertIsOnMainThread()
DispatchQueue.global().async {
self.restoreAttachments()
}
}
private func restoreAttachments() {
let temporaryDirectory = NSTemporaryDirectory()
let jobTempDirPath = (temporaryDirectory as NSString).appendingPathComponent(NSUUID().uuidString)
guard OWSFileSystem.ensureDirectoryExists(jobTempDirPath) else {
Logger.error("\(logTag) could not create temp directory.")
return
}
self.jobTempDirPath = jobTempDirPath
let backupIO = OWSBackupIO(jobTempDirPath: jobTempDirPath)
let attachmentIds = OWSBackup.shared().attachmentIdsForLazyRestore()
guard attachmentIds.count > 0 else {
Logger.info("\(logTag) No attachments need lazy restore.")
return
}
Logger.info("\(logTag) Lazy restoring \(attachmentIds.count) attachments.")
self.tryToRestoreNextAttachment(attachmentIds: attachmentIds, backupIO: backupIO)
}
private func tryToRestoreNextAttachment(attachmentIds: [String], backupIO: OWSBackupIO) {
var attachmentIdsCopy = attachmentIds
guard let attachmentId = attachmentIdsCopy.last else {
// This job is done.
Logger.verbose("\(logTag) job is done.")
return
}
attachmentIdsCopy.removeLast()
guard let attachment = TSAttachmentStream.fetch(uniqueId: attachmentId) else {
Logger.warn("\(logTag) could not load attachment.")
// Not necessarily an error.
// The attachment might have been deleted since the job began.
// Continue trying to restore the other attachments.
tryToRestoreNextAttachment(attachmentIds: attachmentIds, backupIO: backupIO)
return
}
OWSBackup.shared().lazyRestoreAttachment(attachment,
backupIO: backupIO,
completion: { (success) in
if success {
Logger.info("\(self.logTag) restored attachment.")
} else {
Logger.warn("\(self.logTag) could not restore attachment.")
}
// Continue trying to restore the other attachments.
self.tryToRestoreNextAttachment(attachmentIds: attachmentIdsCopy, backupIO: backupIO)
})
}
}

@ -3,6 +3,7 @@
//
#import "DataSource.h"
#import "OWSBackupFragment.h"
#import "TSAttachment.h"
#if TARGET_OS_IPHONE
@ -62,6 +63,17 @@ NS_ASSUME_NONNULL_BEGIN
+ (nullable NSError *)migrateToSharedData;
// Non-nil for attachments which need "lazy backup restore."
- (nullable OWSBackupFragment *)lazyRestoreFragment;
#pragma mark - Update With... Methods
// Marks attachment as needing "lazy backup restore."
- (void)markForLazyRestoreWithFragment:(OWSBackupFragment *)lazyRestoreFragment
transaction:(YapDatabaseReadWriteTransaction *)transaction;
// Marks attachment as having completed "lazy backup restore."
- (void)updateWithLazyRestoreComplete;
@end
NS_ASSUME_NONNULL_END

@ -26,6 +26,9 @@ NS_ASSUME_NONNULL_BEGIN
// This property should only be accessed on the main thread.
@property (nullable, nonatomic) NSNumber *cachedAudioDurationSeconds;
// Optional property. Only set for attachments which need "lazy backup restore."
@property (nonatomic, nullable) NSString *lazyRestoreFragmentId;
@end
#pragma mark -
@ -359,7 +362,9 @@ NS_ASSUME_NONNULL_BEGIN
}
if (![[NSFileManager defaultManager] fileExistsAtPath:self.mediaURL.path]) {
OWSFail(@"%@ while generating thumbnail, source file doesn't exist: %@", self.logTag, self.mediaURL);
DDLogError(@"%@ while generating thumbnail, source file doesn't exist: %@", self.logTag, self.mediaURL);
// If we're not lazy-restoring this message, the attachment should exist on disk.
OWSAssert(self.lazyRestoreFragmentId);
return;
}
@ -610,6 +615,44 @@ NS_ASSUME_NONNULL_BEGIN
return audioDurationSeconds;
}
- (nullable OWSBackupFragment *)lazyRestoreFragment
{
if (!self.lazyRestoreFragmentId) {
return nil;
}
return [OWSBackupFragment fetchObjectWithUniqueID:self.lazyRestoreFragmentId];
}
#pragma mark - Update With... Methods
- (void)markForLazyRestoreWithFragment:(OWSBackupFragment *)lazyRestoreFragment
transaction:(YapDatabaseReadWriteTransaction *)transaction
{
OWSAssert(lazyRestoreFragment);
OWSAssert(transaction);
if (!lazyRestoreFragment.uniqueId) {
// If metadata hasn't been saved yet, save now.
[lazyRestoreFragment saveWithTransaction:transaction];
OWSAssert(lazyRestoreFragment.uniqueId);
}
[self applyChangeToSelfAndLatestCopy:transaction
changeBlock:^(TSAttachmentStream *attachment) {
[attachment setLazyRestoreFragmentId:lazyRestoreFragment.uniqueId];
}];
}
- (void)updateWithLazyRestoreComplete
{
[self.dbReadWriteConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
[self applyChangeToSelfAndLatestCopy:transaction
changeBlock:^(TSAttachmentStream *attachment) {
[attachment setLazyRestoreFragmentId:nil];
}];
}];
}
@end
NS_ASSUME_NONNULL_END

@ -57,6 +57,7 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage)
[OWSFailedMessagesJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage];
[OWSFailedAttachmentDownloadsJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage];
[OWSMediaGalleryFinder asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage];
[TSDatabaseView asyncRegisterLazyRestoreAttachmentsDatabaseView:storage];
}
#pragma mark -

@ -17,6 +17,9 @@ extern NSString *const TSUnreadDatabaseViewExtensionName;
extern NSString *const TSSecondaryDevicesDatabaseViewExtensionName;
extern NSString *const TSLazyRestoreAttachmentsGroup;
extern NSString *const TSLazyRestoreAttachmentsDatabaseViewExtensionName;
@interface TSDatabaseView : NSObject
- (instancetype)init NS_UNAVAILABLE;
@ -55,4 +58,6 @@ extern NSString *const TSSecondaryDevicesDatabaseViewExtensionName;
+ (void)asyncRegisterSecondaryDevicesDatabaseView:(OWSStorage *)storage;
+ (void)asyncRegisterLazyRestoreAttachmentsDatabaseView:(OWSStorage *)storage;
@end

@ -5,6 +5,8 @@
#import "TSDatabaseView.h"
#import "OWSDevice.h"
#import "OWSReadTracking.h"
#import "TSAttachment.h"
#import "TSAttachmentStream.h"
#import "TSIncomingMessage.h"
#import "TSInvalidIdentityKeyErrorMessage.h"
#import "TSOutgoingMessage.h"
@ -28,6 +30,9 @@ NSString *const TSUnreadDatabaseViewExtensionName = @"TSUnreadDatabaseViewExtens
NSString *const TSUnseenDatabaseViewExtensionName = @"TSUnseenDatabaseViewExtensionName";
NSString *const TSThreadSpecialMessagesDatabaseViewExtensionName = @"TSThreadSpecialMessagesDatabaseViewExtensionName";
NSString *const TSSecondaryDevicesDatabaseViewExtensionName = @"TSSecondaryDevicesDatabaseViewExtensionName";
NSString *const TSLazyRestoreAttachmentsDatabaseViewExtensionName
= @"TSLazyRestoreAttachmentsDatabaseViewExtensionName";
NSString *const TSLazyRestoreAttachmentsGroup = @"TSLazyRestoreAttachmentsGroup";
@interface OWSStorage (TSDatabaseView)
@ -295,30 +300,26 @@ NSString *const TSSecondaryDevicesDatabaseViewExtensionName = @"TSSecondaryDevic
+ (void)asyncRegisterSecondaryDevicesDatabaseView:(OWSStorage *)storage
{
YapDatabaseViewGrouping *viewGrouping =
[YapDatabaseViewGrouping withObjectBlock:^NSString *_Nullable(YapDatabaseReadTransaction *_Nonnull transaction,
NSString *_Nonnull collection,
NSString *_Nonnull key,
id _Nonnull object) {
if ([object isKindOfClass:[OWSDevice class]]) {
OWSDevice *device = (OWSDevice *)object;
if (![device isPrimaryDevice]) {
return TSSecondaryDevicesGroup;
}
YapDatabaseViewGrouping *viewGrouping = [YapDatabaseViewGrouping withObjectBlock:^NSString *_Nullable(
YapDatabaseReadTransaction *transaction, NSString *collection, NSString *key, id object) {
if ([object isKindOfClass:[OWSDevice class]]) {
OWSDevice *device = (OWSDevice *)object;
if (![device isPrimaryDevice]) {
return TSSecondaryDevicesGroup;
}
return nil;
}];
}
return nil;
}];
YapDatabaseViewSorting *viewSorting =
[YapDatabaseViewSorting withObjectBlock:^NSComparisonResult(YapDatabaseReadTransaction *_Nonnull transaction,
NSString *_Nonnull group,
NSString *_Nonnull collection1,
NSString *_Nonnull key1,
id _Nonnull object1,
NSString *_Nonnull collection2,
NSString *_Nonnull key2,
id _Nonnull object2) {
[YapDatabaseViewSorting withObjectBlock:^NSComparisonResult(YapDatabaseReadTransaction *transaction,
NSString *group,
NSString *collection1,
NSString *key1,
id object1,
NSString *collection2,
NSString *key2,
id object2) {
if ([object1 isKindOfClass:[OWSDevice class]] && [object2 isKindOfClass:[OWSDevice class]]) {
OWSDevice *device1 = (OWSDevice *)object1;
OWSDevice *device2 = (OWSDevice *)object2;
@ -341,6 +342,58 @@ NSString *const TSSecondaryDevicesDatabaseViewExtensionName = @"TSSecondaryDevic
[storage asyncRegisterExtension:view withName:TSSecondaryDevicesDatabaseViewExtensionName];
}
+ (void)asyncRegisterLazyRestoreAttachmentsDatabaseView:(OWSStorage *)storage
{
YapDatabaseViewGrouping *viewGrouping = [YapDatabaseViewGrouping withObjectBlock:^NSString *_Nullable(
YapDatabaseReadTransaction *transaction, NSString *collection, NSString *key, id object) {
if (![object isKindOfClass:[TSAttachment class]]) {
OWSProdLogAndFail(@"%@ Unexpected entity %@ in collection: %@", self.logTag, [object class], collection);
return nil;
}
if (![object isKindOfClass:[TSAttachmentStream class]]) {
return nil;
}
TSAttachmentStream *attachmentStream = (TSAttachmentStream *)object;
if (attachmentStream.lazyRestoreFragment) {
return TSLazyRestoreAttachmentsGroup;
} else {
return nil;
}
}];
YapDatabaseViewSorting *viewSorting = [YapDatabaseViewSorting withObjectBlock:^NSComparisonResult(
YapDatabaseReadTransaction *transaction,
NSString *group,
NSString *collection1,
NSString *key1,
id object1,
NSString *collection2,
NSString *key2,
id object2) {
if (![object1 isKindOfClass:[TSAttachment class]]) {
OWSProdLogAndFail(@"%@ Unexpected entity %@ in collection: %@", self.logTag, [object1 class], collection1);
return NSOrderedSame;
}
if (![object2 isKindOfClass:[TSAttachment class]]) {
OWSProdLogAndFail(@"%@ Unexpected entity %@ in collection: %@", self.logTag, [object2 class], collection2);
return NSOrderedSame;
}
// Specific ordering doesn't matter; we just need a stable ordering.
TSAttachmentStream *attachmentStream1 = (TSAttachmentStream *)object1;
TSAttachmentStream *attachmentStream2 = (TSAttachmentStream *)object2;
return [attachmentStream2.creationTimestamp compare:attachmentStream1.creationTimestamp];
}];
YapDatabaseViewOptions *options = [YapDatabaseViewOptions new];
options.isPersistent = YES;
options.allowedCollections =
[[YapWhitelistBlacklist alloc] initWithWhitelist:[NSSet setWithObject:[TSAttachment collection]]];
YapDatabaseView *view =
[[YapDatabaseAutoView alloc] initWithGrouping:viewGrouping sorting:viewSorting versionTag:@"2" options:options];
[storage asyncRegisterExtension:view withName:TSLazyRestoreAttachmentsDatabaseViewExtensionName];
}
+ (id)unseenDatabaseViewExtension:(YapDatabaseReadTransaction *)transaction
{
OWSAssert(transaction);

@ -0,0 +1,42 @@
//
// Copyright (c) 2018 Open Whisper Systems. All rights reserved.
//
#import "TSYapDatabaseObject.h"
NS_ASSUME_NONNULL_BEGIN
// We store metadata for known backup fragments (i.e. CloudKit record) in
// the database. We might learn about them from:
//
// * Past backup exports.
// * An import downloading and parsing the manifest of the last complete backup.
//
// Storing this data in the database provides continuity.
//
// * Backup exports can reuse fragments from previous Backup exports even if they
// don't complete (i.e. backup export resume).
// * Backup exports can reuse fragments from the backup import, if any.
@interface OWSBackupFragment : TSYapDatabaseObject
@property (nonatomic) NSString *recordName;
@property (nonatomic) NSData *encryptionKey;
// This property is only set for certain types of manifest item,
// namely attachments where we need to know where the attachment's
// file should reside relative to the attachments folder.
@property (nonatomic, nullable) NSString *relativeFilePath;
// This property is only set for attachments.
@property (nonatomic, nullable) NSString *attachmentId;
// This property is only set if the manifest item is downloaded.
@property (nonatomic, nullable) NSString *downloadFilePath;
// This property is only set if the manifest item is compressed.
@property (nonatomic, nullable) NSNumber *uncompressedDataLength;
@end
NS_ASSUME_NONNULL_END

@ -0,0 +1,23 @@
//
// Copyright (c) 2018 Open Whisper Systems. All rights reserved.
//
#import "OWSBackupFragment.h"
NS_ASSUME_NONNULL_BEGIN
@implementation OWSBackupFragment
- (void)saveWithTransaction:(YapDatabaseReadWriteTransaction *)transaction
{
OWSAssert(self.recordName.length > 0);
if (!self.uniqueId) {
self.uniqueId = self.recordName;
}
[super saveWithTransaction:transaction];
}
@end
NS_ASSUME_NONNULL_END

@ -28,6 +28,8 @@ NS_ASSUME_NONNULL_BEGIN
// Returns NO IFF the directory does not exist and could not be created.
+ (BOOL)ensureDirectoryExists:(NSString *)dirPath;
+ (BOOL)ensureFileExists:(NSString *)filePath;
+ (BOOL)deleteFile:(NSString *)filePath;
+ (BOOL)deleteFileIfExists:(NSString *)filePath;

@ -227,6 +227,21 @@ NS_ASSUME_NONNULL_BEGIN
}
}
+ (BOOL)ensureFileExists:(NSString *)filePath
{
BOOL exists = [[NSFileManager defaultManager] fileExistsAtPath:filePath];
if (exists) {
return [self protectFileOrFolderAtPath:filePath];
} else {
BOOL success = [[NSFileManager defaultManager] createFileAtPath:filePath contents:nil attributes:nil];
if (!success) {
OWSFail(@"%@ Failed to create file.", self.logTag);
return NO;
}
return [self protectFileOrFolderAtPath:filePath];
}
}
+ (BOOL)deleteFile:(NSString *)filePath
{
NSError *error;

Loading…
Cancel
Save