Add local cache of backup fragment metadata.

pull/1/head
Matthew Chen 7 years ago
parent e88f5643f7
commit 61dc2c0249

@ -171,7 +171,7 @@ NS_ASSUME_NONNULL_BEGIN
[OWSPrimaryStorage.sharedManager.newDatabaseConnection [OWSPrimaryStorage.sharedManager.newDatabaseConnection
readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
[transaction removeAllObjectsInCollection:[OWSBackupManifestItem collection]]; [transaction removeAllObjectsInCollection:[OWSBackupFragment collection]];
}]; }];
} }

@ -507,14 +507,14 @@ NS_ASSUME_NONNULL_BEGIN
return; return;
} }
TSAttachmentStream *attachmentStream = object; TSAttachmentStream *attachmentStream = object;
if (!attachmentStream.backupRestoreMetadata) { if (!attachmentStream.lazyRestoreFragment) {
OWSProdLogAndFail(@"%@ Invalid object: %@ in collection:%@", OWSProdLogAndFail(@"%@ Invalid object: %@ in collection:%@",
self.logTag, self.logTag,
[object class], [object class],
collection); collection);
return; return;
} }
[recordNames addObject:attachmentStream.backupRestoreMetadata.recordName]; [recordNames addObject:attachmentStream.lazyRestoreFragment.recordName];
}]; }];
}]; }];
return recordNames; return recordNames;
@ -556,13 +556,13 @@ NS_ASSUME_NONNULL_BEGIN
return completion(NO); return completion(NO);
} }
OWSBackupManifestItem *_Nullable backupRestoreMetadata = attachment.backupRestoreMetadata; OWSBackupFragment *_Nullable lazyRestoreFragment = attachment.lazyRestoreFragment;
if (!backupRestoreMetadata) { if (!lazyRestoreFragment) {
DDLogWarn(@"%@ Attachment missing lazy restore metadata.", self.logTag); DDLogWarn(@"%@ Attachment missing lazy restore metadata.", self.logTag);
return completion(NO); return completion(NO);
} }
if (backupRestoreMetadata.recordName.length < 1 || backupRestoreMetadata.encryptionKey.length < 1) { if (lazyRestoreFragment.recordName.length < 1 || lazyRestoreFragment.encryptionKey.length < 1) {
DDLogError(@"%@ Incomplete attachment metadata.", self.logTag); DDLogError(@"%@ Incomplete lazy restore metadata.", self.logTag);
return completion(NO); return completion(NO);
} }
@ -575,14 +575,14 @@ NS_ASSUME_NONNULL_BEGIN
return completion(NO); return completion(NO);
} }
[OWSBackupAPI downloadFileFromCloudWithRecordName:backupRestoreMetadata.recordName [OWSBackupAPI downloadFileFromCloudWithRecordName:lazyRestoreFragment.recordName
toFileUrl:[NSURL fileURLWithPath:tempFilePath] toFileUrl:[NSURL fileURLWithPath:tempFilePath]
success:^{ success:^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[self lazyRestoreAttachment:attachment [self lazyRestoreAttachment:attachment
backupIO:backupIO backupIO:backupIO
encryptedFilePath:tempFilePath encryptedFilePath:tempFilePath
encryptionKey:backupRestoreMetadata.encryptionKey encryptionKey:lazyRestoreFragment.encryptionKey
completion:completion]; completion:completion];
}); });
} }
@ -634,7 +634,7 @@ NS_ASSUME_NONNULL_BEGIN
return completion(NO); return completion(NO);
} }
[attachment updateWithBackupRestoreComplete]; [attachment updateWithLazyRestoreComplete];
completion(YES); completion(YES);
} }

@ -743,17 +743,16 @@ NS_ASSUME_NONNULL_BEGIN
// this record's metadata. // this record's metadata.
// * That this record does in fact exist in our CloudKit database. // * That this record does in fact exist in our CloudKit database.
NSString *lastRecordName = [OWSBackupAPI recordNameForPersistentFileWithFileId:attachmentExport.attachmentId]; NSString *lastRecordName = [OWSBackupAPI recordNameForPersistentFileWithFileId:attachmentExport.attachmentId];
OWSBackupManifestItem *_Nullable lastManifestItem = OWSBackupFragment *_Nullable lastBackupFragment = [OWSBackupFragment fetchObjectWithUniqueID:lastRecordName];
[OWSBackupManifestItem fetchObjectWithUniqueID:lastRecordName]; if (lastBackupFragment && [self.lastValidRecordNames containsObject:lastRecordName]) {
if (lastManifestItem && [self.lastValidRecordNames containsObject:lastRecordName]) { OWSAssert(lastBackupFragment.encryptionKey.length > 0);
OWSAssert(lastManifestItem.encryptionKey.length > 0); OWSAssert(lastBackupFragment.relativeFilePath.length > 0);
OWSAssert(lastManifestItem.relativeFilePath.length > 0);
// Recycle the metadata from the last backup's manifest. // Recycle the metadata from the last backup's manifest.
OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new]; OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new];
encryptedItem.encryptionKey = lastManifestItem.encryptionKey; encryptedItem.encryptionKey = lastBackupFragment.encryptionKey;
attachmentExport.encryptedItem = encryptedItem; attachmentExport.encryptedItem = encryptedItem;
attachmentExport.relativeFilePath = lastManifestItem.relativeFilePath; attachmentExport.relativeFilePath = lastBackupFragment.relativeFilePath;
OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
exportItem.encryptedItem = attachmentExport.encryptedItem; exportItem.encryptedItem = attachmentExport.encryptedItem;
@ -814,12 +813,12 @@ NS_ASSUME_NONNULL_BEGIN
[strongSelf.savedAttachmentItems addObject:exportItem]; [strongSelf.savedAttachmentItems addObject:exportItem];
// Immediately save the record metadata to facilitate export resume. // Immediately save the record metadata to facilitate export resume.
OWSBackupManifestItem *backupRestoreMetadata = [OWSBackupManifestItem new]; OWSBackupFragment *backupFragment = [OWSBackupFragment new];
backupRestoreMetadata.recordName = recordName; backupFragment.recordName = recordName;
backupRestoreMetadata.encryptionKey = exportItem.encryptedItem.encryptionKey; backupFragment.encryptionKey = exportItem.encryptedItem.encryptionKey;
backupRestoreMetadata.relativeFilePath = attachmentExport.relativeFilePath; backupFragment.relativeFilePath = attachmentExport.relativeFilePath;
backupRestoreMetadata.uncompressedDataLength = exportItem.uncompressedDataLength; backupFragment.uncompressedDataLength = exportItem.uncompressedDataLength;
[backupRestoreMetadata save]; [backupFragment save];
DDLogVerbose(@"%@ saved attachment: %@ as %@", DDLogVerbose(@"%@ saved attachment: %@ as %@",
self.logTag, self.logTag,
@ -992,11 +991,10 @@ NS_ASSUME_NONNULL_BEGIN
// the latest backup export. // the latest backup export.
[self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { [self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
NSMutableSet<NSString *> *obsoleteRecordNames = [NSMutableSet new]; NSMutableSet<NSString *> *obsoleteRecordNames = [NSMutableSet new];
[obsoleteRecordNames addObjectsFromArray:[transaction allKeysInCollection:[OWSBackupManifestItem collection]]]; [obsoleteRecordNames addObjectsFromArray:[transaction allKeysInCollection:[OWSBackupFragment collection]]];
[obsoleteRecordNames minusSet:activeRecordNames]; [obsoleteRecordNames minusSet:activeRecordNames];
[transaction removeObjectsForKeys:obsoleteRecordNames.allObjects [transaction removeObjectsForKeys:obsoleteRecordNames.allObjects inCollection:[OWSBackupFragment collection]];
inCollection:[OWSBackupManifestItem collection]];
}]; }];
} }

@ -26,8 +26,8 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
@property (nonatomic) OWSBackupIO *backupIO; @property (nonatomic) OWSBackupIO *backupIO;
@property (nonatomic) NSArray<OWSBackupManifestItem *> *databaseItems; @property (nonatomic) NSArray<OWSBackupFragment *> *databaseItems;
@property (nonatomic) NSArray<OWSBackupManifestItem *> *attachmentsItems; @property (nonatomic) NSArray<OWSBackupFragment *> *attachmentsItems;
@end @end
@ -105,13 +105,13 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
OWSAssert(self.databaseItems); OWSAssert(self.databaseItems);
OWSAssert(self.attachmentsItems); OWSAssert(self.attachmentsItems);
NSMutableArray<OWSBackupManifestItem *> *allItems = [NSMutableArray new]; NSMutableArray<OWSBackupFragment *> *allItems = [NSMutableArray new];
[allItems addObjectsFromArray:self.databaseItems]; [allItems addObjectsFromArray:self.databaseItems];
[allItems addObjectsFromArray:self.attachmentsItems]; [allItems addObjectsFromArray:self.attachmentsItems];
// Record metadata for all items, so that we can re-use them in incremental backups after the restore. // Record metadata for all items, so that we can re-use them in incremental backups after the restore.
[self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { [self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
for (OWSBackupManifestItem *item in allItems) { for (OWSBackupFragment *item in allItems) {
[item saveWithTransaction:transaction]; [item saveWithTransaction:transaction];
} }
}]; }];
@ -181,7 +181,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
return YES; return YES;
} }
- (void)downloadFilesFromCloud:(NSMutableArray<OWSBackupManifestItem *> *)items - (void)downloadFilesFromCloud:(NSMutableArray<OWSBackupFragment *> *)items
completion:(OWSBackupJobCompletion)completion completion:(OWSBackupJobCompletion)completion
{ {
OWSAssert(items.count > 0); OWSAssert(items.count > 0);
@ -192,7 +192,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
[self downloadNextItemFromCloud:items recordCount:items.count completion:completion]; [self downloadNextItemFromCloud:items recordCount:items.count completion:completion];
} }
- (void)downloadNextItemFromCloud:(NSMutableArray<OWSBackupManifestItem *> *)items - (void)downloadNextItemFromCloud:(NSMutableArray<OWSBackupFragment *> *)items
recordCount:(NSUInteger)recordCount recordCount:(NSUInteger)recordCount
completion:(OWSBackupJobCompletion)completion completion:(OWSBackupJobCompletion)completion
{ {
@ -208,7 +208,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
// All downloads are complete; exit. // All downloads are complete; exit.
return completion(nil); return completion(nil);
} }
OWSBackupManifestItem *item = items.lastObject; OWSBackupFragment *item = items.lastObject;
[items removeLastObject]; [items removeLastObject];
CGFloat progress = (recordCount > 0 ? ((recordCount - items.count) / (CGFloat)recordCount) : 0.f); CGFloat progress = (recordCount > 0 ? ((recordCount - items.count) / (CGFloat)recordCount) : 0.f);
@ -258,7 +258,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder];
NSUInteger count = 0; NSUInteger count = 0;
for (OWSBackupManifestItem *item in self.attachmentsItems) { for (OWSBackupFragment *item in self.attachmentsItems) {
if (self.isComplete) { if (self.isComplete) {
return; return;
} }
@ -357,7 +357,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe
} }
NSUInteger count = 0; NSUInteger count = 0;
for (OWSBackupManifestItem *item in self.databaseItems) { for (OWSBackupFragment *item in self.databaseItems) {
if (self.isComplete) { if (self.isComplete) {
return; return;
} }

@ -3,7 +3,7 @@
// //
#import "TSYapDatabaseObject.h" #import "TSYapDatabaseObject.h"
#import <SignalServiceKit/OWSBackupManifestItem.h> #import <SignalServiceKit/OWSBackupFragment.h>
NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_BEGIN
@ -25,8 +25,8 @@ typedef void (^OWSBackupJobManifestFailure)(NSError *error);
@interface OWSBackupManifestContents : NSObject @interface OWSBackupManifestContents : NSObject
@property (nonatomic) NSArray<OWSBackupManifestItem *> *databaseItems; @property (nonatomic) NSArray<OWSBackupFragment *> *databaseItems;
@property (nonatomic) NSArray<OWSBackupManifestItem *> *attachmentsItems; @property (nonatomic) NSArray<OWSBackupFragment *> *attachmentsItems;
@end @end

@ -216,12 +216,12 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
DDLogVerbose(@"%@ json: %@", self.logTag, json); DDLogVerbose(@"%@ json: %@", self.logTag, json);
NSArray<OWSBackupManifestItem *> *_Nullable databaseItems = NSArray<OWSBackupFragment *> *_Nullable databaseItems =
[self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles]; [self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles];
if (!databaseItems) { if (!databaseItems) {
return failure(); return failure();
} }
NSArray<OWSBackupManifestItem *> *_Nullable attachmentsItems = NSArray<OWSBackupFragment *> *_Nullable attachmentsItems =
[self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles]; [self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles];
if (!attachmentsItems) { if (!attachmentsItems) {
return failure(); return failure();
@ -234,7 +234,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
return success(contents); return success(contents);
} }
- (nullable NSArray<OWSBackupManifestItem *> *)parseItems:(id)json key:(NSString *)key - (nullable NSArray<OWSBackupFragment *> *)parseItems:(id)json key:(NSString *)key
{ {
OWSAssert(json); OWSAssert(json);
OWSAssert(key.length); OWSAssert(key.length);
@ -248,7 +248,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key); OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key);
return nil; return nil;
} }
NSMutableArray<OWSBackupManifestItem *> *items = [NSMutableArray new]; NSMutableArray<OWSBackupFragment *> *items = [NSMutableArray new];
for (NSDictionary *itemMap in itemMaps) { for (NSDictionary *itemMap in itemMaps) {
if (![itemMap isKindOfClass:[NSDictionary class]]) { if (![itemMap isKindOfClass:[NSDictionary class]]) {
OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key); OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key);
@ -282,7 +282,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService";
return nil; return nil;
} }
OWSBackupManifestItem *item = [OWSBackupManifestItem new]; OWSBackupFragment *item = [OWSBackupFragment new];
item.recordName = recordName; item.recordName = recordName;
item.encryptionKey = encryptionKey; item.encryptionKey = encryptionKey;
item.relativeFilePath = relativeFilePath; item.relativeFilePath = relativeFilePath;

@ -3,7 +3,7 @@
// //
#import "DataSource.h" #import "DataSource.h"
#import "OWSBackupManifestItem.h" #import "OWSBackupFragment.h"
#import "TSAttachment.h" #import "TSAttachment.h"
#if TARGET_OS_IPHONE #if TARGET_OS_IPHONE
@ -35,9 +35,6 @@ NS_ASSUME_NONNULL_BEGIN
@property (nonatomic, readonly) NSDate *creationTimestamp; @property (nonatomic, readonly) NSDate *creationTimestamp;
// Optional property. Only set for attachments which need "lazy backup restore."
@property (nonatomic, readonly, nullable) NSString *backupRestoreMetadataId;
#if TARGET_OS_IPHONE #if TARGET_OS_IPHONE
- (nullable UIImage *)image; - (nullable UIImage *)image;
- (nullable UIImage *)thumbnailImage; - (nullable UIImage *)thumbnailImage;
@ -66,14 +63,15 @@ NS_ASSUME_NONNULL_BEGIN
+ (nullable NSError *)migrateToSharedData; + (nullable NSError *)migrateToSharedData;
- (nullable OWSBackupManifestItem *)backupRestoreMetadata; // Non-nil for attachments which need "lazy backup restore."
- (nullable OWSBackupFragment *)lazyRestoreFragment;
#pragma mark - Update With... Methods #pragma mark - Update With... Methods
// Marks attachment as needing "lazy backup restore." // Marks attachment as needing "lazy backup restore."
- (void)updateWithBackupRestoreMetadata:(OWSBackupManifestItem *)backupRestoreMetadata; - (void)updateWithLazyRestoreFragment:(OWSBackupFragment *)lazyRestoreFragment;
// Marks attachment as having completed "lazy backup restore." // Marks attachment as having completed "lazy backup restore."
- (void)updateWithBackupRestoreComplete; - (void)updateWithLazyRestoreComplete;
@end @end

@ -26,7 +26,8 @@ NS_ASSUME_NONNULL_BEGIN
// This property should only be accessed on the main thread. // This property should only be accessed on the main thread.
@property (nullable, nonatomic) NSNumber *cachedAudioDurationSeconds; @property (nullable, nonatomic) NSNumber *cachedAudioDurationSeconds;
@property (nonatomic, nullable) NSString *backupRestoreMetadataId; // Optional property. Only set for attachments which need "lazy backup restore."
@property (nonatomic, nullable) NSString *lazyRestoreFragmentId;
@end @end
@ -612,40 +613,40 @@ NS_ASSUME_NONNULL_BEGIN
return audioDurationSeconds; return audioDurationSeconds;
} }
- (nullable OWSBackupManifestItem *)backupRestoreMetadata - (nullable OWSBackupFragment *)lazyRestoreFragment
{ {
if (!self.backupRestoreMetadataId) { if (!self.lazyRestoreFragmentId) {
return nil; return nil;
} }
return [OWSBackupManifestItem fetchObjectWithUniqueID:self.backupRestoreMetadataId]; return [OWSBackupFragment fetchObjectWithUniqueID:self.lazyRestoreFragmentId];
} }
#pragma mark - Update With... Methods #pragma mark - Update With... Methods
- (void)updateWithBackupRestoreMetadata:(OWSBackupManifestItem *)backupRestoreMetadata - (void)updateWithLazyRestoreFragment:(OWSBackupFragment *)lazyRestoreFragment
{ {
OWSAssert(backupRestoreMetadata); OWSAssert(lazyRestoreFragment);
[self.dbReadWriteConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { [self.dbReadWriteConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
if (!backupRestoreMetadata.uniqueId) { if (!lazyRestoreFragment.uniqueId) {
// If metadata hasn't been saved yet, save now. // If metadata hasn't been saved yet, save now.
[backupRestoreMetadata saveWithTransaction:transaction]; [lazyRestoreFragment saveWithTransaction:transaction];
OWSAssert(backupRestoreMetadata.uniqueId); OWSAssert(lazyRestoreFragment.uniqueId);
} }
[self applyChangeToSelfAndLatestCopy:transaction [self applyChangeToSelfAndLatestCopy:transaction
changeBlock:^(TSAttachmentStream *attachment) { changeBlock:^(TSAttachmentStream *attachment) {
[attachment setBackupRestoreMetadataId:backupRestoreMetadata.uniqueId]; [attachment setLazyRestoreFragmentId:lazyRestoreFragment.uniqueId];
}]; }];
}]; }];
} }
- (void)updateWithBackupRestoreComplete - (void)updateWithLazyRestoreComplete
{ {
[self.dbReadWriteConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { [self.dbReadWriteConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
[self applyChangeToSelfAndLatestCopy:transaction [self applyChangeToSelfAndLatestCopy:transaction
changeBlock:^(TSAttachmentStream *attachment) { changeBlock:^(TSAttachmentStream *attachment) {
[attachment setBackupRestoreMetadataId:nil]; [attachment setLazyRestoreFragmentId:nil];
}]; }];
}]; }];
} }

@ -354,7 +354,7 @@ NSString *const TSLazyRestoreAttachmentsGroup = @"TSLazyRestoreAttachmentsGroup"
return nil; return nil;
} }
TSAttachmentStream *attachmentStream = (TSAttachmentStream *)object; TSAttachmentStream *attachmentStream = (TSAttachmentStream *)object;
if (attachmentStream.backupRestoreMetadata) { if (attachmentStream.lazyRestoreFragment) {
return TSLazyRestoreAttachmentsGroup; return TSLazyRestoreAttachmentsGroup;
} else { } else {
return nil; return nil;

@ -17,7 +17,7 @@ NS_ASSUME_NONNULL_BEGIN
// * Backup exports can reuse fragments from previous Backup exports even if they // * Backup exports can reuse fragments from previous Backup exports even if they
// don't complete (i.e. backup export resume). // don't complete (i.e. backup export resume).
// * Backup exports can reuse fragments from the backup import, if any. // * Backup exports can reuse fragments from the backup import, if any.
@interface OWSBackupManifestItem : TSYapDatabaseObject @interface OWSBackupFragment : TSYapDatabaseObject
@property (nonatomic) NSString *recordName; @property (nonatomic) NSString *recordName;

@ -2,11 +2,11 @@
// Copyright (c) 2018 Open Whisper Systems. All rights reserved. // Copyright (c) 2018 Open Whisper Systems. All rights reserved.
// //
#import "OWSBackupManifestItem.h" #import "OWSBackupFragment.h"
NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_BEGIN
@implementation OWSBackupManifestItem @implementation OWSBackupFragment
- (void)saveWithTransaction:(YapDatabaseReadWriteTransaction *)transaction - (void)saveWithTransaction:(YapDatabaseReadWriteTransaction *)transaction
{ {
Loading…
Cancel
Save