From bec7235d57ad604d67b9c62d8c0ebfa769648579 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Thu, 26 Apr 2018 16:17:12 -0400 Subject: [PATCH 1/3] "Bump build to 2.24.1.0." --- Signal/Signal-Info.plist | 4 ++-- SignalShareExtension/Info.plist | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Signal/Signal-Info.plist b/Signal/Signal-Info.plist index 12c62ece3..cea0f19d6 100644 --- a/Signal/Signal-Info.plist +++ b/Signal/Signal-Info.plist @@ -21,7 +21,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 2.24.0 + 2.24.1 CFBundleSignature ???? CFBundleURLTypes @@ -38,7 +38,7 @@ CFBundleVersion - 2.24.0.10 + 2.24.1.0 ITSAppUsesNonExemptEncryption LOGS_EMAIL diff --git a/SignalShareExtension/Info.plist b/SignalShareExtension/Info.plist index 950b147bb..6854536de 100644 --- a/SignalShareExtension/Info.plist +++ b/SignalShareExtension/Info.plist @@ -17,9 +17,9 @@ CFBundlePackageType XPC! CFBundleShortVersionString - 2.24.0 + 2.24.1 CFBundleVersion - 2.24.0.10 + 2.24.1.0 ITSAppUsesNonExemptEncryption NSAppTransportSecurity From c5e7b10bd93dac257a618a4385d51c886724ec28 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Tue, 24 Apr 2018 13:17:37 -0400 Subject: [PATCH 2/3] Merge branch 'charlesmchen/dbMigrationsVsStorageReadiness' --- Signal/src/AppDelegate.m | 17 +++--- SignalMessaging/environment/AppSetup.h | 7 ++- SignalMessaging/environment/AppSetup.m | 21 +++++-- .../environment/VersionMigrations.m | 8 ++- .../migrations/OWSDatabaseMigration.m | 16 ------ SignalServiceKit/src/Storage/OWSStorage.h | 8 +-- SignalServiceKit/src/Storage/OWSStorage.m | 57 +++++++------------ .../ShareViewController.swift | 21 +++---- 8 files changed, 68 insertions(+), 87 deletions(-) diff --git a/Signal/src/AppDelegate.m b/Signal/src/AppDelegate.m index 2f669220c..885bb984e 100644 --- a/Signal/src/AppDelegate.m +++ b/Signal/src/AppDelegate.m @@ -7,6 +7,7 @@ #import "AppUpdateNag.h" #import "CodeVerificationViewController.h" #import "DebugLogger.h" +#import "HomeViewController.h" #import "MainAppContext.h" #import "NotificationsManager.h" #import "OWS2FASettingsViewController.h" @@ -155,11 +156,16 @@ static NSString *const kURLHostVerifyPrefix = @"verify"; // This block will be cleared in storageIsReady. [DeviceSleepManager.sharedInstance addBlockWithBlockObject:self]; - [AppSetup setupEnvironment:^{ + [AppSetup setupEnvironmentWithCallMessageHandlerBlock:^{ return SignalApp.sharedApp.callMessageHandler; } notificationsProtocolBlock:^{ return SignalApp.sharedApp.notificationsManager; + } + migrationCompletion:^{ + OWSAssertIsOnMainThread(); + + [self versionMigrationsDidComplete]; }]; [UIUtil applySignalAppearence]; @@ -175,14 +181,6 @@ static NSString *const kURLHostVerifyPrefix = @"verify"; mainWindow.rootViewController = [self loadingRootViewController]; [mainWindow makeKeyAndVisible]; - // performUpdateCheck must be invoked after Environment has been initialized because - // upgrade process may depend on Environment. - [VersionMigrations performUpdateCheckWithCompletion:^{ - OWSAssertIsOnMainThread(); - - [self versionMigrationsDidComplete]; - }]; - // Accept push notification when app is not open NSDictionary *remoteNotif = launchOptions[UIApplicationLaunchOptionsRemoteNotificationKey]; if (remoteNotif) { @@ -1096,6 +1094,7 @@ static NSString *const kURLHostVerifyPrefix = @"verify"; [AppVersion.instance mainAppLaunchDidComplete]; [Environment.current.contactsManager loadSignalAccountsFromCache]; + [Environment.current.contactsManager startObserving]; // If there were any messages in our local queue which we hadn't yet processed. [[OWSMessageReceiver sharedInstance] handleAnyUnprocessedEnvelopesAsync]; diff --git a/SignalMessaging/environment/AppSetup.h b/SignalMessaging/environment/AppSetup.h index a24695c44..cc8d57eca 100644 --- a/SignalMessaging/environment/AppSetup.h +++ b/SignalMessaging/environment/AppSetup.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2017 Open Whisper Systems. All rights reserved. +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. // NS_ASSUME_NONNULL_BEGIN @@ -13,8 +13,9 @@ typedef id _Nonnull (^NotificationsManagerBlock)(void); // This is _NOT_ a singleton and will be instantiated each time that the SAE is used. @interface AppSetup : NSObject -+ (void)setupEnvironment:(CallMessageHandlerBlock)callMessageHandlerBlock - notificationsProtocolBlock:(NotificationsManagerBlock)notificationsManagerBlock; ++ (void)setupEnvironmentWithCallMessageHandlerBlock:(CallMessageHandlerBlock)callMessageHandlerBlock + notificationsProtocolBlock:(NotificationsManagerBlock)notificationsManagerBlock + migrationCompletion:(dispatch_block_t)migrationCompletion; @end diff --git a/SignalMessaging/environment/AppSetup.m b/SignalMessaging/environment/AppSetup.m index 48f663d83..bf9f37261 100644 --- a/SignalMessaging/environment/AppSetup.m +++ b/SignalMessaging/environment/AppSetup.m @@ -18,11 +18,16 @@ NS_ASSUME_NONNULL_BEGIN @implementation AppSetup -+ (void)setupEnvironment:(CallMessageHandlerBlock)callMessageHandlerBlock - notificationsProtocolBlock:(NotificationsManagerBlock)notificationsManagerBlock ++ (void)setupEnvironmentWithCallMessageHandlerBlock:(CallMessageHandlerBlock)callMessageHandlerBlock + notificationsProtocolBlock:(NotificationsManagerBlock)notificationsManagerBlock + migrationCompletion:(dispatch_block_t)migrationCompletion { OWSAssert(callMessageHandlerBlock); OWSAssert(notificationsManagerBlock); + OWSAssert(migrationCompletion); + + __block OWSBackgroundTask *_Nullable backgroundTask = + [OWSBackgroundTask backgroundTaskWithLabelStr:__PRETTY_FUNCTION__]; static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ @@ -46,8 +51,16 @@ NS_ASSUME_NONNULL_BEGIN [NSKeyedUnarchiver setClass:[OWSUserProfile class] forClassName:[OWSUserProfile collection]]; [NSKeyedUnarchiver setClass:[OWSDatabaseMigration class] forClassName:[OWSDatabaseMigration collection]]; - [OWSStorage setupStorage]; - [[Environment current].contactsManager startObserving]; + [OWSStorage registerExtensionsWithMigrationBlock:^() { + // Don't start database migrations until storage is ready. + [VersionMigrations performUpdateCheckWithCompletion:^() { + OWSAssertIsOnMainThread(); + + migrationCompletion(); + + backgroundTask = nil; + }]; + }]; }); } diff --git a/SignalMessaging/environment/VersionMigrations.m b/SignalMessaging/environment/VersionMigrations.m index 2e4c4d4be..b9312489b 100644 --- a/SignalMessaging/environment/VersionMigrations.m +++ b/SignalMessaging/environment/VersionMigrations.m @@ -33,6 +33,8 @@ NS_ASSUME_NONNULL_BEGIN + (void)performUpdateCheckWithCompletion:(VersionMigrationCompletion)completion { + DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + // performUpdateCheck must be invoked after Environment has been initialized because // upgrade process may depend on Environment. OWSAssert([Environment current]); @@ -86,8 +88,10 @@ NS_ASSUME_NONNULL_BEGIN [self clearBloomFilterCache]; } - [[[OWSDatabaseMigrationRunner alloc] initWithPrimaryStorage:[OWSPrimaryStorage sharedManager]] - runAllOutstandingWithCompletion:completion]; + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + [[[OWSDatabaseMigrationRunner alloc] initWithPrimaryStorage:[OWSPrimaryStorage sharedManager]] + runAllOutstandingWithCompletion:completion]; + }); } + (BOOL)isVersion:(NSString *)thisVersionString diff --git a/SignalMessaging/environment/migrations/OWSDatabaseMigration.m b/SignalMessaging/environment/migrations/OWSDatabaseMigration.m index 7ff231076..90c1e0e1f 100644 --- a/SignalMessaging/environment/migrations/OWSDatabaseMigration.m +++ b/SignalMessaging/environment/migrations/OWSDatabaseMigration.m @@ -58,16 +58,6 @@ NS_ASSUME_NONNULL_BEGIN OWSAssert(completion); OWSDatabaseConnection *dbConnection = (OWSDatabaseConnection *)self.primaryStorage.newDatabaseConnection; - // These migrations won't be run until storage registrations are enqueued, - // but this transaction might begin before all registrations are marked as - // complete, so disable this checking. - // - // TODO: Once we move "app readiness" into AppSetup, we should explicitly - // not start these migrations until storage is ready. We can then remove - // this statement which disables checking. -#ifdef DEBUG - dbConnection.canWriteBeforeStorageReady = YES; -#endif [dbConnection asyncReadWriteWithBlock:^(YapDatabaseReadWriteTransaction *_Nonnull transaction) { [self runUpWithTransaction:transaction]; @@ -88,18 +78,12 @@ NS_ASSUME_NONNULL_BEGIN return self.dbReadWriteConnection; } -// Database migrations need to occur _before_ storage is ready (by definition), -// so we need to use a connection with canWriteBeforeStorageReady set in -// debug builds. + (YapDatabaseConnection *)dbReadWriteConnection { static dispatch_once_t onceToken; static YapDatabaseConnection *sharedDBConnection; dispatch_once(&onceToken, ^{ sharedDBConnection = [OWSPrimaryStorage sharedManager].newDatabaseConnection; - - OWSAssert([sharedDBConnection isKindOfClass:[OWSDatabaseConnection class]]); - ((OWSDatabaseConnection *)sharedDBConnection).canWriteBeforeStorageReady = YES; }); return sharedDBConnection; diff --git a/SignalServiceKit/src/Storage/OWSStorage.h b/SignalServiceKit/src/Storage/OWSStorage.h index 12198be48..6fafcc657 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.h +++ b/SignalServiceKit/src/Storage/OWSStorage.h @@ -22,10 +22,6 @@ extern NSString *const StorageIsReadyNotification; @property (atomic, weak) id delegate; -#ifdef DEBUG -@property (atomic) BOOL canWriteBeforeStorageReady; -#endif - - (instancetype)init NS_UNAVAILABLE; - (instancetype)initWithDatabase:(YapDatabase *)database delegate:(id)delegate NS_DESIGNATED_INITIALIZER; @@ -48,6 +44,8 @@ extern NSString *const StorageIsReadyNotification; #pragma mark - +typedef void (^OWSStorageMigrationBlock)(void); + @interface OWSStorage : NSObject - (instancetype)init NS_UNAVAILABLE; @@ -60,7 +58,7 @@ extern NSString *const StorageIsReadyNotification; // This object can be used to filter database notifications. @property (nonatomic, readonly, nullable) id dbNotificationObject; -+ (void)setupStorage; ++ (void)registerExtensionsWithMigrationBlock:(OWSStorageMigrationBlock)migrationBlock; + (void)resetAllStorage; diff --git a/SignalServiceKit/src/Storage/OWSStorage.m b/SignalServiceKit/src/Storage/OWSStorage.m index 75e97c45e..c56d1c06b 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.m +++ b/SignalServiceKit/src/Storage/OWSStorage.m @@ -74,7 +74,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); { id delegate = self.delegate; OWSAssert(delegate); - OWSAssert(delegate.areAllRegistrationsComplete || self.canWriteBeforeStorageReady); + OWSAssert(delegate.areAllRegistrationsComplete); OWSBackgroundTask *_Nullable backgroundTask = nil; if (CurrentAppContext().isMainApp) { @@ -101,7 +101,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); { id delegate = self.delegate; OWSAssert(delegate); - OWSAssert(delegate.areAllRegistrationsComplete || self.canWriteBeforeStorageReady); + OWSAssert(delegate.areAllRegistrationsComplete); __block OWSBackgroundTask *_Nullable backgroundTask = nil; if (CurrentAppContext().isMainApp) { @@ -176,13 +176,6 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (YapDatabaseConnection *)registrationConnection { YapDatabaseConnection *connection = [super registrationConnection]; - -#ifdef DEBUG - // Flag the registration connection as such. - OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]); - ((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES; -#endif - return connection; } @@ -332,29 +325,24 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); OWS_ABSTRACT_METHOD(); } -+ (NSArray *)allStorages ++ (void)registerExtensionsWithMigrationBlock:(OWSStorageMigrationBlock)migrationBlock { - return @[ - OWSPrimaryStorage.sharedManager, - ]; -} + OWSAssert(migrationBlock); -+ (void)setupStorage -{ __block OWSBackgroundTask *_Nullable backgroundTask = [OWSBackgroundTask backgroundTaskWithLabelStr:__PRETTY_FUNCTION__]; - for (OWSStorage *storage in self.allStorages) { - [storage runSyncRegistrations]; - } + [OWSPrimaryStorage.sharedManager runSyncRegistrations]; - for (OWSStorage *storage in self.allStorages) { - [storage runAsyncRegistrationsWithCompletion:^{ - if ([self postRegistrationCompleteNotificationIfPossible]) { - backgroundTask = nil; - } - }]; - } + [OWSPrimaryStorage.sharedManager runAsyncRegistrationsWithCompletion:^{ + OWSAssert(self.isStorageReady); + + [self postRegistrationCompleteNotification]; + + migrationBlock(); + + backgroundTask = nil; + }]; } - (YapDatabaseConnection *)registrationConnection @@ -363,11 +351,11 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); } // Returns YES IFF all registrations are complete. -+ (BOOL)postRegistrationCompleteNotificationIfPossible ++ (void)postRegistrationCompleteNotification { - if (!self.isStorageReady) { - return NO; - } + OWSAssert(self.isStorageReady); + + DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ @@ -375,18 +363,11 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); object:nil userInfo:nil]; }); - - return YES; } + (BOOL)isStorageReady { - for (OWSStorage *storage in self.allStorages) { - if (!storage.areAllRegistrationsComplete) { - return NO; - } - } - return YES; + return OWSPrimaryStorage.sharedManager.areAllRegistrationsComplete; } - (BOOL)tryToLoadDatabase diff --git a/SignalShareExtension/ShareViewController.swift b/SignalShareExtension/ShareViewController.swift index c952e4a23..2c6edbd9d 100644 --- a/SignalShareExtension/ShareViewController.swift +++ b/SignalShareExtension/ShareViewController.swift @@ -88,20 +88,20 @@ public class ShareViewController: UIViewController, ShareViewDelegate, SAEFailed }.retainUntilComplete() // We shouldn't set up our environment until after we've consulted isReadyForAppExtensions. - AppSetup.setupEnvironment({ + AppSetup.setupEnvironment(callMessageHandlerBlock: { return NoopCallMessageHandler() - }) { + }, + notificationsProtocolBlock: { return NoopNotificationsManager() - } + }, + migrationCompletion: { [weak self] in + SwiftAssertIsOnMainThread(#function) - // performUpdateCheck must be invoked after Environment has been initialized because - // upgrade process may depend on Environment. - VersionMigrations.performUpdateCheck(completion: { [weak self] in - SwiftAssertIsOnMainThread(#function) - - guard let strongSelf = self else { return } + guard let strongSelf = self else { return } - strongSelf.versionMigrationsDidComplete() + // performUpdateCheck must be invoked after Environment has been initialized because + // upgrade process may depend on Environment. + strongSelf.versionMigrationsDidComplete() }) // We don't need to use "screen protection" in the SAE. @@ -268,6 +268,7 @@ public class ShareViewController: UIViewController, ShareViewDelegate, SAEFailed AppVersion.instance().saeLaunchDidComplete() Environment.current().contactsManager.loadSignalAccountsFromCache() + Environment.current().contactsManager.startObserving() ensureRootViewController() From 9cdf489ace81c389b084758b57d7305501da062b Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Wed, 25 Apr 2018 09:31:36 -0400 Subject: [PATCH 3/3] Merge branch 'charlesmchen/corruptDatabaseViews' --- .../src/ViewControllers/DebugUI/DebugUIMisc.m | 9 ++ .../src/Messages/OWSBatchMessageProcessor.h | 2 + .../src/Messages/OWSBatchMessageProcessor.m | 5 + .../Messages/OWSDisappearingMessagesFinder.h | 5 +- .../Messages/OWSDisappearingMessagesFinder.m | 7 +- .../OWSFailedAttachmentDownloadsJob.h | 4 +- .../OWSFailedAttachmentDownloadsJob.m | 7 +- .../src/Messages/OWSFailedMessagesJob.h | 4 +- .../src/Messages/OWSFailedMessagesJob.m | 7 +- .../src/Messages/OWSMessageReceiver.h | 2 + .../src/Messages/OWSMessageReceiver.m | 5 + .../src/Storage/OWSIncomingMessageFinder.h | 4 +- .../src/Storage/OWSIncomingMessageFinder.m | 7 +- .../src/Storage/OWSMediaGalleryFinder.h | 1 + .../src/Storage/OWSMediaGalleryFinder.m | 7 +- .../src/Storage/OWSPrimaryStorage.m | 36 +++++- SignalServiceKit/src/Storage/OWSStorage.h | 10 +- SignalServiceKit/src/Storage/OWSStorage.m | 103 ++++++++++++++++++ .../src/Storage/TSDatabaseSecondaryIndexes.h | 2 + .../src/Storage/TSDatabaseSecondaryIndexes.m | 11 +- SignalServiceKit/src/Util/OWSAsserts.h | 7 ++ 21 files changed, 219 insertions(+), 26 deletions(-) diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIMisc.m b/Signal/src/ViewControllers/DebugUI/DebugUIMisc.m index 113c5ceb3..7bd496294 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIMisc.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIMisc.m @@ -120,6 +120,15 @@ NS_ASSUME_NONNULL_BEGIN }]]; #endif + [items + addObject:[OWSTableItem + itemWithTitle:@"Increment Database Extension Versions" + actionBlock:^() { + for (NSString *extensionName in OWSPrimaryStorage.sharedManager.registeredExtensionNames) { + [OWSStorage incrementVersionOfDatabaseExtension:extensionName]; + } + }]]; + return [OWSTableSection sectionWithTitle:self.name items:items]; } diff --git a/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.h b/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.h index c929dcee9..b4b3743b7 100644 --- a/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.h +++ b/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.h @@ -14,6 +14,8 @@ NS_ASSUME_NONNULL_BEGIN @interface OWSBatchMessageProcessor : NSObject + (instancetype)sharedInstance; + ++ (NSString *)databaseExtensionName; + (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage; - (void)enqueueEnvelopeData:(NSData *)envelopeData diff --git a/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.m b/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.m index fc41a9ec8..eebc76308 100644 --- a/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.m +++ b/SignalServiceKit/src/Messages/OWSBatchMessageProcessor.m @@ -458,6 +458,11 @@ NSString *const OWSMessageContentJobFinderExtensionGroup = @"OWSMessageContentJo #pragma mark - class methods ++ (NSString *)databaseExtensionName +{ + return OWSMessageContentJobFinderExtensionName; +} + + (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage { [OWSMessageContentJobFinder asyncRegisterDatabaseExtension:storage]; diff --git a/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.h b/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.h index 808b6c487..b1e375e7c 100644 --- a/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.h +++ b/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.h @@ -25,9 +25,8 @@ NS_ASSUME_NONNULL_BEGIN */ - (nullable NSNumber *)nextExpirationTimestampWithTransaction:(YapDatabaseReadTransaction *_Nonnull)transaction; -/** - * Database extensions required for class to work. - */ ++ (NSString *)databaseExtensionName; + + (void)asyncRegisterDatabaseExtensions:(OWSStorage *)storage; #ifdef DEBUG diff --git a/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.m b/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.m index 4628f6410..dbff1a2f5 100644 --- a/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.m +++ b/SignalServiceKit/src/Messages/OWSDisappearingMessagesFinder.m @@ -185,7 +185,7 @@ static NSString *const OWSDisappearingMessageFinderExpiresAtIndex = @"index_mess dict[OWSDisappearingMessageFinderThreadIdColumn] = message.uniqueThreadId; }]; - return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler]; + return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil]; } #ifdef DEBUG @@ -196,6 +196,11 @@ static NSString *const OWSDisappearingMessageFinderExpiresAtIndex = @"index_mess } #endif ++ (NSString *)databaseExtensionName +{ + return OWSDisappearingMessageFinderExpiresAtIndex; +} + + (void)asyncRegisterDatabaseExtensions:(OWSStorage *)storage { [storage asyncRegisterExtension:[self indexDatabaseExtension] withName:OWSDisappearingMessageFinderExpiresAtIndex]; diff --git a/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.h b/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.h index 6e694bddc..e858a49f6 100644 --- a/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.h +++ b/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.h @@ -14,9 +14,7 @@ NS_ASSUME_NONNULL_BEGIN - (void)run; -/** - * Database extensions required for class to work. - */ ++ (NSString *)databaseExtensionName; + (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage; #ifdef DEBUG diff --git a/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.m b/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.m index 1637ad827..aa829bca2 100644 --- a/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.m +++ b/SignalServiceKit/src/Messages/OWSFailedAttachmentDownloadsJob.m @@ -114,7 +114,7 @@ static NSString *const OWSFailedAttachmentDownloadsJobAttachmentStateIndex = @"i dict[OWSFailedAttachmentDownloadsJobAttachmentStateColumn] = @(attachment.state); }]; - return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler]; + return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil]; } #ifdef DEBUG @@ -126,6 +126,11 @@ static NSString *const OWSFailedAttachmentDownloadsJobAttachmentStateIndex = @"i } #endif ++ (NSString *)databaseExtensionName +{ + return OWSFailedAttachmentDownloadsJobAttachmentStateIndex; +} + + (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage { [storage asyncRegisterExtension:[self indexDatabaseExtension] diff --git a/SignalServiceKit/src/Messages/OWSFailedMessagesJob.h b/SignalServiceKit/src/Messages/OWSFailedMessagesJob.h index 90d760867..eea3d0822 100644 --- a/SignalServiceKit/src/Messages/OWSFailedMessagesJob.h +++ b/SignalServiceKit/src/Messages/OWSFailedMessagesJob.h @@ -14,9 +14,7 @@ NS_ASSUME_NONNULL_BEGIN - (void)run; -/** - * Database extensions required for class to work. - */ ++ (NSString *)databaseExtensionName; + (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage; #ifdef DEBUG diff --git a/SignalServiceKit/src/Messages/OWSFailedMessagesJob.m b/SignalServiceKit/src/Messages/OWSFailedMessagesJob.m index 60abdbbf3..5ea221472 100644 --- a/SignalServiceKit/src/Messages/OWSFailedMessagesJob.m +++ b/SignalServiceKit/src/Messages/OWSFailedMessagesJob.m @@ -124,7 +124,7 @@ static NSString *const OWSFailedMessagesJobMessageStateIndex = @"index_outoing_m dict[OWSFailedMessagesJobMessageStateColumn] = @(message.messageState); }]; - return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler]; + return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil]; } #ifdef DEBUG @@ -136,6 +136,11 @@ static NSString *const OWSFailedMessagesJobMessageStateIndex = @"index_outoing_m } #endif ++ (NSString *)databaseExtensionName +{ + return OWSFailedMessagesJobMessageStateIndex; +} + + (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage { [storage asyncRegisterExtension:[self indexDatabaseExtension] withName:OWSFailedMessagesJobMessageStateIndex]; diff --git a/SignalServiceKit/src/Messages/OWSMessageReceiver.h b/SignalServiceKit/src/Messages/OWSMessageReceiver.h index 8af362d77..ad7a0d957 100644 --- a/SignalServiceKit/src/Messages/OWSMessageReceiver.h +++ b/SignalServiceKit/src/Messages/OWSMessageReceiver.h @@ -14,6 +14,8 @@ NS_ASSUME_NONNULL_BEGIN @interface OWSMessageReceiver : NSObject + (instancetype)sharedInstance; + ++ (NSString *)databaseExtensionName; + (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage; - (void)handleReceivedEnvelope:(OWSSignalServiceProtosEnvelope *)envelope; diff --git a/SignalServiceKit/src/Messages/OWSMessageReceiver.m b/SignalServiceKit/src/Messages/OWSMessageReceiver.m index 0af8b6baa..3c0bc633e 100644 --- a/SignalServiceKit/src/Messages/OWSMessageReceiver.m +++ b/SignalServiceKit/src/Messages/OWSMessageReceiver.m @@ -413,6 +413,11 @@ NSString *const OWSMessageDecryptJobFinderExtensionGroup = @"OWSMessageProcessin #pragma mark - class methods ++ (NSString *)databaseExtensionName +{ + return OWSMessageDecryptJobFinderExtensionName; +} + + (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage { [OWSMessageDecryptJobFinder asyncRegisterDatabaseExtension:storage]; diff --git a/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.h b/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.h index 4a1e7bb50..d62800957 100644 --- a/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.h +++ b/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.h @@ -12,9 +12,7 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)initWithPrimaryStorage:(OWSPrimaryStorage *)primaryStorage NS_DESIGNATED_INITIALIZER; -/** - * Must be called before using this finder. - */ ++ (NSString *)databaseExtensionName; + (void)asyncRegisterExtensionWithPrimaryStorage:(OWSStorage *)storage; /** diff --git a/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.m b/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.m index 94e4dedc2..42b114200 100644 --- a/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.m +++ b/SignalServiceKit/src/Storage/OWSIncomingMessageFinder.m @@ -89,7 +89,12 @@ NSString *const OWSIncomingMessageFinderColumnSourceDeviceId = @"OWSIncomingMess YapDatabaseSecondaryIndexHandler *handler = [YapDatabaseSecondaryIndexHandler withObjectBlock:block]; - return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler]; + return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil]; +} + ++ (NSString *)databaseExtensionName +{ + return OWSIncomingMessageFinderExtensionName; } + (void)asyncRegisterExtensionWithPrimaryStorage:(OWSStorage *)storage diff --git a/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.h b/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.h index 349d818bc..602e10a1e 100644 --- a/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.h +++ b/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.h @@ -28,6 +28,7 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - Extension registration ++ (NSString *)databaseExtensionName; + (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage; @end diff --git a/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.m b/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.m index 8d923a01a..cb1ec1ef9 100644 --- a/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.m +++ b/SignalServiceKit/src/Storage/OWSMediaGalleryFinder.m @@ -111,6 +111,11 @@ static NSString *const OWSMediaGalleryFinderExtensionName = @"OWSMediaGalleryFin #pragma mark - Extension registration ++ (NSString *)databaseExtensionName +{ + return OWSMediaGalleryFinderExtensionName; +} + + (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage { [storage asyncRegisterExtension:[self mediaGalleryDatabaseExtension] @@ -158,7 +163,7 @@ static NSString *const OWSMediaGalleryFinderExtensionName = @"OWSMediaGalleryFin YapDatabaseViewOptions *options = [YapDatabaseViewOptions new]; options.allowedCollections = [[YapWhitelistBlacklist alloc] initWithWhitelist:[NSSet setWithObject:TSMessage.collection]]; - + return [[YapDatabaseAutoView alloc] initWithGrouping:grouping sorting:sorting versionTag:@"1" options:options]; } diff --git a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m index 3329e494b..e9f0f7ebd 100644 --- a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m +++ b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m @@ -22,7 +22,7 @@ NS_ASSUME_NONNULL_BEGIN NSString *const OWSPrimaryStorageExceptionName_CouldNotCreateDatabaseDirectory = @"TSStorageManagerExceptionName_CouldNotCreateDatabaseDirectory"; -void runSyncRegistrationsForStorage(OWSStorage *storage) +void RunSyncRegistrationsForStorage(OWSStorage *storage) { OWSCAssert(storage); @@ -30,7 +30,7 @@ void runSyncRegistrationsForStorage(OWSStorage *storage) [TSDatabaseView registerCrossProcessNotifier:storage]; } -void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t completion) +void RunAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t completion) { OWSCAssert(storage); OWSCAssert(completion); @@ -43,7 +43,8 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl [TSDatabaseView asyncRegisterThreadInteractionsDatabaseView:storage]; [TSDatabaseView asyncRegisterThreadDatabaseView:storage]; [TSDatabaseView asyncRegisterUnreadDatabaseView:storage]; - [storage asyncRegisterExtension:[TSDatabaseSecondaryIndexes registerTimeStampIndex] withName:@"idx"]; + [storage asyncRegisterExtension:[TSDatabaseSecondaryIndexes registerTimeStampIndex] + withName:[TSDatabaseSecondaryIndexes registerTimeStampIndexExtensionName]]; [OWSMessageReceiver asyncRegisterDatabaseExtension:storage]; [OWSBatchMessageProcessor asyncRegisterDatabaseExtension:storage]; @@ -63,6 +64,24 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl [TSDatabaseView asyncRegisterLazyRestoreAttachmentsDatabaseView:storage completion:completion]; } +void VerifyRegistrationsForPrimaryStorage(OWSStorage *storage) +{ + OWSCAssert(storage); + + [[storage newDatabaseConnection] asyncReadWithBlock:^(YapDatabaseReadTransaction *transaction) { + for (NSString *extensionName in storage.registeredExtensionNames) { + DDLogVerbose(@"Verifying database extension: %@", extensionName); + YapDatabaseViewTransaction *_Nullable viewTransaction = [transaction ext:extensionName]; + if (!viewTransaction) { + OWSProdLogAndCFail( + @"VerifyRegistrationsForPrimaryStorage missing database extension: %@", extensionName); + + [OWSStorage incrementVersionOfDatabaseExtension:extensionName]; + } + } + }]; +} + #pragma mark - @interface OWSPrimaryStorage () @@ -119,7 +138,7 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl - (void)runSyncRegistrations { - runSyncRegistrationsForStorage(self); + RunSyncRegistrationsForStorage(self); // See comments on OWSDatabaseConnection. // @@ -137,7 +156,7 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl DDLogVerbose(@"%@ async registrations enqueuing.", self.logTag); - runAsyncRegistrationsForStorage(self, ^{ + RunAsyncRegistrationsForStorage(self, ^{ OWSAssertIsOnMainThread(); OWSAssert(!self.areAsyncRegistrationsComplete); @@ -147,9 +166,16 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl self.areAsyncRegistrationsComplete = YES; completion(); + + [self verifyDatabaseViews]; }); } +- (void)verifyDatabaseViews +{ + VerifyRegistrationsForPrimaryStorage(self); +} + + (void)protectFiles { DDLogInfo( diff --git a/SignalServiceKit/src/Storage/OWSStorage.h b/SignalServiceKit/src/Storage/OWSStorage.h index 6fafcc657..94d6f65cd 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.h +++ b/SignalServiceKit/src/Storage/OWSStorage.h @@ -64,9 +64,11 @@ typedef void (^OWSStorageMigrationBlock)(void); - (YapDatabaseConnection *)newDatabaseConnection; -#ifdef DEBUG +#pragma mark - Extension Registration + ++ (void)incrementVersionOfDatabaseExtension:(NSString *)extensionName; + - (BOOL)registerExtension:(YapDatabaseExtension *)extension withName:(NSString *)extensionName; -#endif - (void)asyncRegisterExtension:(YapDatabaseExtension *)extension withName:(NSString *)extensionName; - (void)asyncRegisterExtension:(YapDatabaseExtension *)extension @@ -75,6 +77,10 @@ typedef void (^OWSStorageMigrationBlock)(void); - (nullable id)registeredExtension:(NSString *)extensionName; +- (NSArray *)registeredExtensionNames; + +#pragma mark - + - (unsigned long long)databaseFileSize; - (unsigned long long)databaseWALFileSize; - (unsigned long long)databaseSHMFileSize; diff --git a/SignalServiceKit/src/Storage/OWSStorage.m b/SignalServiceKit/src/Storage/OWSStorage.m index c56d1c06b..d4dd65b1b 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.m +++ b/SignalServiceKit/src/Storage/OWSStorage.m @@ -6,6 +6,7 @@ #import "AppContext.h" #import "NSData+Base64.h" #import "NSNotificationCenter+OWS.h" +#import "NSUserDefaults+OWS.h" #import "OWSBackgroundTask.h" #import "OWSFileSystem.h" #import "OWSPrimaryStorage.h" @@ -14,7 +15,11 @@ #import #import #import +#import +#import #import +#import +#import NS_ASSUME_NONNULL_BEGIN @@ -36,6 +41,8 @@ const NSUInteger kDatabasePasswordLength = 30; typedef NSData *_Nullable (^LoadDatabaseMetadataBlock)(NSError **_Nullable); typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); +NSString *const kNSUserDefaults_DatabaseExtensionVersionMap = @"kNSUserDefaults_DatabaseExtensionVersionMap"; + #pragma mark - @interface YapDatabaseConnection () @@ -234,6 +241,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); @property (atomic, nullable) YapDatabase *database; +@property (nonatomic) NSMutableArray *extensionNames; + @end #pragma mark - @@ -249,6 +258,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); selector:@selector(resetStorage) name:OWSResetStorageNotification object:nil]; + + self.extensionNames = [NSMutableArray new]; } return self; @@ -461,8 +472,90 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); return dbConnection; } +#pragma mark - Extension Registration + ++ (void)incrementVersionOfDatabaseExtension:(NSString *)extensionName +{ + DDLogError(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + NSUserDefaults *appUserDefaults = [NSUserDefaults appUserDefaults]; + OWSAssert(appUserDefaults); + NSMutableDictionary *_Nullable versionMap = + [[appUserDefaults valueForKey:kNSUserDefaults_DatabaseExtensionVersionMap] mutableCopy]; + if (!versionMap) { + versionMap = [NSMutableDictionary new]; + } + NSNumber *_Nullable versionSuffix = versionMap[extensionName]; + versionMap[extensionName] = @(versionSuffix.intValue + 1); + [appUserDefaults setValue:versionMap forKey:kNSUserDefaults_DatabaseExtensionVersionMap]; + [appUserDefaults synchronize]; +} + +- (nullable NSString *)appendSuffixToDatabaseExtensionVersionIfNecessary:(nullable NSString *)versionTag + extensionName:(NSString *)extensionName +{ + OWSAssertIsOnMainThread(); + + NSUserDefaults *appUserDefaults = [NSUserDefaults appUserDefaults]; + OWSAssert(appUserDefaults); + NSDictionary *_Nullable versionMap = + [appUserDefaults valueForKey:kNSUserDefaults_DatabaseExtensionVersionMap]; + NSNumber *_Nullable versionSuffix = versionMap[extensionName]; + + if (versionSuffix) { + NSString *result = + [NSString stringWithFormat:@"%@.%@", (versionTag.length < 1 ? @"0" : versionTag), versionSuffix]; + DDLogWarn(@"%@ database extension version: %@ + %@ -> %@", self.logTag, versionTag, versionSuffix, result); + return result; + } + return versionTag; +} + +- (YapDatabaseExtension *)updateExtensionVersion:(YapDatabaseExtension *)extension withName:(NSString *)extensionName +{ + OWSAssert(extension); + OWSAssert(extensionName.length > 0); + + if ([extension isKindOfClass:[YapDatabaseAutoView class]]) { + YapDatabaseAutoView *databaseView = (YapDatabaseAutoView *)extension; + YapDatabaseAutoView *databaseViewCopy = [[YapDatabaseAutoView alloc] + initWithGrouping:databaseView.grouping + sorting:databaseView.sorting + versionTag:[self appendSuffixToDatabaseExtensionVersionIfNecessary:databaseView.versionTag + extensionName:extensionName] + options:databaseView.options]; + return databaseViewCopy; + } else if ([extension isKindOfClass:[YapDatabaseSecondaryIndex class]]) { + YapDatabaseSecondaryIndex *secondaryIndex = (YapDatabaseSecondaryIndex *)extension; + OWSAssert(secondaryIndex->setup); + OWSAssert(secondaryIndex->handler); + YapDatabaseSecondaryIndex *secondaryIndexCopy = [[YapDatabaseSecondaryIndex alloc] + initWithSetup:secondaryIndex->setup + handler:secondaryIndex->handler + versionTag:[self appendSuffixToDatabaseExtensionVersionIfNecessary:secondaryIndex.versionTag + extensionName:extensionName] + options:secondaryIndex->options]; + return secondaryIndexCopy; + } else if ([extension isKindOfClass:[YapDatabaseCrossProcessNotification class]]) { + // versionTag doesn't matter for YapDatabaseCrossProcessNotification. + return extension; + } else { + // This method needs to be able to update the versionTag of all extensions. + // If we start using other extension types, we need to modify this method to + // handle them as well. + OWSProdLogAndFail(@"%@ Unknown extension type: %@", self.logTag, [extension class]); + + return extension; + } +} + - (BOOL)registerExtension:(YapDatabaseExtension *)extension withName:(NSString *)extensionName { + extension = [self updateExtensionVersion:extension withName:extensionName]; + + OWSAssert(![self.extensionNames containsObject:extensionName]); + [self.extensionNames addObject:extensionName]; + return [self.database registerExtension:extension withName:extensionName]; } @@ -476,6 +569,11 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); withName:(NSString *)extensionName completion:(nullable dispatch_block_t)completion { + extension = [self updateExtensionVersion:extension withName:extensionName]; + + OWSAssert(![self.extensionNames containsObject:extensionName]); + [self.extensionNames addObject:extensionName]; + [self.database asyncRegisterExtension:extension withName:extensionName completionBlock:^(BOOL ready) { @@ -498,6 +596,11 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); return [self.database registeredExtension:extensionName]; } +- (NSArray *)registeredExtensionNames +{ + return [self.extensionNames copy]; +} + #pragma mark - Password + (void)deleteDatabaseFiles diff --git a/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.h b/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.h index 5db2ac8cb..81cab7d1b 100644 --- a/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.h +++ b/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.h @@ -7,6 +7,8 @@ @interface TSDatabaseSecondaryIndexes : NSObject ++ (NSString *)registerTimeStampIndexExtensionName; + + (YapDatabaseSecondaryIndex *)registerTimeStampIndex; + (void)enumerateMessagesWithTimestamp:(uint64_t)timestamp diff --git a/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.m b/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.m index f61055edd..b0e0f46d0 100644 --- a/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.m +++ b/SignalServiceKit/src/Storage/TSDatabaseSecondaryIndexes.m @@ -3,12 +3,18 @@ // #import "TSDatabaseSecondaryIndexes.h" +#import "OWSStorage.h" #import "TSInteraction.h" #define TSTimeStampSQLiteIndex @"messagesTimeStamp" @implementation TSDatabaseSecondaryIndexes ++ (NSString *)registerTimeStampIndexExtensionName +{ + return @"idx"; +} + + (YapDatabaseSecondaryIndex *)registerTimeStampIndex { YapDatabaseSecondaryIndexSetup *setup = [[YapDatabaseSecondaryIndexSetup alloc] init]; [setup addColumn:TSTimeStampSQLiteIndex withType:YapDatabaseSecondaryIndexTypeReal]; @@ -25,7 +31,8 @@ YapDatabaseSecondaryIndexHandler *handler = [YapDatabaseSecondaryIndexHandler withObjectBlock:block]; - YapDatabaseSecondaryIndex *secondaryIndex = [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler]; + YapDatabaseSecondaryIndex *secondaryIndex = + [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil]; return secondaryIndex; } @@ -37,7 +44,7 @@ { NSString *formattedString = [NSString stringWithFormat:@"WHERE %@ = %lld", TSTimeStampSQLiteIndex, timestamp]; YapDatabaseQuery *query = [YapDatabaseQuery queryWithFormat:formattedString]; - [[transaction ext:@"idx"] enumerateKeysMatchingQuery:query usingBlock:block]; + [[transaction ext:[self registerTimeStampIndexExtensionName]] enumerateKeysMatchingQuery:query usingBlock:block]; } @end diff --git a/SignalServiceKit/src/Util/OWSAsserts.h b/SignalServiceKit/src/Util/OWSAsserts.h index c9f34a660..c33b75713 100755 --- a/SignalServiceKit/src/Util/OWSAsserts.h +++ b/SignalServiceKit/src/Util/OWSAsserts.h @@ -128,6 +128,13 @@ NS_ASSUME_NONNULL_BEGIN OWSFail(_messageFormat, ##__VA_ARGS__); \ } +#define OWSProdLogAndCFail(_messageFormat, ...) \ + { \ + DDLogError(_messageFormat, ##__VA_ARGS__); \ + [DDLog flushLog]; \ + OWSCFail(_messageFormat, ##__VA_ARGS__); \ + } + // This function is intended for use in Swift. void SwiftAssertIsOnMainThread(NSString *functionName);