From df01c7e63e9bec71c608ab951fb6b171d7b31ea5 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 2 Aug 2018 11:08:31 -0600 Subject: [PATCH 1/3] Update to latest YapDB (with Signal patches applied) Applied branches: - mkirk/invalidFinalIndices - mkirk/upstream-unencrypted-headers Also fixed up test suite. --- Podfile | 2 +- Podfile.lock | 85 ++++++++++--------- Pods | 2 +- .../test/util/DisplayableTextFilterTest.swift | 9 +- Signal/test/util/OWSDatabaseConverterTest.m | 7 ++ .../src/Storage/FullTextSearchFinder.swift | 80 ++++++++--------- 6 files changed, 101 insertions(+), 84 deletions(-) diff --git a/Podfile b/Podfile index 1f1778fec..28cd1a677 100644 --- a/Podfile +++ b/Podfile @@ -8,7 +8,7 @@ def shared_pods # pod 'SQLCipher', path: '../sqlcipher2' pod 'SQLCipher', :git => 'https://github.com/sqlcipher/sqlcipher.git', :commit => 'd5c2bec' # pod 'YapDatabase/SQLCipher', path: '../YapDatabase' - pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'release/unencryptedHeaders' + pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'signal-release' # pod 'AxolotlKit', path: '../SignalProtocolKit' pod 'SignalServiceKit', path: '.' pod 'AxolotlKit', git: 'https://github.com/signalapp/SignalProtocolKit.git' diff --git a/Podfile.lock b/Podfile.lock index c6e0f22f9..2f6f738ab 100644 --- a/Podfile.lock +++ b/Podfile.lock @@ -70,64 +70,67 @@ PODS: - SQLCipher/common - SSZipArchive (2.1.2) - SwiftProtobuf (1.0.3) - - YapDatabase/SQLCipher (3.0.2): - - YapDatabase/SQLCipher/Core (= 3.0.2) - - YapDatabase/SQLCipher/Extensions (= 3.0.2) - - YapDatabase/SQLCipher/Core (3.0.2): + - YapDatabase/SQLCipher (3.1.1): + - YapDatabase/SQLCipher/Core (= 3.1.1) + - YapDatabase/SQLCipher/Extensions (= 3.1.1) + - YapDatabase/SQLCipher/Core (3.1.1): - CocoaLumberjack - SQLCipher (>= 3.4.0) - - YapDatabase/SQLCipher/Extensions (3.0.2): - - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/AutoView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/Hooks (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/ManualView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/Relationships (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/View (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/ActionManager (3.0.2): + - YapDatabase/SQLCipher/Extensions (3.1.1): + - YapDatabase/SQLCipher/Core + - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/AutoView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ConnectionPool (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/Hooks (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ManualView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/Relationships (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/View (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ActionManager (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/AutoView - - YapDatabase/SQLCipher/Extensions/AutoView (3.0.2): + - YapDatabase/SQLCipher/Extensions/AutoView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/View - - YapDatabase/SQLCipher/Extensions/CloudCore (3.0.2): + - YapDatabase/SQLCipher/Extensions/CloudCore (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/CloudKit (3.0.2): + - YapDatabase/SQLCipher/Extensions/CloudKit (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.0.2): + - YapDatabase/SQLCipher/Extensions/ConnectionPool (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.0.2): + - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/FilteredView (3.0.2): + - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.1.1): + - YapDatabase/SQLCipher/Core + - YapDatabase/SQLCipher/Extensions/FilteredView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/View - - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.0.2): + - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/Hooks (3.0.2): + - YapDatabase/SQLCipher/Extensions/Hooks (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/ManualView (3.0.2): + - YapDatabase/SQLCipher/Extensions/ManualView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/View - - YapDatabase/SQLCipher/Extensions/Relationships (3.0.2): + - YapDatabase/SQLCipher/Extensions/Relationships (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.0.2): + - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.0.2): + - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/AutoView - YapDatabase/SQLCipher/Extensions/FullTextSearch - - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.0.2): + - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/View (3.0.2): + - YapDatabase/SQLCipher/Extensions/View (3.1.1): - YapDatabase/SQLCipher/Core - YYImage (1.0.4): - YYImage/Core (= 1.0.4) @@ -147,7 +150,7 @@ DEPENDENCIES: - SocketRocket (from `https://github.com/facebook/SocketRocket.git`) - SQLCipher (from `https://github.com/sqlcipher/sqlcipher.git`, commit `d5c2bec`) - SSZipArchive - - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `release/unencryptedHeaders`) + - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `signal-release`) - YYImage SPEC REPOS: @@ -185,7 +188,7 @@ EXTERNAL SOURCES: :commit: d5c2bec :git: https://github.com/sqlcipher/sqlcipher.git YapDatabase: - :branch: release/unencryptedHeaders + :branch: signal-release :git: https://github.com/signalapp/YapDatabase.git CHECKOUT OPTIONS: @@ -208,7 +211,7 @@ CHECKOUT OPTIONS: :commit: d5c2bec :git: https://github.com/sqlcipher/sqlcipher.git YapDatabase: - :commit: 764e949142ba1bada99aeedeeaccfb68047a6e79 + :commit: f1fa4545e1e1594fb80065ffca52a682f5a2e71c :git: https://github.com/signalapp/YapDatabase.git SPEC CHECKSUMS: @@ -231,9 +234,9 @@ SPEC CHECKSUMS: SQLCipher: f9fcf29b2e59ced7defc2a2bdd0ebe79b40d4990 SSZipArchive: d4009d2ce5520a421f231fd97028cc0e2667eed8 SwiftProtobuf: 5ccc0e4054e37c75800e5744acb2aa80bb72b39c - YapDatabase: 299a32de9d350d37a9ac5b0532609d87d5d2a5de + YapDatabase: b418a4baa6906e8028748938f9159807fd039af4 YYImage: 1e1b62a9997399593e4b9c4ecfbbabbf1d3f3b54 -PODFILE CHECKSUM: db797890d3df475827063ba956cb42afa148ff19 +PODFILE CHECKSUM: b378e1ac40edbfc9d4f66410d26ea18a8a0a32dc COCOAPODS: 1.5.3 diff --git a/Pods b/Pods index 74a0f3857..9f36102cb 160000 --- a/Pods +++ b/Pods @@ -1 +1 @@ -Subproject commit 74a0f3857847cf9658b10b4bdce7b19f53e891ec +Subproject commit 9f36102cbbd5555045626b87fc2dde4dde05ab61 diff --git a/Signal/test/util/DisplayableTextFilterTest.swift b/Signal/test/util/DisplayableTextFilterTest.swift index 72e7297d2..e1dbb40d6 100644 --- a/Signal/test/util/DisplayableTextFilterTest.swift +++ b/Signal/test/util/DisplayableTextFilterTest.swift @@ -64,7 +64,14 @@ class DisplayableTextTest: XCTestCase { XCTAssertEqual("Příliš žluťoučký kůň úpěl ďábelské ódy.".glyphCount, 39) // Excessive diacritics - XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115) + + // some insignificant discrepencies across iOS versions + if #available(iOS 11, *) { + XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115) + } else { + XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 109) + } + XCTAssertEqual("L̷̳͔̲͝Ģ̵̮̯̤̩̙͍̬̟͉̹̘̹͍͈̮̦̰̣͟͝O̶̴̮̻̮̗͘͡!̴̷̟͓͓".glyphCount, 43) } diff --git a/Signal/test/util/OWSDatabaseConverterTest.m b/Signal/test/util/OWSDatabaseConverterTest.m index 85c6c8864..9219157aa 100644 --- a/Signal/test/util/OWSDatabaseConverterTest.m +++ b/Signal/test/util/OWSDatabaseConverterTest.m @@ -14,6 +14,13 @@ NS_ASSUME_NONNULL_BEGIN +@interface YapDatabase (OWSDatabaseConverterTest) + +- (void)flushInternalQueue; +- (void)flushCheckpointQueue; + +@end + @interface OWSStorage (OWSDatabaseConverterTest) + (YapDatabaseDeserializer)logOnFailureDeserializer; diff --git a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift index b95fbfc6b..f8636bd88 100644 --- a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift +++ b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift @@ -7,14 +7,14 @@ import Foundation // Create a searchable index for objects of type T public class SearchIndexer { - private let indexBlock: (T) -> String + private let indexBlock: (T, YapDatabaseReadTransaction) -> String - public init(indexBlock: @escaping (T) -> String) { + public init(indexBlock: @escaping (T, YapDatabaseReadTransaction) -> String) { self.indexBlock = indexBlock } - public func index(_ item: T) -> String { - return normalize(indexingText: indexBlock(item)) + public func index(_ item: T, transaction: YapDatabaseReadTransaction) -> String { + return normalize(indexingText: indexBlock(item, transaction)) } private func normalize(indexingText: String) -> String { @@ -157,22 +157,22 @@ public class FullTextSearchFinder: NSObject { return TextSecureKitEnv.shared().contactsManager } - private static let groupThreadIndexer: SearchIndexer = SearchIndexer { (groupThread: TSGroupThread) in + private static let groupThreadIndexer: SearchIndexer = SearchIndexer { (groupThread: TSGroupThread, transaction: YapDatabaseReadTransaction) in let groupName = groupThread.groupModel.groupName ?? "" let memberStrings = groupThread.groupModel.groupMemberIds.map { recipientId in - recipientIndexer.index(recipientId) + recipientIndexer.index(recipientId, transaction: transaction) }.joined(separator: " ") return "\(groupName) \(memberStrings)" } - private static let contactThreadIndexer: SearchIndexer = SearchIndexer { (contactThread: TSContactThread) in + private static let contactThreadIndexer: SearchIndexer = SearchIndexer { (contactThread: TSContactThread, transaction: YapDatabaseReadTransaction) in let recipientId = contactThread.contactIdentifier() - return recipientIndexer.index(recipientId) + return recipientIndexer.index(recipientId, transaction: transaction) } - private static let recipientIndexer: SearchIndexer = SearchIndexer { (recipientId: String) in + private static let recipientIndexer: SearchIndexer = SearchIndexer { (recipientId: String, _: YapDatabaseReadTransaction) in let displayName = contactsManager.displayName(forPhoneIdentifier: recipientId) let nationalNumber: String = { (recipientId: String) -> String in @@ -193,46 +193,46 @@ public class FullTextSearchFinder: NSObject { return "\(recipientId) \(nationalNumber) \(displayName)" } - private static let messageIndexer: SearchIndexer = SearchIndexer { (message: TSMessage) in + private static let messageIndexer: SearchIndexer = SearchIndexer { (message: TSMessage, transaction: YapDatabaseReadTransaction) in if let body = message.body, body.count > 0 { return body } - if let oversizeText = oversizeText(forMessage: message) { + if let oversizeText = oversizeText(forMessage: message, transaction: transaction) { return oversizeText } return "" } - private static func oversizeText(forMessage message: TSMessage) -> String? { + private static func oversizeText(forMessage message: TSMessage, transaction: YapDatabaseReadTransaction) -> String? { guard message.hasAttachments() else { return nil } - let dbConnection = OWSPrimaryStorage.shared().dbReadConnection - var oversizeText: String? - dbConnection.read({ (transaction) in - guard let attachment = message.attachment(with: transaction) else { - // This can happen during the initial save of incoming messages. - Logger.warn("Could not load attachment for search indexing.") - return - } - guard let attachmentStream = attachment as? TSAttachmentStream else { - return - } - guard attachmentStream.isOversizeText() else { - return - } - guard let text = attachmentStream.readOversizeText() else { - owsFail("Could not load oversize text attachment") - return - } - oversizeText = text - }) - return oversizeText + + guard let attachment = message.attachment(with: transaction) else { + // This can happen during the initial save of incoming messages. + Logger.warn("Could not load attachment for search indexing.") + return nil + } + + guard let attachmentStream = attachment as? TSAttachmentStream else { + return nil + } + + guard attachmentStream.isOversizeText() else { + return nil + } + + guard let text = attachmentStream.readOversizeText() else { + owsFail("Could not load oversize text attachment") + return nil + } + + return text } - private class func indexContent(object: Any) -> String? { + private class func indexContent(object: Any, transaction: YapDatabaseReadTransaction) -> String? { if let groupThread = object as? TSGroupThread { - return self.groupThreadIndexer.index(groupThread) + return self.groupThreadIndexer.index(groupThread, transaction: transaction) } else if let contactThread = object as? TSContactThread { guard contactThread.hasEverHadMessage else { // If we've never sent/received a message in a TSContactThread, @@ -240,11 +240,11 @@ public class FullTextSearchFinder: NSObject { // than in the "Conversations" section. return nil } - return self.contactThreadIndexer.index(contactThread) + return self.contactThreadIndexer.index(contactThread, transaction: transaction) } else if let message = object as? TSMessage { - return self.messageIndexer.index(message) + return self.messageIndexer.index(message, transaction: transaction) } else if let signalAccount = object as? SignalAccount { - return self.recipientIndexer.index(signalAccount.recipientId) + return self.recipientIndexer.index(signalAccount.recipientId, transaction: transaction) } else { return nil } @@ -277,8 +277,8 @@ public class FullTextSearchFinder: NSObject { let contentColumnName = "content" - let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (dict: NSMutableDictionary, _: String, _: String, object: Any) in - if let content: String = indexContent(object: object) { + let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (transaction: YapDatabaseReadTransaction, dict: NSMutableDictionary, _: String, _: String, object: Any) in + if let content: String = indexContent(object: object, transaction: transaction) { dict[contentColumnName] = content } } From 1eb7fc986cd9423fe59decebc65fca5cdf477600 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 2 Aug 2018 11:44:00 -0600 Subject: [PATCH 2/3] YapDB introduced a method purpose built to do what we were approximating. This seems a little more future proof / less error prone if we change the registered extensions. --- .../src/Storage/OWSPrimaryStorage.m | 99 ++++++++----------- SignalServiceKit/src/Storage/TSDatabaseView.h | 3 +- SignalServiceKit/src/Storage/TSDatabaseView.m | 5 +- 3 files changed, 43 insertions(+), 64 deletions(-) diff --git a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m index 9cfacba44..d21dcebea 100644 --- a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m +++ b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m @@ -30,50 +30,6 @@ NSString *const OWSUIDatabaseConnectionNotificationsKey = @"OWSUIDatabaseConnect NSString *const OWSPrimaryStorageExceptionName_CouldNotCreateDatabaseDirectory = @"TSStorageManagerExceptionName_CouldNotCreateDatabaseDirectory"; -void RunSyncRegistrationsForStorage(OWSStorage *storage) -{ - OWSCAssert(storage); - - // Synchronously register extensions which are essential for views. - [TSDatabaseView registerCrossProcessNotifier:storage]; -} - -void RunAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t completion) -{ - OWSCAssert(storage); - OWSCAssert(completion); - - // Asynchronously register other extensions. - // - // All sync registrations must be done before all async registrations, - // or the sync registrations will block on the async registrations. - - [TSDatabaseView asyncRegisterThreadInteractionsDatabaseView:storage]; - [TSDatabaseView asyncRegisterThreadDatabaseView:storage]; - [TSDatabaseView asyncRegisterUnreadDatabaseView:storage]; - [storage asyncRegisterExtension:[TSDatabaseSecondaryIndexes registerTimeStampIndex] - withName:[TSDatabaseSecondaryIndexes registerTimeStampIndexExtensionName]]; - [OWSMessageReceiver asyncRegisterDatabaseExtension:storage]; - [OWSBatchMessageProcessor asyncRegisterDatabaseExtension:storage]; - - [TSDatabaseView asyncRegisterUnseenDatabaseView:storage]; - [TSDatabaseView asyncRegisterThreadOutgoingMessagesDatabaseView:storage]; - [TSDatabaseView asyncRegisterThreadSpecialMessagesDatabaseView:storage]; - - [FullTextSearchFinder asyncRegisterDatabaseExtensionWithStorage:storage]; - [OWSIncomingMessageFinder asyncRegisterExtensionWithPrimaryStorage:storage]; - [TSDatabaseView asyncRegisterSecondaryDevicesDatabaseView:storage]; - [OWSDisappearingMessagesFinder asyncRegisterDatabaseExtensions:storage]; - [OWSFailedMessagesJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage]; - [OWSIncompleteCallsJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage]; - [OWSFailedAttachmentDownloadsJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage]; - [OWSMediaGalleryFinder asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage]; - - // NOTE: Always pass the completion to the _LAST_ of the async database - // view registrations. - [TSDatabaseView asyncRegisterLazyRestoreAttachmentsDatabaseView:storage completion:completion]; -} - void VerifyRegistrationsForPrimaryStorage(OWSStorage *storage) { OWSCAssert(storage); @@ -208,7 +164,8 @@ void VerifyRegistrationsForPrimaryStorage(OWSStorage *storage) - (void)runSyncRegistrations { - RunSyncRegistrationsForStorage(self); + // Synchronously register extensions which are essential for views. + [TSDatabaseView registerCrossProcessNotifier:self]; // See comments on OWSDatabaseConnection. // @@ -223,22 +180,48 @@ void VerifyRegistrationsForPrimaryStorage(OWSStorage *storage) - (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion { OWSAssert(completion); + OWSAssert(self.database); DDLogVerbose(@"%@ async registrations enqueuing.", self.logTag); - RunAsyncRegistrationsForStorage(self, ^{ - OWSAssertIsOnMainThread(); - - OWSAssert(!self.areAsyncRegistrationsComplete); - - DDLogVerbose(@"%@ async registrations complete.", self.logTag); - - self.areAsyncRegistrationsComplete = YES; - - completion(); - - [self verifyDatabaseViews]; - }); + // Asynchronously register other extensions. + // + // All sync registrations must be done before all async registrations, + // or the sync registrations will block on the async registrations. + [TSDatabaseView asyncRegisterThreadInteractionsDatabaseView:self]; + [TSDatabaseView asyncRegisterThreadDatabaseView:self]; + [TSDatabaseView asyncRegisterUnreadDatabaseView:self]; + [self asyncRegisterExtension:[TSDatabaseSecondaryIndexes registerTimeStampIndex] + withName:[TSDatabaseSecondaryIndexes registerTimeStampIndexExtensionName]]; + + [OWSMessageReceiver asyncRegisterDatabaseExtension:self]; + [OWSBatchMessageProcessor asyncRegisterDatabaseExtension:self]; + + [TSDatabaseView asyncRegisterUnseenDatabaseView:self]; + [TSDatabaseView asyncRegisterThreadOutgoingMessagesDatabaseView:self]; + [TSDatabaseView asyncRegisterThreadSpecialMessagesDatabaseView:self]; + + [FullTextSearchFinder asyncRegisterDatabaseExtensionWithStorage:self]; + [OWSIncomingMessageFinder asyncRegisterExtensionWithPrimaryStorage:self]; + [TSDatabaseView asyncRegisterSecondaryDevicesDatabaseView:self]; + [OWSDisappearingMessagesFinder asyncRegisterDatabaseExtensions:self]; + [OWSFailedMessagesJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:self]; + [OWSIncompleteCallsJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:self]; + [OWSFailedAttachmentDownloadsJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:self]; + [OWSMediaGalleryFinder asyncRegisterDatabaseExtensionsWithPrimaryStorage:self]; + [TSDatabaseView asyncRegisterLazyRestoreAttachmentsDatabaseView:self]; + + [self.database flushExtensionRequestsWithCompletionQueue:nil + completionBlock:^{ + OWSAssertIsOnMainThread(); + OWSAssert(!self.areAsyncRegistrationsComplete); + DDLogVerbose(@"%@ async registrations complete.", self.logTag); + self.areAsyncRegistrationsComplete = YES; + + completion(); + + [self verifyDatabaseViews]; + }]; } - (void)verifyDatabaseViews diff --git a/SignalServiceKit/src/Storage/TSDatabaseView.h b/SignalServiceKit/src/Storage/TSDatabaseView.h index 9954804a6..3e83d174b 100644 --- a/SignalServiceKit/src/Storage/TSDatabaseView.h +++ b/SignalServiceKit/src/Storage/TSDatabaseView.h @@ -58,7 +58,6 @@ extern NSString *const TSLazyRestoreAttachmentsDatabaseViewExtensionName; + (void)asyncRegisterSecondaryDevicesDatabaseView:(OWSStorage *)storage; -+ (void)asyncRegisterLazyRestoreAttachmentsDatabaseView:(OWSStorage *)storage - completion:(nullable dispatch_block_t)completion; ++ (void)asyncRegisterLazyRestoreAttachmentsDatabaseView:(OWSStorage *)storage; @end diff --git a/SignalServiceKit/src/Storage/TSDatabaseView.m b/SignalServiceKit/src/Storage/TSDatabaseView.m index 956ac1e12..8e0d1e0b2 100644 --- a/SignalServiceKit/src/Storage/TSDatabaseView.m +++ b/SignalServiceKit/src/Storage/TSDatabaseView.m @@ -362,7 +362,6 @@ NSString *const TSLazyRestoreAttachmentsGroup = @"TSLazyRestoreAttachmentsGroup" } + (void)asyncRegisterLazyRestoreAttachmentsDatabaseView:(OWSStorage *)storage - completion:(nullable dispatch_block_t)completion { YapDatabaseViewGrouping *viewGrouping = [YapDatabaseViewGrouping withObjectBlock:^NSString *_Nullable( YapDatabaseReadTransaction *transaction, NSString *collection, NSString *key, id object) { @@ -411,9 +410,7 @@ NSString *const TSLazyRestoreAttachmentsGroup = @"TSLazyRestoreAttachmentsGroup" [[YapWhitelistBlacklist alloc] initWithWhitelist:[NSSet setWithObject:[TSAttachment collection]]]; YapDatabaseView *view = [[YapDatabaseAutoView alloc] initWithGrouping:viewGrouping sorting:viewSorting versionTag:@"3" options:options]; - [storage asyncRegisterExtension:view - withName:TSLazyRestoreAttachmentsDatabaseViewExtensionName - completion:completion]; + [storage asyncRegisterExtension:view withName:TSLazyRestoreAttachmentsDatabaseViewExtensionName]; } + (id)unseenDatabaseViewExtension:(YapDatabaseReadTransaction *)transaction From fc1ce02ae52fe7534158b8e3c24d7f6caefcbef3 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Fri, 3 Aug 2018 14:10:46 -0600 Subject: [PATCH 3/3] CR: Now that we have transaction semantics, this shouldn't happen. // FREEBIE --- SignalServiceKit/src/Storage/FullTextSearchFinder.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift index f8636bd88..38461d101 100644 --- a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift +++ b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift @@ -209,8 +209,7 @@ public class FullTextSearchFinder: NSObject { } guard let attachment = message.attachment(with: transaction) else { - // This can happen during the initial save of incoming messages. - Logger.warn("Could not load attachment for search indexing.") + owsFail("\(self.logTag) in \(#function) attachment was unexpectedly nil") return nil }