From df01c7e63e9bec71c608ab951fb6b171d7b31ea5 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 2 Aug 2018 11:08:31 -0600 Subject: [PATCH] Update to latest YapDB (with Signal patches applied) Applied branches: - mkirk/invalidFinalIndices - mkirk/upstream-unencrypted-headers Also fixed up test suite. --- Podfile | 2 +- Podfile.lock | 85 ++++++++++--------- Pods | 2 +- .../test/util/DisplayableTextFilterTest.swift | 9 +- Signal/test/util/OWSDatabaseConverterTest.m | 7 ++ .../src/Storage/FullTextSearchFinder.swift | 80 ++++++++--------- 6 files changed, 101 insertions(+), 84 deletions(-) diff --git a/Podfile b/Podfile index 1f1778fec..28cd1a677 100644 --- a/Podfile +++ b/Podfile @@ -8,7 +8,7 @@ def shared_pods # pod 'SQLCipher', path: '../sqlcipher2' pod 'SQLCipher', :git => 'https://github.com/sqlcipher/sqlcipher.git', :commit => 'd5c2bec' # pod 'YapDatabase/SQLCipher', path: '../YapDatabase' - pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'release/unencryptedHeaders' + pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'signal-release' # pod 'AxolotlKit', path: '../SignalProtocolKit' pod 'SignalServiceKit', path: '.' pod 'AxolotlKit', git: 'https://github.com/signalapp/SignalProtocolKit.git' diff --git a/Podfile.lock b/Podfile.lock index c6e0f22f9..2f6f738ab 100644 --- a/Podfile.lock +++ b/Podfile.lock @@ -70,64 +70,67 @@ PODS: - SQLCipher/common - SSZipArchive (2.1.2) - SwiftProtobuf (1.0.3) - - YapDatabase/SQLCipher (3.0.2): - - YapDatabase/SQLCipher/Core (= 3.0.2) - - YapDatabase/SQLCipher/Extensions (= 3.0.2) - - YapDatabase/SQLCipher/Core (3.0.2): + - YapDatabase/SQLCipher (3.1.1): + - YapDatabase/SQLCipher/Core (= 3.1.1) + - YapDatabase/SQLCipher/Extensions (= 3.1.1) + - YapDatabase/SQLCipher/Core (3.1.1): - CocoaLumberjack - SQLCipher (>= 3.4.0) - - YapDatabase/SQLCipher/Extensions (3.0.2): - - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/AutoView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/Hooks (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/ManualView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/Relationships (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/View (= 3.0.2) - - YapDatabase/SQLCipher/Extensions/ActionManager (3.0.2): + - YapDatabase/SQLCipher/Extensions (3.1.1): + - YapDatabase/SQLCipher/Core + - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/AutoView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ConnectionPool (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/Hooks (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ManualView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/Relationships (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/View (= 3.1.1) + - YapDatabase/SQLCipher/Extensions/ActionManager (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/AutoView - - YapDatabase/SQLCipher/Extensions/AutoView (3.0.2): + - YapDatabase/SQLCipher/Extensions/AutoView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/View - - YapDatabase/SQLCipher/Extensions/CloudCore (3.0.2): + - YapDatabase/SQLCipher/Extensions/CloudCore (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/CloudKit (3.0.2): + - YapDatabase/SQLCipher/Extensions/CloudKit (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.0.2): + - YapDatabase/SQLCipher/Extensions/ConnectionPool (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.0.2): + - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/FilteredView (3.0.2): + - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.1.1): + - YapDatabase/SQLCipher/Core + - YapDatabase/SQLCipher/Extensions/FilteredView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/View - - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.0.2): + - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/Hooks (3.0.2): + - YapDatabase/SQLCipher/Extensions/Hooks (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/ManualView (3.0.2): + - YapDatabase/SQLCipher/Extensions/ManualView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/View - - YapDatabase/SQLCipher/Extensions/Relationships (3.0.2): + - YapDatabase/SQLCipher/Extensions/Relationships (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.0.2): + - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.0.2): + - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.1.1): - YapDatabase/SQLCipher/Core - YapDatabase/SQLCipher/Extensions/AutoView - YapDatabase/SQLCipher/Extensions/FullTextSearch - - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.0.2): + - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.1.1): - YapDatabase/SQLCipher/Core - - YapDatabase/SQLCipher/Extensions/View (3.0.2): + - YapDatabase/SQLCipher/Extensions/View (3.1.1): - YapDatabase/SQLCipher/Core - YYImage (1.0.4): - YYImage/Core (= 1.0.4) @@ -147,7 +150,7 @@ DEPENDENCIES: - SocketRocket (from `https://github.com/facebook/SocketRocket.git`) - SQLCipher (from `https://github.com/sqlcipher/sqlcipher.git`, commit `d5c2bec`) - SSZipArchive - - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `release/unencryptedHeaders`) + - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `signal-release`) - YYImage SPEC REPOS: @@ -185,7 +188,7 @@ EXTERNAL SOURCES: :commit: d5c2bec :git: https://github.com/sqlcipher/sqlcipher.git YapDatabase: - :branch: release/unencryptedHeaders + :branch: signal-release :git: https://github.com/signalapp/YapDatabase.git CHECKOUT OPTIONS: @@ -208,7 +211,7 @@ CHECKOUT OPTIONS: :commit: d5c2bec :git: https://github.com/sqlcipher/sqlcipher.git YapDatabase: - :commit: 764e949142ba1bada99aeedeeaccfb68047a6e79 + :commit: f1fa4545e1e1594fb80065ffca52a682f5a2e71c :git: https://github.com/signalapp/YapDatabase.git SPEC CHECKSUMS: @@ -231,9 +234,9 @@ SPEC CHECKSUMS: SQLCipher: f9fcf29b2e59ced7defc2a2bdd0ebe79b40d4990 SSZipArchive: d4009d2ce5520a421f231fd97028cc0e2667eed8 SwiftProtobuf: 5ccc0e4054e37c75800e5744acb2aa80bb72b39c - YapDatabase: 299a32de9d350d37a9ac5b0532609d87d5d2a5de + YapDatabase: b418a4baa6906e8028748938f9159807fd039af4 YYImage: 1e1b62a9997399593e4b9c4ecfbbabbf1d3f3b54 -PODFILE CHECKSUM: db797890d3df475827063ba956cb42afa148ff19 +PODFILE CHECKSUM: b378e1ac40edbfc9d4f66410d26ea18a8a0a32dc COCOAPODS: 1.5.3 diff --git a/Pods b/Pods index 74a0f3857..9f36102cb 160000 --- a/Pods +++ b/Pods @@ -1 +1 @@ -Subproject commit 74a0f3857847cf9658b10b4bdce7b19f53e891ec +Subproject commit 9f36102cbbd5555045626b87fc2dde4dde05ab61 diff --git a/Signal/test/util/DisplayableTextFilterTest.swift b/Signal/test/util/DisplayableTextFilterTest.swift index 72e7297d2..e1dbb40d6 100644 --- a/Signal/test/util/DisplayableTextFilterTest.swift +++ b/Signal/test/util/DisplayableTextFilterTest.swift @@ -64,7 +64,14 @@ class DisplayableTextTest: XCTestCase { XCTAssertEqual("Příliš žluťoučký kůň úpěl ďábelské ódy.".glyphCount, 39) // Excessive diacritics - XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115) + + // some insignificant discrepencies across iOS versions + if #available(iOS 11, *) { + XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115) + } else { + XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 109) + } + XCTAssertEqual("L̷̳͔̲͝Ģ̵̮̯̤̩̙͍̬̟͉̹̘̹͍͈̮̦̰̣͟͝O̶̴̮̻̮̗͘͡!̴̷̟͓͓".glyphCount, 43) } diff --git a/Signal/test/util/OWSDatabaseConverterTest.m b/Signal/test/util/OWSDatabaseConverterTest.m index 85c6c8864..9219157aa 100644 --- a/Signal/test/util/OWSDatabaseConverterTest.m +++ b/Signal/test/util/OWSDatabaseConverterTest.m @@ -14,6 +14,13 @@ NS_ASSUME_NONNULL_BEGIN +@interface YapDatabase (OWSDatabaseConverterTest) + +- (void)flushInternalQueue; +- (void)flushCheckpointQueue; + +@end + @interface OWSStorage (OWSDatabaseConverterTest) + (YapDatabaseDeserializer)logOnFailureDeserializer; diff --git a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift index b95fbfc6b..f8636bd88 100644 --- a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift +++ b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift @@ -7,14 +7,14 @@ import Foundation // Create a searchable index for objects of type T public class SearchIndexer { - private let indexBlock: (T) -> String + private let indexBlock: (T, YapDatabaseReadTransaction) -> String - public init(indexBlock: @escaping (T) -> String) { + public init(indexBlock: @escaping (T, YapDatabaseReadTransaction) -> String) { self.indexBlock = indexBlock } - public func index(_ item: T) -> String { - return normalize(indexingText: indexBlock(item)) + public func index(_ item: T, transaction: YapDatabaseReadTransaction) -> String { + return normalize(indexingText: indexBlock(item, transaction)) } private func normalize(indexingText: String) -> String { @@ -157,22 +157,22 @@ public class FullTextSearchFinder: NSObject { return TextSecureKitEnv.shared().contactsManager } - private static let groupThreadIndexer: SearchIndexer = SearchIndexer { (groupThread: TSGroupThread) in + private static let groupThreadIndexer: SearchIndexer = SearchIndexer { (groupThread: TSGroupThread, transaction: YapDatabaseReadTransaction) in let groupName = groupThread.groupModel.groupName ?? "" let memberStrings = groupThread.groupModel.groupMemberIds.map { recipientId in - recipientIndexer.index(recipientId) + recipientIndexer.index(recipientId, transaction: transaction) }.joined(separator: " ") return "\(groupName) \(memberStrings)" } - private static let contactThreadIndexer: SearchIndexer = SearchIndexer { (contactThread: TSContactThread) in + private static let contactThreadIndexer: SearchIndexer = SearchIndexer { (contactThread: TSContactThread, transaction: YapDatabaseReadTransaction) in let recipientId = contactThread.contactIdentifier() - return recipientIndexer.index(recipientId) + return recipientIndexer.index(recipientId, transaction: transaction) } - private static let recipientIndexer: SearchIndexer = SearchIndexer { (recipientId: String) in + private static let recipientIndexer: SearchIndexer = SearchIndexer { (recipientId: String, _: YapDatabaseReadTransaction) in let displayName = contactsManager.displayName(forPhoneIdentifier: recipientId) let nationalNumber: String = { (recipientId: String) -> String in @@ -193,46 +193,46 @@ public class FullTextSearchFinder: NSObject { return "\(recipientId) \(nationalNumber) \(displayName)" } - private static let messageIndexer: SearchIndexer = SearchIndexer { (message: TSMessage) in + private static let messageIndexer: SearchIndexer = SearchIndexer { (message: TSMessage, transaction: YapDatabaseReadTransaction) in if let body = message.body, body.count > 0 { return body } - if let oversizeText = oversizeText(forMessage: message) { + if let oversizeText = oversizeText(forMessage: message, transaction: transaction) { return oversizeText } return "" } - private static func oversizeText(forMessage message: TSMessage) -> String? { + private static func oversizeText(forMessage message: TSMessage, transaction: YapDatabaseReadTransaction) -> String? { guard message.hasAttachments() else { return nil } - let dbConnection = OWSPrimaryStorage.shared().dbReadConnection - var oversizeText: String? - dbConnection.read({ (transaction) in - guard let attachment = message.attachment(with: transaction) else { - // This can happen during the initial save of incoming messages. - Logger.warn("Could not load attachment for search indexing.") - return - } - guard let attachmentStream = attachment as? TSAttachmentStream else { - return - } - guard attachmentStream.isOversizeText() else { - return - } - guard let text = attachmentStream.readOversizeText() else { - owsFail("Could not load oversize text attachment") - return - } - oversizeText = text - }) - return oversizeText + + guard let attachment = message.attachment(with: transaction) else { + // This can happen during the initial save of incoming messages. + Logger.warn("Could not load attachment for search indexing.") + return nil + } + + guard let attachmentStream = attachment as? TSAttachmentStream else { + return nil + } + + guard attachmentStream.isOversizeText() else { + return nil + } + + guard let text = attachmentStream.readOversizeText() else { + owsFail("Could not load oversize text attachment") + return nil + } + + return text } - private class func indexContent(object: Any) -> String? { + private class func indexContent(object: Any, transaction: YapDatabaseReadTransaction) -> String? { if let groupThread = object as? TSGroupThread { - return self.groupThreadIndexer.index(groupThread) + return self.groupThreadIndexer.index(groupThread, transaction: transaction) } else if let contactThread = object as? TSContactThread { guard contactThread.hasEverHadMessage else { // If we've never sent/received a message in a TSContactThread, @@ -240,11 +240,11 @@ public class FullTextSearchFinder: NSObject { // than in the "Conversations" section. return nil } - return self.contactThreadIndexer.index(contactThread) + return self.contactThreadIndexer.index(contactThread, transaction: transaction) } else if let message = object as? TSMessage { - return self.messageIndexer.index(message) + return self.messageIndexer.index(message, transaction: transaction) } else if let signalAccount = object as? SignalAccount { - return self.recipientIndexer.index(signalAccount.recipientId) + return self.recipientIndexer.index(signalAccount.recipientId, transaction: transaction) } else { return nil } @@ -277,8 +277,8 @@ public class FullTextSearchFinder: NSObject { let contentColumnName = "content" - let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (dict: NSMutableDictionary, _: String, _: String, object: Any) in - if let content: String = indexContent(object: object) { + let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (transaction: YapDatabaseReadTransaction, dict: NSMutableDictionary, _: String, _: String, object: Any) in + if let content: String = indexContent(object: object, transaction: transaction) { dict[contentColumnName] = content } }