Update to latest YapDB (with Signal patches applied)

Applied branches:

- mkirk/invalidFinalIndices
- mkirk/upstream-unencrypted-headers

Also fixed up test suite.

Podfile

@@ -8,7 +8,7 @@ def shared_pods
   # pod 'SQLCipher', path: '../sqlcipher2'
   pod 'SQLCipher', :git => 'https://github.com/sqlcipher/sqlcipher.git', :commit => 'd5c2bec'
   # pod 'YapDatabase/SQLCipher', path: '../YapDatabase'
-  pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'release/unencryptedHeaders'
+  pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'signal-release'
   # pod 'AxolotlKit',   path: '../SignalProtocolKit'
   pod 'SignalServiceKit', path: '.'
   pod 'AxolotlKit', git: 'https://github.com/signalapp/SignalProtocolKit.git'

Podfile.lock

@@ -70,64 +70,67 @@ PODS:
     - SQLCipher/common
   - SSZipArchive (2.1.2)
   - SwiftProtobuf (1.0.3)
-  - YapDatabase/SQLCipher (3.0.2):
-    - YapDatabase/SQLCipher/Core (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions (= 3.0.2)
-  - YapDatabase/SQLCipher/Core (3.0.2):
+  - YapDatabase/SQLCipher (3.1.1):
+    - YapDatabase/SQLCipher/Core (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions (= 3.1.1)
+  - YapDatabase/SQLCipher/Core (3.1.1):
     - CocoaLumberjack
     - SQLCipher (>= 3.4.0)
-  - YapDatabase/SQLCipher/Extensions (3.0.2):
-    - YapDatabase/SQLCipher/Core
-    - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/AutoView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/Hooks (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/ManualView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/Relationships (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/View (= 3.0.2)
-  - YapDatabase/SQLCipher/Extensions/ActionManager (3.0.2):
+  - YapDatabase/SQLCipher/Extensions (3.1.1):
+    - YapDatabase/SQLCipher/Core
+    - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/AutoView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/ConnectionPool (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/Hooks (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/ManualView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/Relationships (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/View (= 3.1.1)
+  - YapDatabase/SQLCipher/Extensions/ActionManager (3.1.1):
     - YapDatabase/SQLCipher/Core
     - YapDatabase/SQLCipher/Extensions/AutoView
-  - YapDatabase/SQLCipher/Extensions/AutoView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/AutoView (3.1.1):
     - YapDatabase/SQLCipher/Core
     - YapDatabase/SQLCipher/Extensions/View
-  - YapDatabase/SQLCipher/Extensions/CloudCore (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/CloudCore (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/CloudKit (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/CloudKit (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/ConnectionPool (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/FilteredView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.1.1):
+    - YapDatabase/SQLCipher/Core
+  - YapDatabase/SQLCipher/Extensions/FilteredView (3.1.1):
     - YapDatabase/SQLCipher/Core
     - YapDatabase/SQLCipher/Extensions/View
-  - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/Hooks (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/Hooks (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/ManualView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/ManualView (3.1.1):
     - YapDatabase/SQLCipher/Core
     - YapDatabase/SQLCipher/Extensions/View
-  - YapDatabase/SQLCipher/Extensions/Relationships (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/Relationships (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.1.1):
     - YapDatabase/SQLCipher/Core
     - YapDatabase/SQLCipher/Extensions/AutoView
     - YapDatabase/SQLCipher/Extensions/FullTextSearch
-  - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.1.1):
     - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/View (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/View (3.1.1):
     - YapDatabase/SQLCipher/Core
   - YYImage (1.0.4):
     - YYImage/Core (= 1.0.4)
@@ -147,7 +150,7 @@ DEPENDENCIES:
   - SocketRocket (from `https://github.com/facebook/SocketRocket.git`)
   - SQLCipher (from `https://github.com/sqlcipher/sqlcipher.git`, commit `d5c2bec`)
   - SSZipArchive
-  - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `release/unencryptedHeaders`)
+  - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `signal-release`)
   - YYImage
 
 SPEC REPOS:
@@ -185,7 +188,7 @@ EXTERNAL SOURCES:
     :commit: d5c2bec
     :git: https://github.com/sqlcipher/sqlcipher.git
   YapDatabase:
-    :branch: release/unencryptedHeaders
+    :branch: signal-release
     :git: https://github.com/signalapp/YapDatabase.git
 
 CHECKOUT OPTIONS:
@@ -208,7 +211,7 @@ CHECKOUT OPTIONS:
     :commit: d5c2bec
     :git: https://github.com/sqlcipher/sqlcipher.git
   YapDatabase:
-    :commit: 764e949142ba1bada99aeedeeaccfb68047a6e79
+    :commit: f1fa4545e1e1594fb80065ffca52a682f5a2e71c
     :git: https://github.com/signalapp/YapDatabase.git
 
 SPEC CHECKSUMS:
@@ -231,9 +234,9 @@ SPEC CHECKSUMS:
   SQLCipher: f9fcf29b2e59ced7defc2a2bdd0ebe79b40d4990
   SSZipArchive: d4009d2ce5520a421f231fd97028cc0e2667eed8
   SwiftProtobuf: 5ccc0e4054e37c75800e5744acb2aa80bb72b39c
-  YapDatabase: 299a32de9d350d37a9ac5b0532609d87d5d2a5de
+  YapDatabase: b418a4baa6906e8028748938f9159807fd039af4
   YYImage: 1e1b62a9997399593e4b9c4ecfbbabbf1d3f3b54
 
-PODFILE CHECKSUM: db797890d3df475827063ba956cb42afa148ff19
+PODFILE CHECKSUM: b378e1ac40edbfc9d4f66410d26ea18a8a0a32dc
 
 COCOAPODS: 1.5.3

Pods

@@ -1 +1 @@
-Subproject commit 74a0f3857847cf9658b10b4bdce7b19f53e891ec
+Subproject commit 9f36102cbbd5555045626b87fc2dde4dde05ab61

Signal/test/util/DisplayableTextFilterTest.swift

@@ -64,7 +64,14 @@ class DisplayableTextTest: XCTestCase {
         XCTAssertEqual("Příliš žluťoučký kůň úpěl ďábelské ódy.".glyphCount, 39)
 
         // Excessive diacritics
-        XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115)
+
+        // some insignificant discrepencies across iOS versions        
+        if #available(iOS 11, *) {
+            XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115)
+        } else {
+            XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 109)
+        }
+
         XCTAssertEqual("L̷̳͔̲͝Ģ̵̮̯̤̩̙͍̬̟͉̹̘̹͍͈̮̦̰̣͟͝O̶̴̮̻̮̗͘͡!̴̷̟͓͓".glyphCount, 43)
     }
 

Signal/test/util/OWSDatabaseConverterTest.m

@@ -14,6 +14,13 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
+@interface YapDatabase (OWSDatabaseConverterTest)
+
+- (void)flushInternalQueue;
+- (void)flushCheckpointQueue;
+
+@end
+
 @interface OWSStorage (OWSDatabaseConverterTest)
 
 + (YapDatabaseDeserializer)logOnFailureDeserializer;

SignalServiceKit/src/Storage/FullTextSearchFinder.swift

@@ -7,14 +7,14 @@ import Foundation
 // Create a searchable index for objects of type T
 public class SearchIndexer<T> {
 
-    private let indexBlock: (T) -> String
+    private let indexBlock: (T, YapDatabaseReadTransaction) -> String
 
-    public init(indexBlock: @escaping (T) -> String) {
+    public init(indexBlock: @escaping (T, YapDatabaseReadTransaction) -> String) {
         self.indexBlock = indexBlock
     }
 
-    public func index(_ item: T) -> String {
-        return normalize(indexingText: indexBlock(item))
+    public func index(_ item: T, transaction: YapDatabaseReadTransaction) -> String {
+        return normalize(indexingText: indexBlock(item, transaction))
     }
 
     private func normalize(indexingText: String) -> String {
@@ -157,22 +157,22 @@ public class FullTextSearchFinder: NSObject {
         return TextSecureKitEnv.shared().contactsManager
     }
 
-    private static let groupThreadIndexer: SearchIndexer<TSGroupThread> = SearchIndexer { (groupThread: TSGroupThread) in
+    private static let groupThreadIndexer: SearchIndexer<TSGroupThread> = SearchIndexer { (groupThread: TSGroupThread, transaction: YapDatabaseReadTransaction) in
         let groupName = groupThread.groupModel.groupName ?? ""
 
         let memberStrings = groupThread.groupModel.groupMemberIds.map { recipientId in
-            recipientIndexer.index(recipientId)
+            recipientIndexer.index(recipientId, transaction: transaction)
         }.joined(separator: " ")
 
         return "\(groupName) \(memberStrings)"
     }
 
-    private static let contactThreadIndexer: SearchIndexer<TSContactThread> = SearchIndexer { (contactThread: TSContactThread) in
+    private static let contactThreadIndexer: SearchIndexer<TSContactThread> = SearchIndexer { (contactThread: TSContactThread, transaction: YapDatabaseReadTransaction) in
         let recipientId =  contactThread.contactIdentifier()
-        return recipientIndexer.index(recipientId)
+        return recipientIndexer.index(recipientId, transaction: transaction)
     }
 
-    private static let recipientIndexer: SearchIndexer<String> = SearchIndexer { (recipientId: String) in
+    private static let recipientIndexer: SearchIndexer<String> = SearchIndexer { (recipientId: String, _: YapDatabaseReadTransaction) in
         let displayName = contactsManager.displayName(forPhoneIdentifier: recipientId)
 
         let nationalNumber: String = { (recipientId: String) -> String in
@@ -193,46 +193,46 @@ public class FullTextSearchFinder: NSObject {
         return "\(recipientId) \(nationalNumber) \(displayName)"
     }
 
-    private static let messageIndexer: SearchIndexer<TSMessage> = SearchIndexer { (message: TSMessage) in
+    private static let messageIndexer: SearchIndexer<TSMessage> = SearchIndexer { (message: TSMessage, transaction: YapDatabaseReadTransaction) in
         if let body = message.body, body.count > 0 {
             return body
         }
-        if let oversizeText = oversizeText(forMessage: message) {
+        if let oversizeText = oversizeText(forMessage: message, transaction: transaction) {
             return oversizeText
         }
         return ""
     }
 
-    private static func oversizeText(forMessage message: TSMessage) -> String? {
+    private static func oversizeText(forMessage message: TSMessage, transaction: YapDatabaseReadTransaction) -> String? {
         guard message.hasAttachments() else {
             return nil
         }
-        let dbConnection = OWSPrimaryStorage.shared().dbReadConnection
-        var oversizeText: String?
-        dbConnection.read({ (transaction) in
-            guard let attachment = message.attachment(with: transaction) else {
-                // This can happen during the initial save of incoming messages.
-                Logger.warn("Could not load attachment for search indexing.")
-                return
-            }
-            guard let attachmentStream = attachment as? TSAttachmentStream else {
-                return
-            }
-            guard attachmentStream.isOversizeText() else {
-                return
-            }
-            guard let text = attachmentStream.readOversizeText() else {
-                owsFail("Could not load oversize text attachment")
-                return
-            }
-            oversizeText = text
-        })
-        return oversizeText
+
+        guard let attachment = message.attachment(with: transaction) else {
+            // This can happen during the initial save of incoming messages.
+            Logger.warn("Could not load attachment for search indexing.")
+            return nil
+        }
+
+        guard let attachmentStream = attachment as? TSAttachmentStream else {
+            return nil
+        }
+
+        guard attachmentStream.isOversizeText() else {
+            return nil
+        }
+
+        guard let text = attachmentStream.readOversizeText() else {
+            owsFail("Could not load oversize text attachment")
+            return nil
+        }
+
+        return text
     }
 
-    private class func indexContent(object: Any) -> String? {
+    private class func indexContent(object: Any, transaction: YapDatabaseReadTransaction) -> String? {
         if let groupThread = object as? TSGroupThread {
-            return self.groupThreadIndexer.index(groupThread)
+            return self.groupThreadIndexer.index(groupThread, transaction: transaction)
         } else if let contactThread = object as? TSContactThread {
             guard contactThread.hasEverHadMessage else {
                 // If we've never sent/received a message in a TSContactThread,
@@ -240,11 +240,11 @@ public class FullTextSearchFinder: NSObject {
                 // than in the "Conversations" section.
                 return nil
             }
-            return self.contactThreadIndexer.index(contactThread)
+            return self.contactThreadIndexer.index(contactThread, transaction: transaction)
         } else if let message = object as? TSMessage {
-            return self.messageIndexer.index(message)
+            return self.messageIndexer.index(message, transaction: transaction)
         } else if let signalAccount = object as? SignalAccount {
-            return self.recipientIndexer.index(signalAccount.recipientId)
+            return self.recipientIndexer.index(signalAccount.recipientId, transaction: transaction)
         } else {
             return nil
         }
@@ -277,8 +277,8 @@ public class FullTextSearchFinder: NSObject {
 
         let contentColumnName = "content"
 
-        let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (dict: NSMutableDictionary, _: String, _: String, object: Any) in
-            if let content: String = indexContent(object: object) {
+        let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (transaction: YapDatabaseReadTransaction, dict: NSMutableDictionary, _: String, _: String, object: Any) in
+            if let content: String = indexContent(object: object, transaction: transaction) {
                 dict[contentColumnName] = content
             }
         }