Merge pull request #596 from mpretty-cyro/feature/improve-open-group-deletion-processing

Performance: Improve open group deletion processing

Session.xcodeproj/project.pbxproj

@@ -774,6 +774,8 @@
 		F5765D284BC6ECAC0C1D33F0 /* Pods_GlobalDependencies_FrameworkAndExtensionDependencies_SessionNotificationServiceExtension.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = E4A93ECA93B3DE800CC7D7F6 /* Pods_GlobalDependencies_FrameworkAndExtensionDependencies_SessionNotificationServiceExtension.framework */; };
 		FC3BD9881A30A790005B96BB /* Social.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FC3BD9871A30A790005B96BB /* Social.framework */; };
 		FCB11D8C1A129A76002F93FB /* CoreMedia.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FCB11D8B1A129A76002F93FB /* CoreMedia.framework */; };
+		FD3C907327E8387300CD579F /* OpenGroupServerIdLookupMigration.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD3C907227E8387300CD579F /* OpenGroupServerIdLookupMigration.swift */; };
+		FD3C907527E83AC200CD579F /* OpenGroupServerIdLookup.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD3C907427E83AC200CD579F /* OpenGroupServerIdLookup.swift */; };
 		FD5D200F27AA2B6000FEA984 /* MessageRequestResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD5D200E27AA2B6000FEA984 /* MessageRequestResponse.swift */; };
 		FD5D201127AA331F00FEA984 /* ConfigurationMessage+Convenience.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD5D201027AA331F00FEA984 /* ConfigurationMessage+Convenience.swift */; };
 		FD659AC027A7649600F12C02 /* MessageRequestsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD659ABF27A7649600F12C02 /* MessageRequestsViewController.swift */; };
@@ -1817,6 +1819,8 @@
 		F9BBF530D71905BA9007675F /* Pods-SessionShareExtension.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SessionShareExtension.debug.xcconfig"; path = "Pods/Target Support Files/Pods-SessionShareExtension/Pods-SessionShareExtension.debug.xcconfig"; sourceTree = "<group>"; };
 		FC3BD9871A30A790005B96BB /* Social.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Social.framework; path = System/Library/Frameworks/Social.framework; sourceTree = SDKROOT; };
 		FCB11D8B1A129A76002F93FB /* CoreMedia.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreMedia.framework; path = System/Library/Frameworks/CoreMedia.framework; sourceTree = SDKROOT; };
+		FD3C907227E8387300CD579F /* OpenGroupServerIdLookupMigration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenGroupServerIdLookupMigration.swift; sourceTree = "<group>"; };
+		FD3C907427E83AC200CD579F /* OpenGroupServerIdLookup.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenGroupServerIdLookup.swift; sourceTree = "<group>"; };
 		FD5D200E27AA2B6000FEA984 /* MessageRequestResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessageRequestResponse.swift; sourceTree = "<group>"; };
 		FD5D201027AA331F00FEA984 /* ConfigurationMessage+Convenience.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ConfigurationMessage+Convenience.swift"; sourceTree = "<group>"; };
 		FD659ABF27A7649600F12C02 /* MessageRequestsViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessageRequestsViewController.swift; sourceTree = "<group>"; };
@@ -3065,6 +3069,7 @@
 			children = (
 				B8B32044258C117C0020074B /* ContactsMigration.swift */,
 				FD88BADA27A750F200BBC442 /* MessageRequestsMigration.swift */,
+				FD3C907227E8387300CD579F /* OpenGroupServerIdLookupMigration.swift */,
 				C38EF271255B6D79007E1867 /* OWSDatabaseMigration.h */,
 				C38EF270255B6D79007E1867 /* OWSDatabaseMigration.m */,
 				C38EF26F255B6D79007E1867 /* OWSDatabaseMigrationRunner.h */,
@@ -3194,6 +3199,7 @@
 			isa = PBXGroup;
 			children = (
 				C3DB6694260AC923001EFC55 /* OpenGroupV2.swift */,
+				FD3C907427E83AC200CD579F /* OpenGroupServerIdLookup.swift */,
 				B88FA7B726045D100049422F /* OpenGroupAPIV2.swift */,
 				C3DB66CB260AF1F3001EFC55 /* OpenGroupAPIV2+ObjC.swift */,
 				C3DB66AB260ACA42001EFC55 /* OpenGroupManagerV2.swift */,
@@ -4533,6 +4539,7 @@
 				C38EF407255B6DF7007E1867 /* Toast.swift in Sources */,
 				C38EF38C255B6DD2007E1867 /* ApprovalRailCellView.swift in Sources */,
 				C38EF409255B6DF7007E1867 /* ContactTableViewCell.m in Sources */,
+				FD3C907327E8387300CD579F /* OpenGroupServerIdLookupMigration.swift in Sources */,
 				C38EF32A255B6DBF007E1867 /* UIUtil.m in Sources */,
 				C38EF335255B6DBF007E1867 /* BlockListCache.swift in Sources */,
 				C38EF2A6255B6D93007E1867 /* PlaceholderIcon.swift in Sources */,
@@ -4770,6 +4777,7 @@
 				B8EB20EE2640F28000773E52 /* VisibleMessage+OpenGroupInvitation.swift in Sources */,
 				C3C2A7712553A41E00C340D1 /* ControlMessage.swift in Sources */,
 				C32C5D19256DD493003C73A2 /* OWSLinkPreview.swift in Sources */,
+				FD3C907527E83AC200CD579F /* OpenGroupServerIdLookup.swift in Sources */,
 				C32C5CF0256DD3E4003C73A2 /* Storage+Shared.swift in Sources */,
 				C300A5BD2554B00D00555489 /* ReadReceipt.swift in Sources */,
 				C32C5AB5256DBE8F003C73A2 /* TSOutgoingMessage+Conversion.swift in Sources */,

SessionMessagingKit/Database/Storage+OpenGroups.swift

@@ -152,8 +152,31 @@ extension Storage {
         let key = "\(server).\(room)"
         (transaction as! YapDatabaseReadWriteTransaction).removeObject(forKey: key, inCollection: collection)
     }
+    
+    // MARK: - OpenGroupServerIdToUniqueIdLookup
+
+    public static let openGroupServerIdToUniqueIdLookupCollection = "SNOpenGroupServerIdToUniqueIdLookup"
+
+    public func getOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadTransaction) -> OpenGroupServerIdLookup? {
+        let key: String = OpenGroupServerIdLookup.id(serverId: serverId, in: room, on: server)
+        return transaction.object(forKey: key, inCollection: Storage.openGroupServerIdToUniqueIdLookupCollection) as? OpenGroupServerIdLookup
+    }
 
+    public func addOpenGroupServerIdLookup(_ serverId: UInt64?, tsMessageId: String?, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction) {
+        guard let serverId: UInt64 = serverId, let tsMessageId: String = tsMessageId else { return }
+        
+        let lookup: OpenGroupServerIdLookup = OpenGroupServerIdLookup(server: server, room: room, serverId: serverId, tsMessageId: tsMessageId)
+        addOpenGroupServerIdLookup(lookup, using: transaction)
+    }
+    
+    public func addOpenGroupServerIdLookup(_ lookup: OpenGroupServerIdLookup, using transaction: YapDatabaseReadWriteTransaction) {
+        transaction.setObject(lookup, forKey: lookup.id, inCollection: Storage.openGroupServerIdToUniqueIdLookupCollection)
+    }
 
+    public func removeOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction) {
+        let key: String = OpenGroupServerIdLookup.id(serverId: serverId, in: room, on: server)
+        transaction.removeObject(forKey: key, inCollection: Storage.openGroupServerIdToUniqueIdLookupCollection)
+    }
 
     // MARK: - Metadata
 

SessionMessagingKit/Open Groups/OpenGroupServerIdLookup.swift

@@ -0,0 +1,46 @@
+// Copyright © 2022 Rangeproof Pty Ltd. All rights reserved.
+
+import Foundation
+
+@objc(SNOpenGroupServerIdLookup)
+public final class OpenGroupServerIdLookup: NSObject, NSCoding {    // NSObject/NSCoding conformance is needed for YapDatabase compatibility
+    @objc public let id: String
+    @objc public let serverId: UInt64
+    @objc public let tsMessageId: String
+    
+    // MARK: - Initialization
+        
+    @objc public init(server: String, room: String, serverId: UInt64, tsMessageId: String) {
+        self.id = OpenGroupServerIdLookup.id(serverId: serverId, in: room, on: server)
+        self.serverId = serverId
+        self.tsMessageId = tsMessageId
+
+        super.init()
+    }
+
+    private override init() { preconditionFailure("Use init(blindedId:sessionId:) instead.") }
+
+    // MARK: - Coding
+    
+    public required init?(coder: NSCoder) {
+        guard let id: String = coder.decodeObject(forKey: "id") as! String? else { return nil }
+        guard let serverId: UInt64 = coder.decodeObject(forKey: "serverId") as! UInt64? else { return nil }
+        guard let tsMessageId: String = coder.decodeObject(forKey: "tsMessageId") as! String? else { return nil }
+
+        self.id = id
+        self.serverId = serverId
+        self.tsMessageId = tsMessageId
+    }
+
+    public func encode(with coder: NSCoder) {
+        coder.encode(id, forKey: "id")
+        coder.encode(serverId, forKey: "serverId")
+        coder.encode(tsMessageId, forKey: "tsMessageId")
+    }
+    
+    // MARK: - Convenience
+    
+    static func id(serverId: UInt64, in room: String, on server: String) -> String {
+        return "\(server).\(room).\(serverId)"
+    }
+}

SessionMessagingKit/Sending & Receiving/MessageReceiver+Handling.swift

@@ -385,7 +385,15 @@ extension MessageReceiver {
         }
         if let tsMessage = TSMessage.fetch(uniqueId: tsMessageID, transaction: transaction) {
             // Keep track of the open group server message ID ↔ message ID relationship
-            if let serverID = message.openGroupServerMessageID { tsMessage.openGroupServerMessageID = serverID }
+            if let serverID = message.openGroupServerMessageID {
+                tsMessage.openGroupServerMessageID = serverID
+                
+                // Create a lookup between the openGroupServerMessageId and the tsMessage id for easy lookup
+                if let openGroup: OpenGroupV2 = storage.getV2OpenGroup(for: threadID) {
+                    storage.addOpenGroupServerIdLookup(serverID, tsMessageId: tsMessageID, in: openGroup.room, on: openGroup.server, using: transaction)
+                }
+            }
+            
             // Keep track of server hash
             if let serverHash = message.serverHash { tsMessage.serverHash = serverHash }
              tsMessage.save(with: transaction)

SessionMessagingKit/Sending & Receiving/MessageSender.swift

@@ -313,6 +313,7 @@ public final class MessageSender : NSObject {
             base64EncodedData: plaintext.base64EncodedString(), base64EncodedSignature: nil)
         OpenGroupAPIV2.send(openGroupMessage, to: room, on: server).done(on: DispatchQueue.global(qos: .userInitiated)) { openGroupMessage in
             message.openGroupServerMessageID = given(openGroupMessage.serverID) { UInt64($0) }
+
             storage.write(with: { transaction in
                 MessageSender.handleSuccessfulMessageSend(message, to: destination, serverTimestamp: openGroupMessage.sentTimestamp, using: transaction)
                 seal.fulfill(())
@@ -341,6 +342,20 @@ public final class MessageSender : NSObject {
                 // Otherwise the quote messages may not be able
                 // to be found by the timestamp on other devices
                 tsMessage.updateOpenGroupServerID(openGroupServerMessageID, serverTimeStamp: timestamp)
+                
+                // Create a lookup between the openGroupServerMessageId and the tsMessage id for easy lookup
+                switch destination {
+                    case .openGroupV2(let room, let server):
+                        Storage.shared.addOpenGroupServerIdLookup(
+                            openGroupServerMessageID,
+                            tsMessageId: tsMessage.uniqueId,
+                            in: room,
+                            on: server,
+                            using: transaction
+                        )
+                        
+                    default: break
+                }
             }
             // Mark the message as sent
             var recipients = [ message.recipient! ]

SessionMessagingKit/Sending & Receiving/Pollers/OpenGroupPollerV2.swift

@@ -82,6 +82,7 @@ public final class OpenGroupPollerV2 : NSObject {
                 }
             }
         }
+        
         // - Moderators
         if var x = OpenGroupAPIV2.moderators[server] {
             x[body.room] = Set(body.moderators)
@@ -89,18 +90,23 @@ public final class OpenGroupPollerV2 : NSObject {
         } else {
             OpenGroupAPIV2.moderators[server] = [ body.room : Set(body.moderators) ]
         }
+        
         // - Deletions
+        guard !body.deletions.isEmpty else { return }
+        
         let deletedMessageServerIDs = Set(body.deletions.map { UInt64($0.deletedMessageID) })
         storage.write { transaction in
-            let transaction = transaction as! YapDatabaseReadWriteTransaction
+            guard let transaction: YapDatabaseReadWriteTransaction = transaction as? YapDatabaseReadWriteTransaction else { return }
-            guard let threadID = storage.v2GetThreadID(for: openGroupID),
+            
-                let thread = TSGroupThread.fetch(uniqueId: threadID, transaction: transaction) else { return }
+            deletedMessageServerIDs.forEach { openGroupServerMessageId in
-            var messagesToRemove: [TSMessage] = []
+                guard let messageLookup: OpenGroupServerIdLookup = storage.getOpenGroupServerIdLookup(openGroupServerMessageId, in: body.room, on: self.server, using: transaction) else {
-            thread.enumerateInteractions(with: transaction) { interaction, stop in
+                    return
-                guard let message = interaction as? TSMessage, deletedMessageServerIDs.contains(message.openGroupServerMessageID) else { return }
+                }
-                messagesToRemove.append(message)
+                guard let tsMessage: TSMessage = TSMessage.fetch(uniqueId: messageLookup.tsMessageId, transaction: transaction) else { return }
+                
+                tsMessage.remove(with: transaction)
+                storage.removeOpenGroupServerIdLookup(openGroupServerMessageId, in: body.room, on: self.server, using: transaction)
             }
-            messagesToRemove.forEach { $0.remove(with: transaction) }
         }
     }
 }

SessionMessagingKit/Storage.swift

@@ -70,6 +70,13 @@ public protocol SessionMessagingKitStorageProtocol {
     func getLastDeletionServerID(for room: String, on server: String) -> Int64?
     func setLastDeletionServerID(for room: String, on server: String, to newValue: Int64, using transaction: Any)
     func removeLastDeletionServerID(for room: String, on server: String, using transaction: Any)
+    
+    // MARK: - OpenGroupServerIdToUniqueIdLookup
+
+    func getOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadTransaction) -> OpenGroupServerIdLookup?
+    func addOpenGroupServerIdLookup(_ serverId: UInt64?, tsMessageId: String?, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction)
+    func addOpenGroupServerIdLookup(_ lookup: OpenGroupServerIdLookup, using transaction: YapDatabaseReadWriteTransaction)
+    func removeOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction)
 
     // MARK: - Open Group Metadata
 

SignalUtilitiesKit/Database/Migrations/OWSDatabaseMigrationRunner.m

@@ -26,6 +26,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (NSArray<OWSDatabaseMigration *> *)allMigrations
 {
     return @[
+        [SNOpenGroupServerIdLookupMigration new],
         [SNMessageRequestsMigration new],
         [SNContactsMigration new]
     ];

SignalUtilitiesKit/Database/Migrations/OpenGroupServerIdLookupMigration.swift

@@ -0,0 +1,48 @@
+// Copyright © 2022 Rangeproof Pty Ltd. All rights reserved.
+
+import Foundation
+
+@objc(SNOpenGroupServerIdLookupMigration)
+public class OpenGroupServerIdLookupMigration: OWSDatabaseMigration {
+    @objc
+    class func migrationId() -> String {
+        return "003"
+    }
+
+    override public func runUp(completion: @escaping OWSDatabaseMigrationCompletion) {
+        self.doMigrationAsync(completion: completion)
+    }
+
+    private func doMigrationAsync(completion: @escaping OWSDatabaseMigrationCompletion) {
+        var lookups: [OpenGroupServerIdLookup] = []
+        
+        Storage.write(with: { transaction in
+            TSGroupThread.enumerateCollectionObjects(with: transaction) { object, _ in
+                guard let thread: TSGroupThread = object as? TSGroupThread else { return }
+                guard let threadId: String = thread.uniqueId else { return }
+                guard let openGroup: OpenGroupV2 = Storage.shared.getV2OpenGroup(for: threadId) else { return }
+                
+                thread.enumerateInteractions(with: transaction) { interaction, _ in
+                    guard let tsMessage: TSMessage = interaction as? TSMessage else { return }
+                    guard let tsMessageId: String = tsMessage.uniqueId else { return }
+                    
+                    lookups.append(
+                        OpenGroupServerIdLookup(
+                            server: openGroup.server,
+                            room: openGroup.room,
+                            serverId: tsMessage.openGroupServerMessageID,
+                            tsMessageId: tsMessageId
+                        )
+                    )
+                }
+            }
+            
+            lookups.forEach { lookup in
+                Storage.shared.addOpenGroupServerIdLookup(lookup, using: transaction)
+            }
+            self.save(with: transaction) // Intentionally capture self
+        }, completion: {
+            completion()
+        })
+    }
+}