From 1492232224d06025b6c1819ec547f8cc6742c3b3 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Mon, 21 Mar 2022 16:53:12 +1100 Subject: [PATCH] Made some optimisations to open group deletion handling Added a new OpenGroupServerIdLookup to make it easier to get a database id via an open group server id Added a migration to generate the above data Updated the handleCompactBody method to stop early if there are no messages to delete (would previously enumerate all interactions regardless) Updated the handleCompactBody to fetch the mapping for the deleted message so we can avoid enumerating all interactions --- Session.xcodeproj/project.pbxproj | 8 ++++ .../Database/Storage+OpenGroups.swift | 23 +++++++++ .../Open Groups/OpenGroupServerIdLookup.swift | 46 ++++++++++++++++++ .../MessageReceiver+Handling.swift | 10 +++- .../Sending & Receiving/MessageSender.swift | 15 ++++++ .../Pollers/OpenGroupPollerV2.swift | 22 +++++---- SessionMessagingKit/Storage.swift | 7 +++ .../Migrations/OWSDatabaseMigrationRunner.m | 1 + .../OpenGroupServerIdLookupMigration.swift | 48 +++++++++++++++++++ 9 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 SessionMessagingKit/Open Groups/OpenGroupServerIdLookup.swift create mode 100644 SignalUtilitiesKit/Database/Migrations/OpenGroupServerIdLookupMigration.swift diff --git a/Session.xcodeproj/project.pbxproj b/Session.xcodeproj/project.pbxproj index 592b9855d..e15463ec9 100644 --- a/Session.xcodeproj/project.pbxproj +++ b/Session.xcodeproj/project.pbxproj @@ -774,6 +774,8 @@ F5765D284BC6ECAC0C1D33F0 /* Pods_GlobalDependencies_FrameworkAndExtensionDependencies_SessionNotificationServiceExtension.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = E4A93ECA93B3DE800CC7D7F6 /* Pods_GlobalDependencies_FrameworkAndExtensionDependencies_SessionNotificationServiceExtension.framework */; }; FC3BD9881A30A790005B96BB /* Social.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FC3BD9871A30A790005B96BB /* Social.framework */; }; FCB11D8C1A129A76002F93FB /* CoreMedia.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FCB11D8B1A129A76002F93FB /* CoreMedia.framework */; }; + FD3C907327E8387300CD579F /* OpenGroupServerIdLookupMigration.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD3C907227E8387300CD579F /* OpenGroupServerIdLookupMigration.swift */; }; + FD3C907527E83AC200CD579F /* OpenGroupServerIdLookup.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD3C907427E83AC200CD579F /* OpenGroupServerIdLookup.swift */; }; FD5D200F27AA2B6000FEA984 /* MessageRequestResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD5D200E27AA2B6000FEA984 /* MessageRequestResponse.swift */; }; FD5D201127AA331F00FEA984 /* ConfigurationMessage+Convenience.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD5D201027AA331F00FEA984 /* ConfigurationMessage+Convenience.swift */; }; FD659AC027A7649600F12C02 /* MessageRequestsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD659ABF27A7649600F12C02 /* MessageRequestsViewController.swift */; }; @@ -1816,6 +1818,8 @@ F9BBF530D71905BA9007675F /* Pods-SessionShareExtension.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SessionShareExtension.debug.xcconfig"; path = "Pods/Target Support Files/Pods-SessionShareExtension/Pods-SessionShareExtension.debug.xcconfig"; sourceTree = ""; }; FC3BD9871A30A790005B96BB /* Social.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Social.framework; path = System/Library/Frameworks/Social.framework; sourceTree = SDKROOT; }; FCB11D8B1A129A76002F93FB /* CoreMedia.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreMedia.framework; path = System/Library/Frameworks/CoreMedia.framework; sourceTree = SDKROOT; }; + FD3C907227E8387300CD579F /* OpenGroupServerIdLookupMigration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenGroupServerIdLookupMigration.swift; sourceTree = ""; }; + FD3C907427E83AC200CD579F /* OpenGroupServerIdLookup.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenGroupServerIdLookup.swift; sourceTree = ""; }; FD5D200E27AA2B6000FEA984 /* MessageRequestResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessageRequestResponse.swift; sourceTree = ""; }; FD5D201027AA331F00FEA984 /* ConfigurationMessage+Convenience.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ConfigurationMessage+Convenience.swift"; sourceTree = ""; }; FD659ABF27A7649600F12C02 /* MessageRequestsViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessageRequestsViewController.swift; sourceTree = ""; }; @@ -3062,6 +3066,7 @@ children = ( B8B32044258C117C0020074B /* ContactsMigration.swift */, FD88BADA27A750F200BBC442 /* MessageRequestsMigration.swift */, + FD3C907227E8387300CD579F /* OpenGroupServerIdLookupMigration.swift */, C38EF271255B6D79007E1867 /* OWSDatabaseMigration.h */, C38EF270255B6D79007E1867 /* OWSDatabaseMigration.m */, C38EF26F255B6D79007E1867 /* OWSDatabaseMigrationRunner.h */, @@ -3191,6 +3196,7 @@ isa = PBXGroup; children = ( C3DB6694260AC923001EFC55 /* OpenGroupV2.swift */, + FD3C907427E83AC200CD579F /* OpenGroupServerIdLookup.swift */, B88FA7B726045D100049422F /* OpenGroupAPIV2.swift */, C3DB66CB260AF1F3001EFC55 /* OpenGroupAPIV2+ObjC.swift */, C3DB66AB260ACA42001EFC55 /* OpenGroupManagerV2.swift */, @@ -4530,6 +4536,7 @@ C38EF407255B6DF7007E1867 /* Toast.swift in Sources */, C38EF38C255B6DD2007E1867 /* ApprovalRailCellView.swift in Sources */, C38EF409255B6DF7007E1867 /* ContactTableViewCell.m in Sources */, + FD3C907327E8387300CD579F /* OpenGroupServerIdLookupMigration.swift in Sources */, C38EF32A255B6DBF007E1867 /* UIUtil.m in Sources */, C38EF335255B6DBF007E1867 /* BlockListCache.swift in Sources */, C38EF2A6255B6D93007E1867 /* PlaceholderIcon.swift in Sources */, @@ -4766,6 +4773,7 @@ B8EB20EE2640F28000773E52 /* VisibleMessage+OpenGroupInvitation.swift in Sources */, C3C2A7712553A41E00C340D1 /* ControlMessage.swift in Sources */, C32C5D19256DD493003C73A2 /* OWSLinkPreview.swift in Sources */, + FD3C907527E83AC200CD579F /* OpenGroupServerIdLookup.swift in Sources */, C32C5CF0256DD3E4003C73A2 /* Storage+Shared.swift in Sources */, C300A5BD2554B00D00555489 /* ReadReceipt.swift in Sources */, C32C5AB5256DBE8F003C73A2 /* TSOutgoingMessage+Conversion.swift in Sources */, diff --git a/SessionMessagingKit/Database/Storage+OpenGroups.swift b/SessionMessagingKit/Database/Storage+OpenGroups.swift index 5ea663550..ddcbc5268 100644 --- a/SessionMessagingKit/Database/Storage+OpenGroups.swift +++ b/SessionMessagingKit/Database/Storage+OpenGroups.swift @@ -152,8 +152,31 @@ extension Storage { let key = "\(server).\(room)" (transaction as! YapDatabaseReadWriteTransaction).removeObject(forKey: key, inCollection: collection) } + + // MARK: - OpenGroupServerIdToUniqueIdLookup + + public static let openGroupServerIdToUniqueIdLookupCollection = "SNOpenGroupServerIdToUniqueIdLookup" + + public func getOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadTransaction) -> OpenGroupServerIdLookup? { + let key: String = OpenGroupServerIdLookup.id(serverId: serverId, in: room, on: server) + return transaction.object(forKey: key, inCollection: Storage.openGroupServerIdToUniqueIdLookupCollection) as? OpenGroupServerIdLookup + } + public func addOpenGroupServerIdLookup(_ serverId: UInt64?, tsMessageId: String?, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction) { + guard let serverId: UInt64 = serverId, let tsMessageId: String = tsMessageId else { return } + + let lookup: OpenGroupServerIdLookup = OpenGroupServerIdLookup(server: server, room: room, serverId: serverId, tsMessageId: tsMessageId) + addOpenGroupServerIdLookup(lookup, using: transaction) + } + + public func addOpenGroupServerIdLookup(_ lookup: OpenGroupServerIdLookup, using transaction: YapDatabaseReadWriteTransaction) { + transaction.setObject(lookup, forKey: lookup.id, inCollection: Storage.openGroupServerIdToUniqueIdLookupCollection) + } + public func removeOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction) { + let key: String = OpenGroupServerIdLookup.id(serverId: serverId, in: room, on: server) + transaction.removeObject(forKey: key, inCollection: Storage.openGroupServerIdToUniqueIdLookupCollection) + } // MARK: - Metadata diff --git a/SessionMessagingKit/Open Groups/OpenGroupServerIdLookup.swift b/SessionMessagingKit/Open Groups/OpenGroupServerIdLookup.swift new file mode 100644 index 000000000..ee79614af --- /dev/null +++ b/SessionMessagingKit/Open Groups/OpenGroupServerIdLookup.swift @@ -0,0 +1,46 @@ +// Copyright © 2022 Rangeproof Pty Ltd. All rights reserved. + +import Foundation + +@objc(SNOpenGroupServerIdLookup) +public final class OpenGroupServerIdLookup: NSObject, NSCoding { // NSObject/NSCoding conformance is needed for YapDatabase compatibility + @objc public let id: String + @objc public let serverId: UInt64 + @objc public let tsMessageId: String + + // MARK: - Initialization + + @objc public init(server: String, room: String, serverId: UInt64, tsMessageId: String) { + self.id = OpenGroupServerIdLookup.id(serverId: serverId, in: room, on: server) + self.serverId = serverId + self.tsMessageId = tsMessageId + + super.init() + } + + private override init() { preconditionFailure("Use init(blindedId:sessionId:) instead.") } + + // MARK: - Coding + + public required init?(coder: NSCoder) { + guard let id: String = coder.decodeObject(forKey: "id") as! String? else { return nil } + guard let serverId: UInt64 = coder.decodeObject(forKey: "serverId") as! UInt64? else { return nil } + guard let tsMessageId: String = coder.decodeObject(forKey: "tsMessageId") as! String? else { return nil } + + self.id = id + self.serverId = serverId + self.tsMessageId = tsMessageId + } + + public func encode(with coder: NSCoder) { + coder.encode(id, forKey: "id") + coder.encode(serverId, forKey: "serverId") + coder.encode(tsMessageId, forKey: "tsMessageId") + } + + // MARK: - Convenience + + static func id(serverId: UInt64, in room: String, on server: String) -> String { + return "\(server).\(room).\(serverId)" + } +} diff --git a/SessionMessagingKit/Sending & Receiving/MessageReceiver+Handling.swift b/SessionMessagingKit/Sending & Receiving/MessageReceiver+Handling.swift index ab23a33f0..55bef7b88 100644 --- a/SessionMessagingKit/Sending & Receiving/MessageReceiver+Handling.swift +++ b/SessionMessagingKit/Sending & Receiving/MessageReceiver+Handling.swift @@ -385,7 +385,15 @@ extension MessageReceiver { } if let tsMessage = TSMessage.fetch(uniqueId: tsMessageID, transaction: transaction) { // Keep track of the open group server message ID ↔ message ID relationship - if let serverID = message.openGroupServerMessageID { tsMessage.openGroupServerMessageID = serverID } + if let serverID = message.openGroupServerMessageID { + tsMessage.openGroupServerMessageID = serverID + + // Create a lookup between the openGroupServerMessageId and the tsMessage id for easy lookup + if let openGroup: OpenGroupV2 = storage.getV2OpenGroup(for: threadID) { + storage.addOpenGroupServerIdLookup(serverID, tsMessageId: tsMessageID, in: openGroup.room, on: openGroup.server, using: transaction) + } + } + // Keep track of server hash if let serverHash = message.serverHash { tsMessage.serverHash = serverHash } tsMessage.save(with: transaction) diff --git a/SessionMessagingKit/Sending & Receiving/MessageSender.swift b/SessionMessagingKit/Sending & Receiving/MessageSender.swift index dd44c8197..c02ef9088 100644 --- a/SessionMessagingKit/Sending & Receiving/MessageSender.swift +++ b/SessionMessagingKit/Sending & Receiving/MessageSender.swift @@ -313,6 +313,7 @@ public final class MessageSender : NSObject { base64EncodedData: plaintext.base64EncodedString(), base64EncodedSignature: nil) OpenGroupAPIV2.send(openGroupMessage, to: room, on: server).done(on: DispatchQueue.global(qos: .userInitiated)) { openGroupMessage in message.openGroupServerMessageID = given(openGroupMessage.serverID) { UInt64($0) } + storage.write(with: { transaction in MessageSender.handleSuccessfulMessageSend(message, to: destination, serverTimestamp: openGroupMessage.sentTimestamp, using: transaction) seal.fulfill(()) @@ -341,6 +342,20 @@ public final class MessageSender : NSObject { // Otherwise the quote messages may not be able // to be found by the timestamp on other devices tsMessage.updateOpenGroupServerID(openGroupServerMessageID, serverTimeStamp: timestamp) + + // Create a lookup between the openGroupServerMessageId and the tsMessage id for easy lookup + switch destination { + case .openGroupV2(let room, let server): + Storage.shared.addOpenGroupServerIdLookup( + openGroupServerMessageID, + tsMessageId: tsMessage.uniqueId, + in: room, + on: server, + using: transaction + ) + + default: break + } } // Mark the message as sent var recipients = [ message.recipient! ] diff --git a/SessionMessagingKit/Sending & Receiving/Pollers/OpenGroupPollerV2.swift b/SessionMessagingKit/Sending & Receiving/Pollers/OpenGroupPollerV2.swift index a1b3a47b3..930361510 100644 --- a/SessionMessagingKit/Sending & Receiving/Pollers/OpenGroupPollerV2.swift +++ b/SessionMessagingKit/Sending & Receiving/Pollers/OpenGroupPollerV2.swift @@ -82,6 +82,7 @@ public final class OpenGroupPollerV2 : NSObject { } } } + // - Moderators if var x = OpenGroupAPIV2.moderators[server] { x[body.room] = Set(body.moderators) @@ -89,18 +90,23 @@ public final class OpenGroupPollerV2 : NSObject { } else { OpenGroupAPIV2.moderators[server] = [ body.room : Set(body.moderators) ] } + // - Deletions + guard !body.deletions.isEmpty else { return } + let deletedMessageServerIDs = Set(body.deletions.map { UInt64($0.deletedMessageID) }) storage.write { transaction in - let transaction = transaction as! YapDatabaseReadWriteTransaction - guard let threadID = storage.v2GetThreadID(for: openGroupID), - let thread = TSGroupThread.fetch(uniqueId: threadID, transaction: transaction) else { return } - var messagesToRemove: [TSMessage] = [] - thread.enumerateInteractions(with: transaction) { interaction, stop in - guard let message = interaction as? TSMessage, deletedMessageServerIDs.contains(message.openGroupServerMessageID) else { return } - messagesToRemove.append(message) + guard let transaction: YapDatabaseReadWriteTransaction = transaction as? YapDatabaseReadWriteTransaction else { return } + + deletedMessageServerIDs.forEach { openGroupServerMessageId in + guard let messageLookup: OpenGroupServerIdLookup = storage.getOpenGroupServerIdLookup(openGroupServerMessageId, in: body.room, on: self.server, using: transaction) else { + return + } + guard let tsMessage: TSMessage = TSMessage.fetch(uniqueId: messageLookup.tsMessageId, transaction: transaction) else { return } + + tsMessage.remove(with: transaction) + storage.removeOpenGroupServerIdLookup(openGroupServerMessageId, in: body.room, on: self.server, using: transaction) } - messagesToRemove.forEach { $0.remove(with: transaction) } } } } diff --git a/SessionMessagingKit/Storage.swift b/SessionMessagingKit/Storage.swift index 127cce708..3ae3a66d4 100644 --- a/SessionMessagingKit/Storage.swift +++ b/SessionMessagingKit/Storage.swift @@ -67,6 +67,13 @@ public protocol SessionMessagingKitStorageProtocol { func getLastDeletionServerID(for room: String, on server: String) -> Int64? func setLastDeletionServerID(for room: String, on server: String, to newValue: Int64, using transaction: Any) func removeLastDeletionServerID(for room: String, on server: String, using transaction: Any) + + // MARK: - OpenGroupServerIdToUniqueIdLookup + + func getOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadTransaction) -> OpenGroupServerIdLookup? + func addOpenGroupServerIdLookup(_ serverId: UInt64?, tsMessageId: String?, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction) + func addOpenGroupServerIdLookup(_ lookup: OpenGroupServerIdLookup, using transaction: YapDatabaseReadWriteTransaction) + func removeOpenGroupServerIdLookup(_ serverId: UInt64, in room: String, on server: String, using transaction: YapDatabaseReadWriteTransaction) // MARK: - Open Group Metadata diff --git a/SignalUtilitiesKit/Database/Migrations/OWSDatabaseMigrationRunner.m b/SignalUtilitiesKit/Database/Migrations/OWSDatabaseMigrationRunner.m index a7d38515a..91b370f4e 100644 --- a/SignalUtilitiesKit/Database/Migrations/OWSDatabaseMigrationRunner.m +++ b/SignalUtilitiesKit/Database/Migrations/OWSDatabaseMigrationRunner.m @@ -26,6 +26,7 @@ NS_ASSUME_NONNULL_BEGIN - (NSArray *)allMigrations { return @[ + [SNOpenGroupServerIdLookupMigration new], [SNMessageRequestsMigration new], [SNContactsMigration new] ]; diff --git a/SignalUtilitiesKit/Database/Migrations/OpenGroupServerIdLookupMigration.swift b/SignalUtilitiesKit/Database/Migrations/OpenGroupServerIdLookupMigration.swift new file mode 100644 index 000000000..a98a0da07 --- /dev/null +++ b/SignalUtilitiesKit/Database/Migrations/OpenGroupServerIdLookupMigration.swift @@ -0,0 +1,48 @@ +// Copyright © 2022 Rangeproof Pty Ltd. All rights reserved. + +import Foundation + +@objc(SNOpenGroupServerIdLookupMigration) +public class OpenGroupServerIdLookupMigration: OWSDatabaseMigration { + @objc + class func migrationId() -> String { + return "003" + } + + override public func runUp(completion: @escaping OWSDatabaseMigrationCompletion) { + self.doMigrationAsync(completion: completion) + } + + private func doMigrationAsync(completion: @escaping OWSDatabaseMigrationCompletion) { + var lookups: [OpenGroupServerIdLookup] = [] + + Storage.write(with: { transaction in + TSGroupThread.enumerateCollectionObjects(with: transaction) { object, _ in + guard let thread: TSGroupThread = object as? TSGroupThread else { return } + guard let threadId: String = thread.uniqueId else { return } + guard let openGroup: OpenGroupV2 = Storage.shared.getV2OpenGroup(for: threadId) else { return } + + thread.enumerateInteractions(with: transaction) { interaction, _ in + guard let tsMessage: TSMessage = interaction as? TSMessage else { return } + guard let tsMessageId: String = tsMessage.uniqueId else { return } + + lookups.append( + OpenGroupServerIdLookup( + server: openGroup.server, + room: openGroup.room, + serverId: tsMessage.openGroupServerMessageID, + tsMessageId: tsMessageId + ) + ) + } + } + + lookups.forEach { lookup in + Storage.shared.addOpenGroupServerIdLookup(lookup, using: transaction) + } + self.save(with: transaction) // Intentionally capture self + }, completion: { + completion() + }) + } +}