Attempted to fix the notification & call reporting issues

Fixed an issue where fileIds weren't correctly getting sent along with open group messages
Fixed an issue where the screens could miss updates if the device was locked with the app in the foreground and then later unlocked after receiving notifications
Added an optimisation to prevent attempting to send a message after it has been deleted
Added logic to report fake calls if the code goes down an invalid code path when handling a call (to prevent Apple blocking the app)
Delayed the core which clears notifications to increase the time the app has to handle interactions (just in case it was a race condition)

Session/Calls/Call Management/SessionCall.swift

@@ -167,7 +167,10 @@ public final class SessionCall: CurrentCallProtocol, WebRTCSessionDelegate {
     }
     
     func reportIncomingCallIfNeeded(completion: @escaping (Error?) -> Void) {
-        guard case .answer = mode else { return }
+        guard case .answer = mode else {
+            SessionCallManager.reportFakeCall(info: "Call not in answer mode")
+            return
+        }
         
         setupTimeoutTimer()
         AppEnvironment.shared.callManager.reportIncomingCall(self, callerName: contactName) { error in

Session/Calls/Call Management/SessionCallManager.swift

@@ -72,6 +72,16 @@ public final class SessionCallManager: NSObject, CallManagerProtocol {
     
     // MARK: - Report calls
     
+    public static func reportFakeCall(info: String) {
+        SessionCallManager.sharedProvider(useSystemCallLog: false)
+            .reportNewIncomingCall(
+                with: UUID(),
+                update: CXCallUpdate()
+            ) { _ in
+                SNLog("[Calls] Reported fake incoming call to CallKit due to: \(info)")
+            }
+    }
+    
     public func reportOutgoingCall(_ call: SessionCall) {
         AssertIsOnMainThread()
         UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
@@ -109,7 +119,9 @@ public final class SessionCallManager: NSObject, CallManagerProtocol {
                 UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
                 completion(nil)
             }
-        } else {
+        }
+        else {
+            SessionCallManager.reportFakeCall(info: "No CXProvider instance")
             UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
             completion(nil)
         }

Session/Conversations/ConversationVC.swift

@@ -450,7 +450,7 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
         recoverInputView()
     }
     
@@ -460,7 +460,7 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes
         dataChangeObservable = Storage.shared.start(
             viewModel.observableThreadData,
@@ -506,6 +506,13 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
                     self?.viewModel.onInteractionChange = { [weak self] updatedInteractionData in
                         self?.handleInteractionUpdates(updatedInteractionData)
                     }
+                    
+                    // Note: When returning from the background we could have received notifications but the
+                    // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+                    // data to ensure everything is up to date
+                    if didReturnFromBackground {
+                        self?.viewModel.pagedDataObserver?.reload()
+                    }
                 }
             }
         )

Session/Home/HomeVC.swift

@@ -239,7 +239,7 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -248,7 +248,7 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes
         dataChangeObservable = Storage.shared.start(
             viewModel.observableState,
@@ -269,6 +269,13 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
         self.viewModel.onThreadChange = { [weak self] updatedThreadData in
             self?.handleThreadUpdates(updatedThreadData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func stopObservingChanges() {

Session/Home/Message Requests/MessageRequestsViewController.swift

@@ -147,7 +147,7 @@ class MessageRequestsViewController: BaseVC, UITableViewDelegate, UITableViewDat
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -186,10 +186,17 @@ class MessageRequestsViewController: BaseVC, UITableViewDelegate, UITableViewDat
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         self.viewModel.onThreadChange = { [weak self] updatedThreadData in
             self?.handleThreadUpdates(updatedThreadData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func handleThreadUpdates(_ updatedData: [MessageRequestsViewModel.SectionModel], initialLoad: Bool = false) {

Session/Media Viewing & Editing/MediaTileViewController.swift

@@ -171,7 +171,7 @@ public class MediaTileViewController: UIViewController, UICollectionViewDataSour
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -243,11 +243,18 @@ public class MediaTileViewController: UIViewController, UICollectionViewDataSour
         }
     }
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes (will callback on the main thread)
         self.viewModel.onGalleryChange = { [weak self] updatedGalleryData in
             self?.handleUpdates(updatedGalleryData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func stopObservingChanges() {

Session/Meta/AppDelegate.swift

@@ -149,13 +149,13 @@ class AppDelegate: UIResponder, UIApplicationDelegate, UNUserNotificationCenterD
             /// no longer always called before `applicationDidBecomeActive` we need to trigger the "clear notifications" logic
             /// within the `runNowOrWhenAppDidBecomeReady` callback and dispatch to the next run loop to ensure it runs after
             /// the notification has actually been handled
-            DispatchQueue.main.async { [weak self] in
+            DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(100)) { [weak self] in
                 self?.clearAllNotificationsAndRestoreBadgeCount()
             }
         }
 
         // On every activation, clear old temp directories.
-        ClearOldTemporaryDirectories();
+        ClearOldTemporaryDirectories()
     }
     
     func applicationWillResignActive(_ application: UIApplication) {

Session/Notifications/PushRegistrationManager.swift

@@ -242,40 +242,52 @@ public enum PushRegistrationError: Error {
         owsAssertDebug(type == .voIP)
         let payload = payload.dictionaryPayload
         
-        if let uuid = payload["uuid"] as? String, let caller = payload["caller"] as? String, let timestampMs = payload["timestamp"] as? Int64 {
+        guard
-            let call: SessionCall? = Storage.shared.write { db in
+            let uuid: String = payload["uuid"] as? String,
-                let messageInfo: CallMessage.MessageInfo = CallMessage.MessageInfo(
+            let caller: String = payload["caller"] as? String,
-                    state: (caller == getUserHexEncodedPublicKey(db) ?
+            let timestampMs: Int64 = payload["timestamp"] as? Int64
-                        .outgoing :
+        else {
-                        .incoming
+            SessionCallManager.reportFakeCall(info: "Missing payload data")
-                    )
+            return
+        }
+        
+        let maybeCall: SessionCall? = Storage.shared.write { db in
+            let messageInfo: CallMessage.MessageInfo = CallMessage.MessageInfo(
+                state: (caller == getUserHexEncodedPublicKey(db) ?
+                    .outgoing :
+                    .incoming
                 )
-                
+            )
-                guard let messageInfoData: Data = try? JSONEncoder().encode(messageInfo) else { return nil }
-                
-                let call: SessionCall = SessionCall(db, for: caller, uuid: uuid, mode: .answer)
-                let thread: SessionThread = try SessionThread.fetchOrCreate(db, id: caller, variant: .contact)
-                
-                let interaction: Interaction = try Interaction(
-                    messageUuid: uuid,
-                    threadId: thread.id,
-                    authorId: caller,
-                    variant: .infoCall,
-                    body: String(data: messageInfoData, encoding: .utf8),
-                    timestampMs: timestampMs
-                ).inserted(db)
-                call.callInteractionId = interaction.id
-                
-                return call
-            }
             
-            // NOTE: Just start 1-1 poller so that it won't wait for polling group messages
+            guard let messageInfoData: Data = try? JSONEncoder().encode(messageInfo) else { return nil }
-            (UIApplication.shared.delegate as? AppDelegate)?.startPollersIfNeeded(shouldStartGroupPollers: false)
             
-            call?.reportIncomingCallIfNeeded { error in
+            let call: SessionCall = SessionCall(db, for: caller, uuid: uuid, mode: .answer)
-                if let error = error {
+            let thread: SessionThread = try SessionThread.fetchOrCreate(db, id: caller, variant: .contact)
-                    SNLog("[Calls] Failed to report incoming call to CallKit due to error: \(error)")
+            
-                }
+            let interaction: Interaction = try Interaction(
+                messageUuid: uuid,
+                threadId: thread.id,
+                authorId: caller,
+                variant: .infoCall,
+                body: String(data: messageInfoData, encoding: .utf8),
+                timestampMs: timestampMs
+            ).inserted(db)
+            call.callInteractionId = interaction.id
+            
+            return call
+        }
+        
+        guard let call: SessionCall = maybeCall else {
+            SessionCallManager.reportFakeCall(info: "Could not retrieve call from database")
+            return
+        }
+        
+        // NOTE: Just start 1-1 poller so that it won't wait for polling group messages
+        (UIApplication.shared.delegate as? AppDelegate)?.startPollersIfNeeded(shouldStartGroupPollers: false)
+        
+        call.reportIncomingCallIfNeeded { error in
+            if let error = error {
+                SNLog("[Calls] Failed to report incoming call to CallKit due to error: \(error)")
             }
         }
     }

SessionMessagingKit/Database/Models/Attachment.swift

@@ -468,6 +468,7 @@ extension Attachment {
         public let attachmentId: String
         public let interactionId: Int64
         public let state: Attachment.State
+        public let downloadUrl: String?
     }
     
     public static func stateInfo(authorId: String, state: State? = nil) -> SQLRequest<Attachment.StateInfo> {
@@ -484,7 +485,8 @@ extension Attachment {
             SELECT DISTINCT
                 \(attachment[.id]) AS attachmentId,
                 \(interaction[.id]) AS interactionId,
-                \(attachment[.state]) AS state
+                \(attachment[.state]) AS state,
+                \(attachment[.downloadUrl]) AS downloadUrl
         
             FROM \(Attachment.self)
             
@@ -529,7 +531,8 @@ extension Attachment {
             SELECT DISTINCT
                 \(attachment[.id]) AS attachmentId,
                 \(interaction[.id]) AS interactionId,
-                \(attachment[.state]) AS state
+                \(attachment[.state]) AS state,
+                \(attachment[.downloadUrl]) AS downloadUrl
         
             FROM \(Attachment.self)
             
@@ -913,6 +916,16 @@ extension Attachment {
 
         return true
     }
+    
+    public static func fileId(for downloadUrl: String?) -> String? {
+        return downloadUrl
+            .map { urlString -> String? in
+                urlString
+                    .split(separator: "/")
+                    .last
+                    .map { String($0) }
+            }
+    }
 }
 
 // MARK: - Upload
@@ -923,14 +936,14 @@ extension Attachment {
         queue: DispatchQueue,
         using upload: (Database, Data) -> Promise<String>,
         encrypt: Bool,
-        success: (() -> Void)?,
+        success: ((String?) -> Void)?,
         failure: ((Error) -> Void)?
     ) {
         // This can occur if an AttachmnetUploadJob was explicitly created for a message
         // dependant on the attachment being uploaded (in this case the attachment has
         // already been uploaded so just succeed)
         guard state != .uploaded else {
-            success?()
+            success?(Attachment.fileId(for: self.downloadUrl))
             return
         }
         
@@ -982,7 +995,7 @@ extension Attachment {
                 return
             }
                 
-            success?()
+            success?(Attachment.fileId(for: self.downloadUrl))
             return
         }
         
@@ -1073,7 +1086,7 @@ extension Attachment {
                     return
                 }
                     
-                success?()
+                success?(fileId)
             }
             .catch(on: queue) { error in
                 Storage.shared.write { db in

SessionMessagingKit/Jobs/Types/AttachmentDownloadJob.swift

@@ -87,10 +87,7 @@ public enum AttachmentDownloadJob: JobExecutor {
         let downloadPromise: Promise<Data> = {
             guard
                 let downloadUrl: String = attachment.downloadUrl,
-                let fileId: String = downloadUrl
+                let fileId: String = Attachment.fileId(for: downloadUrl)
-                    .split(separator: "/")
-                    .last
-                    .map({ String($0) })
             else {
                 return Promise(error: AttachmentDownloadError.invalidUrl)
             }

SessionMessagingKit/Jobs/Types/AttachmentUploadJob.swift

@@ -55,7 +55,7 @@ public enum AttachmentUploadJob: JobExecutor {
                     .map { response -> String in response.id }
             },
             encrypt: (openGroup == nil),
-            success: { success(job, false) },
+            success: { _ in success(job, false) },
             failure: { error in failure(job, error, false) }
         )
     }

SessionMessagingKit/Jobs/Types/MessageSendJob.swift

@@ -27,6 +27,10 @@ public enum MessageSendJob: JobExecutor {
             return
         }
         
+        // We need to include 'fileIds' when sending messages with attachments to Open Groups
+        // so extract them from any associated attachments
+        var messageFileIds: [String] = []
+        
         if details.message is VisibleMessage {
             guard
                 let jobId: Int64 = job.id,
@@ -36,20 +40,30 @@ public enum MessageSendJob: JobExecutor {
                 return
             }
             
+            // If the original interaction no longer exists then don't bother sending the message (ie. the
+            // message was deleted before it even got sent)
+            guard Storage.shared.read({ db in try Interaction.exists(db, id: interactionId) }) == true else {
+                failure(job, StorageError.objectNotFound, true)
+                return
+            }
+            
             // Check if there are any attachments associated to this message, and if so
             // upload them now
             //
             // Note: Normal attachments should be sent in a non-durable way but any
             // attachments for LinkPreviews and Quotes will be processed through this mechanism
-            let attachmentState: (shouldFail: Bool, shouldDefer: Bool)? = Storage.shared.write { db in
+            let attachmentState: (shouldFail: Bool, shouldDefer: Bool, fileIds: [String])? = Storage.shared.write { db in
                 let allAttachmentStateInfo: [Attachment.StateInfo] = try Attachment
                     .stateInfo(interactionId: interactionId)
                     .fetchAll(db)
+                let maybeFileIds: [String?] = allAttachmentStateInfo
+                    .map { Attachment.fileId(for: $0.downloadUrl) }
+                let fileIds: [String] = maybeFileIds.compactMap { $0 }
                 
                 // If there were failed attachments then this job should fail (can't send a
                 // message which has associated attachments if the attachments fail to upload)
                 guard !allAttachmentStateInfo.contains(where: { $0.state == .failedDownload }) else {
-                    return (true, false)
+                    return (true, false, fileIds)
                 }
                 
                 // Create jobs for any pending (or failed) attachment jobs and insert them into the
@@ -102,9 +116,13 @@ public enum MessageSendJob: JobExecutor {
                 // If there were pending or uploading attachments then stop here (we want to
                 // upload them first and then re-run this send job - the 'JobRunner.insert'
                 // method will take care of this)
+                let isMissingFileIds: Bool = (maybeFileIds.count != fileIds.count)
+                let hasPendingUploads: Bool = allAttachmentStateInfo.contains(where: { $0.state != .uploaded })
+                
                 return (
-                    false,
+                    (isMissingFileIds && !hasPendingUploads),
-                    allAttachmentStateInfo.contains(where: { $0.state != .uploaded })
+                    hasPendingUploads,
+                    fileIds
                 )
             }
             
@@ -122,6 +140,9 @@ public enum MessageSendJob: JobExecutor {
                 deferred(job)
                 return
             }
+            
+            // Store the fileIds so they can be sent with the open group message content
+            messageFileIds = (attachmentState?.fileIds ?? [])
         }
         
         // Store the sentTimestamp from the message in case it fails due to a clockOutOfSync error
@@ -135,7 +156,8 @@ public enum MessageSendJob: JobExecutor {
             try MessageSender.sendImmediate(
                 db,
                 message: details.message,
-                to: details.destination,
+                to: details.destination
+                    .with(fileIds: messageFileIds),
                 interactionId: job.interactionId
             )
         }

SessionMessagingKit/Messages/Message+Destination.swift

@@ -49,5 +49,21 @@ public extension Message {
                     return .openGroup(roomToken: openGroup.roomToken, server: openGroup.server, fileIds: fileIds)
             }
         }
+        
+        func with(fileIds: [String]) -> Message.Destination {
+            // Only Open Group messages support receiving the 'fileIds'
+            switch self {
+                case .openGroup(let roomToken, let server, let whisperTo, let whisperMods, _):
+                    return .openGroup(
+                        roomToken: roomToken,
+                        server: server,
+                        whisperTo: whisperTo,
+                        whisperMods: whisperMods,
+                        fileIds: fileIds
+                    )
+                    
+                default: return self
+            }
+        }
     }
 }

SessionMessagingKit/Sending & Receiving/MessageSender+Convenience.swift

@@ -100,7 +100,7 @@ extension MessageSender {
     }
     
     public static func sendNonDurably(_ db: Database, message: Message, interactionId: Int64?, to destination: Message.Destination) -> Promise<Void> {
-        var attachmentUploadPromises: [Promise<Void>] = [Promise.value(())]
+        var attachmentUploadPromises: [Promise<String?>] = [Promise.value(nil)]
         
         // If we have an interactionId then check if it has any attachments and process them first
         if let interactionId: Int64 = interactionId {
@@ -124,8 +124,8 @@ extension MessageSender {
                 .filter(ids: attachmentStateInfo.map { $0.attachmentId })
                 .fetchAll(db))
                 .defaulting(to: [])
-                .map { attachment -> Promise<Void> in
+                .map { attachment -> Promise<String?> in
-                    let (promise, seal) = Promise<Void>.pending()
+                    let (promise, seal) = Promise<String?>.pending()
     
                     attachment.upload(
                         db,
@@ -146,7 +146,7 @@ extension MessageSender {
                                 .map { response -> String in response.id }
                         },
                         encrypt: (openGroup == nil),
-                        success: { seal.fulfill(()) },
+                        success: { fileId in seal.fulfill(fileId) },
                         failure: { seal.reject($0) }
                     )
     
@@ -167,10 +167,18 @@ extension MessageSender {
                 if let error: Error = errors.first { return Promise(error: error) }
                 
                 return Storage.shared.writeAsync { db in
-                    try MessageSender.sendImmediate(
+                    let fileIds: [String] = results
+                        .compactMap { result -> String? in
+                            if case .fulfilled(let value) = result { return value }
+                            
+                            return nil
+                        }
+                    
+                    return try MessageSender.sendImmediate(
                         db,
                         message: message,
-                        to: destination,
+                        to: destination
+                            .with(fileIds: fileIds),
                         interactionId: interactionId
                     )
                 }

SessionMessagingKit/Utilities/ProfileManager.swift

@@ -150,10 +150,7 @@ public struct ProfileManager {
             return
         }
         guard
-            let fileId: String = profileUrlStringAtStart
+            let fileId: String = Attachment.fileId(for: profileUrlStringAtStart),
-                .split(separator: "/")
-                .last
-                .map({ String($0) }),
             let profileKeyAtStart: OWSAES256Key = profile.profileEncryptionKey,
             profileKeyAtStart.keyData.count > 0
         else {

SessionUtilitiesKit/Database/Types/PagedDatabaseObserver.swift

@@ -477,6 +477,13 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
                             cacheCurrentEndIndex,
                             currentPageInfo.pageOffset
                         )
+                        
+                    case .reloadCurrent:
+                        return (
+                            currentPageInfo.currentCount,
+                            currentPageInfo.pageOffset,
+                            currentPageInfo.pageOffset
+                        )
                 }
             }()
             
@@ -570,6 +577,10 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
         
         triggerUpdates()
     }
+    
+    public func reload() {
+        self.load(.reloadCurrent)
+    }
 }
 
 // MARK: - Convenience
@@ -718,6 +729,7 @@ public enum PagedData {
             case pageBefore
             case pageAfter
             case untilInclusive(id: SQLExpression, padding: Int)
+            case reloadCurrent
         }
         
         public enum Target<ID: SQLExpressible> {