From afe1efbd907b2130de0c4324b5abd8c1e9547a83 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Fri, 12 Apr 2024 18:44:32 +1000 Subject: [PATCH] Deduped path building and attempted to improve extension logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit โ€ข Moved the build paths logic into the BuildPathsJob to allow for better deduping โ€ข Updated the notification and share extensions to generate log files and append to the bottom of the app log file --- LibSession-Util | 2 +- Session.xcodeproj/project.pbxproj | 4 +- Session/Meta/AppEnvironment.swift | 45 +++ .../NotificationServiceExtension.swift | 40 ++- .../ShareNavController.swift | 15 +- SessionShareExtension/ThreadPickerVC.swift | 2 + SessionSnodeKit/Configuration.swift | 1 + SessionSnodeKit/Jobs/BuildPathsJob.swift | 264 +++++++++++++++++ SessionSnodeKit/Jobs/GetSnodePoolJob.swift | 2 +- .../Networking/OnionRequestAPI.swift | 274 +----------------- SessionUtilitiesKit/General/Logging.swift | 6 +- SessionUtilitiesKit/JobRunner/JobRunner.swift | 3 +- 12 files changed, 372 insertions(+), 286 deletions(-) diff --git a/LibSession-Util b/LibSession-Util index ec0332bcf..1c4667ba0 160000 --- a/LibSession-Util +++ b/LibSession-Util @@ -1 +1 @@ -Subproject commit ec0332bcf8bd8181698a235779ab0d021a55d380 +Subproject commit 1c4667ba0c56c924d4e957743d1324be2c899040 diff --git a/Session.xcodeproj/project.pbxproj b/Session.xcodeproj/project.pbxproj index 0069019f6..b85a2198e 100644 --- a/Session.xcodeproj/project.pbxproj +++ b/Session.xcodeproj/project.pbxproj @@ -8055,7 +8055,7 @@ CLANG_WARN__ARC_BRIDGE_CAST_NONARC = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_IDENTITY = "iPhone Developer"; - CURRENT_PROJECT_VERSION = 437; + CURRENT_PROJECT_VERSION = 438; ENABLE_BITCODE = NO; ENABLE_STRICT_OBJC_MSGSEND = YES; ENABLE_TESTABILITY = YES; @@ -8133,7 +8133,7 @@ CLANG_WARN__ARC_BRIDGE_CAST_NONARC = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_IDENTITY = "iPhone Distribution"; - CURRENT_PROJECT_VERSION = 437; + CURRENT_PROJECT_VERSION = 438; ENABLE_BITCODE = NO; ENABLE_STRICT_OBJC_MSGSEND = YES; GCC_NO_COMMON_BLOCKS = YES; diff --git a/Session/Meta/AppEnvironment.swift b/Session/Meta/AppEnvironment.swift index 50ec1d2bd..d1c36de19 100644 --- a/Session/Meta/AppEnvironment.swift +++ b/Session/Meta/AppEnvironment.swift @@ -62,5 +62,50 @@ public class AppEnvironment { fileLogger.rollingFrequency = kDayInterval // Refresh everyday fileLogger.logFileManager.maximumNumberOfLogFiles = 3 // Save 3 days' log files DDLog.add(fileLogger) + + // The extensions write their logs to the app shared directory but the main app writes + // to a local directory (so they can be exported via XCode) - the below code reads any + // logs from the shared directly and attempts to add them to the main app logs to make + // debugging user issues in extensions easier + DispatchQueue.global(qos: .background).async { + let extensionDirs: [String] = [ + "\(OWSFileSystem.appSharedDataDirectoryPath())/Logs/NotificationExtension", + "\(OWSFileSystem.appSharedDataDirectoryPath())/Logs/ShareExtension" + ] + let extensionLogs: [String] = extensionDirs.flatMap { dir -> [String] in + guard let files: [String] = try? FileManager.default.contentsOfDirectory(atPath: dir) else { return [] } + + return files.map { "\(dir)/\($0)" } + } + + extensionLogs.forEach { logFilePath in + guard let logs: String = try? String(contentsOfFile: logFilePath) else { + try? FileManager.default.removeItem(atPath: logFilePath) + return + } + + logs.split(separator: "\n").forEach { line in + let lineEmoji: Character? = line + .split(separator: "[") + .first + .map { String($0) }? + .trimmingCharacters(in: .whitespaces) + .last + + switch lineEmoji { + case "๐Ÿ’™": OWSLogger.verbose("Extension: \(String(line))") + case "๐Ÿ’š": OWSLogger.debug("Extension: \(String(line))") + case "๐Ÿ’›": OWSLogger.info("Extension: \(String(line))") + case "๐Ÿงก": OWSLogger.warn("Extension: \(String(line))") + case "โค๏ธ": OWSLogger.error("Extension: \(String(line))") + default: OWSLogger.info("Extension: \(String(line))") + } + } + + // Logs have been added - remove them now + DDLog.flushLog() + try? FileManager.default.removeItem(atPath: logFilePath) + } + } } } diff --git a/SessionNotificationServiceExtension/NotificationServiceExtension.swift b/SessionNotificationServiceExtension/NotificationServiceExtension.swift index c9c3a2a72..a0109053e 100644 --- a/SessionNotificationServiceExtension/NotificationServiceExtension.swift +++ b/SessionNotificationServiceExtension/NotificationServiceExtension.swift @@ -16,6 +16,7 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension private var contentHandler: ((UNNotificationContent) -> Void)? private var request: UNNotificationRequest? private var openGroupPollCancellable: AnyCancellable? + private var fileLogger: DDFileLogger? public static let isFromRemoteKey = "remote" public static let threadIdKey = "Signal.AppNotificationsUserInfoKey.threadId" @@ -88,7 +89,7 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension (lastCallPreOffer ?? Date.distantPast).timeIntervalSinceNow < NotificationServiceExtension.callPreOfferLargeNotificationSupressionDuration else { return self.handleFailure(for: notificationContent, error: .processing(result)) } - NSLog("[NotificationServiceExtension] Suppressing large notification too close to a call.") + SNLog("[NotificationServiceExtension] Suppressing large notification too close to a call.", forceNSLog: true) return case .legacyForceSilent, .failureNoContent: return @@ -218,7 +219,7 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension // to process new messages. guard !didPerformSetup else { return } - NSLog("[NotificationServiceExtension] Performing setup") + SNLog("[NotificationServiceExtension] Performing setup", forceNSLog: true) didPerformSetup = true _ = AppVersion.sharedInstance() @@ -227,16 +228,26 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension AppSetup.setupEnvironment( retrySetupIfDatabaseInvalid: true, - appSpecificBlock: { + appSpecificBlock: { [weak self] in SessionEnvironment.shared?.notificationsManager.mutate { $0 = NSENotificationPresenter() } + + // Add the file logger + let logFileManager: DDLogFileManagerDefault = DDLogFileManagerDefault( + logsDirectory: "\(OWSFileSystem.appSharedDataDirectoryPath())/Logs/NotificationExtension" // stringlint:disable + ) + let fileLogger: DDFileLogger = DDFileLogger(logFileManager: logFileManager) + fileLogger.rollingFrequency = kDayInterval // Refresh everyday + fileLogger.logFileManager.maximumNumberOfLogFiles = 3 // Save 3 days' log files + DDLog.add(fileLogger) + self?.fileLogger = fileLogger }, migrationsCompletion: { [weak self] result, needsConfigSync in switch result { // Only 'NSLog' works in the extension - viewable via Console.app case .failure(let error): - NSLog("[NotificationServiceExtension] Failed to complete migrations: \(error)") + SNLog("[NotificationServiceExtension] Failed to complete migrations: \(error)", forceNSLog: true) self?.completeSilenty() case .success: @@ -246,7 +257,7 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension // so it is possible that could change in the future. If it does, do nothing // and don't disturb the user. Messages will be processed when they open the app. guard Storage.shared[.isReadyForAppExtensions] else { - NSLog("[NotificationServiceExtension] Not ready for extensions") + SNLog("[NotificationServiceExtension] Not ready for extensions", forceNSLog: true) self?.completeSilenty() return } @@ -282,7 +293,7 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension // App isn't ready until storage is ready AND all version migrations are complete. guard Storage.shared.isValid && migrationsCompleted else { - NSLog("[NotificationServiceExtension] Storage invalid") + SNLog("[NotificationServiceExtension] Storage invalid", forceNSLog: true) self.completeSilenty() return } @@ -298,13 +309,14 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension override public func serviceExtensionTimeWillExpire() { // Called just before the extension will be terminated by the system. // Use this as an opportunity to deliver your "best attempt" at modified content, otherwise the original push payload will be used. - NSLog("[NotificationServiceExtension] Execution time expired") + SNLog("[NotificationServiceExtension] Execution time expired", forceNSLog: true) openGroupPollCancellable?.cancel() completeSilenty() } private func completeSilenty() { - NSLog("[NotificationServiceExtension] Complete silently") + SNLog("[NotificationServiceExtension] Complete silently", forceNSLog: true) + DDLog.flushLog() let silentContent: UNMutableNotificationContent = UNMutableNotificationContent() silentContent.badge = Storage.shared .read { db in try Interaction.fetchUnreadCount(db) } @@ -328,10 +340,10 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension CXProvider.reportNewIncomingVoIPPushPayload(payload) { error in if let error = error { self.handleFailureForVoIP(db, for: callMessage) - NSLog("[NotificationServiceExtension] Failed to notify main app of call message: \(error)") + SNLog("[NotificationServiceExtension] Failed to notify main app of call message: \(error)", forceNSLog: true) } else { - NSLog("[NotificationServiceExtension] Successfully notified main app of call message.") + SNLog("[NotificationServiceExtension] Successfully notified main app of call message.", forceNSLog: true) UserDefaults.sharedLokiProject?[.lastCallPreOffer] = Date() self.completeSilenty() } @@ -364,16 +376,18 @@ public final class NotificationServiceExtension: UNNotificationServiceExtension UNUserNotificationCenter.current().add(request) { error in if let error = error { - NSLog("[NotificationServiceExtension] Failed to add notification request due to error: \(error)") + SNLog("[NotificationServiceExtension] Failed to add notification request due to error: \(error)", forceNSLog: true) } semaphore.signal() } semaphore.wait() - NSLog("[NotificationServiceExtension] Add remote notification request") + SNLog("[NotificationServiceExtension] Add remote notification request", forceNSLog: true) + DDLog.flushLog() } private func handleFailure(for content: UNMutableNotificationContent, error: NotificationError) { - NSLog("[NotificationServiceExtension] Show generic failure message due to error: \(error)") + SNLog("[NotificationServiceExtension] Show generic failure message due to error: \(error)", forceNSLog: true) + DDLog.flushLog() Storage.suspendDatabaseAccess() content.title = "Session" diff --git a/SessionShareExtension/ShareNavController.swift b/SessionShareExtension/ShareNavController.swift index b3fd92232..063d0467f 100644 --- a/SessionShareExtension/ShareNavController.swift +++ b/SessionShareExtension/ShareNavController.swift @@ -12,6 +12,7 @@ import SignalCoreKit final class ShareNavController: UINavigationController, ShareViewDelegate { public static var attachmentPrepPublisher: AnyPublisher<[SignalAttachment], Error>? private let versionMigrationsComplete: Atomic = Atomic(false) + private var fileLogger: DDFileLogger? // MARK: - Error @@ -52,10 +53,20 @@ final class ShareNavController: UINavigationController, ShareViewDelegate { } AppSetup.setupEnvironment( - appSpecificBlock: { + appSpecificBlock: { [weak self] in SessionEnvironment.shared?.notificationsManager.mutate { $0 = NoopNotificationsManager() } + + // Add the file logger + let logFileManager: DDLogFileManagerDefault = DDLogFileManagerDefault( + logsDirectory: "\(OWSFileSystem.appSharedDataDirectoryPath())/Logs/ShareExtension" // stringlint:disable + ) + let fileLogger: DDFileLogger = DDFileLogger(logFileManager: logFileManager) + fileLogger.rollingFrequency = kDayInterval // Refresh everyday + fileLogger.logFileManager.maximumNumberOfLogFiles = 3 // Save 3 days' log files + DDLog.add(fileLogger) + self?.fileLogger = fileLogger }, migrationsCompletion: { [weak self] result, needsConfigSync in switch result { @@ -163,6 +174,7 @@ final class ShareNavController: UINavigationController, ShareViewDelegate { @objc public func applicationDidEnterBackground() { AssertIsOnMainThread() + DDLog.flushLog() Logger.info("") @@ -176,6 +188,7 @@ final class ShareNavController: UINavigationController, ShareViewDelegate { deinit { NotificationCenter.default.removeObserver(self) + DDLog.flushLog() // Share extensions reside in a process that may be reused between usages. // That isn't safe; the codebase is full of statics (e.g. singletons) which diff --git a/SessionShareExtension/ThreadPickerVC.swift b/SessionShareExtension/ThreadPickerVC.swift index c6367a433..2e4cb1d7c 100644 --- a/SessionShareExtension/ThreadPickerVC.swift +++ b/SessionShareExtension/ThreadPickerVC.swift @@ -8,6 +8,7 @@ import SessionUIKit import SignalUtilitiesKit import SessionMessagingKit import SessionSnodeKit +import SignalCoreKit final class ThreadPickerVC: UIViewController, UITableViewDataSource, UITableViewDelegate, AttachmentApprovalViewControllerDelegate { private let viewModel: ThreadPickerViewModel = ThreadPickerViewModel() @@ -327,6 +328,7 @@ final class ThreadPickerVC: UIViewController, UITableViewDataSource, UITableView .receive(on: DispatchQueue.main) .sinkUntilComplete( receiveCompletion: { [weak self] result in + DDLog.flushLog() Storage.suspendDatabaseAccess() activityIndicator.dismiss { } diff --git a/SessionSnodeKit/Configuration.swift b/SessionSnodeKit/Configuration.swift index dd221b470..93d3f4ab3 100644 --- a/SessionSnodeKit/Configuration.swift +++ b/SessionSnodeKit/Configuration.swift @@ -32,5 +32,6 @@ public enum SNSnodeKit: MigratableTarget { // Just to make the external API nice public static func configure() { // Configure the job executors JobRunner.setExecutor(GetSnodePoolJob.self, for: .getSnodePool) + JobRunner.setExecutor(BuildPathsJob.self, for: .buildPaths) } } diff --git a/SessionSnodeKit/Jobs/BuildPathsJob.swift b/SessionSnodeKit/Jobs/BuildPathsJob.swift index add8e2954..956f1c372 100644 --- a/SessionSnodeKit/Jobs/BuildPathsJob.swift +++ b/SessionSnodeKit/Jobs/BuildPathsJob.swift @@ -1,3 +1,267 @@ // Copyright ยฉ 2024 Rangeproof Pty Ltd. All rights reserved. import Foundation +import Combine +import GRDB +import SessionUtilitiesKit + +public enum BuildPathsJob: JobExecutor { + public static let maxFailureCount: Int = 0 + public static let requiresThreadId: Bool = false + public static let requiresInteractionId: Bool = false + + /// The number of paths to maintain. + public static let targetPathCount: UInt = 2 + + /// The number of guard snodes required to maintain `targetPathCount` paths. + private static var targetGuardSnodeCount: Int { return Int(targetPathCount) } // One per path + + public static func run( + _ job: Job, + queue: DispatchQueue, + success: @escaping (Job, Bool, Dependencies) -> (), + failure: @escaping (Job, Error?, Bool, Dependencies) -> (), + deferred: @escaping (Job, Dependencies) -> (), + using dependencies: Dependencies + ) { + guard + let detailsData: Data = job.details, + let details: Details = try? JSONDecoder().decode(Details.self, from: detailsData), + let ed25519SecretKey: [UInt8] = details.ed25519SecretKey + else { + SNLog("[BuildPathsJob] Failing due to missing details.") + return failure(job, JobRunnerError.missingRequiredDetails, true, dependencies) + } + + SNLog("[BuildPathsJob] Starting.") + DispatchQueue.main.async { + NotificationCenter.default.post(name: .buildingPaths, object: nil) + } + + /// First we need to get the guard snodes + getGuardSnodes( + reusableGuardSnodes: details.reusablePaths.map { $0[0] }, + ed25519SecretKey: ed25519SecretKey, + queue: queue, + using: dependencies + ) + .tryMap { (guardSnodes: Set) -> [[Snode]] in + var unusedSnodes: Set = SnodeAPI.snodePool.wrappedValue + .subtracting(guardSnodes) + .subtracting(details.reusablePaths.flatMap { $0 }) + let pathSnodeCount: Int = (targetGuardSnodeCount - details.reusablePaths.count) * OnionRequestAPI.pathSize - (targetGuardSnodeCount - details.reusablePaths.count) + + guard unusedSnodes.count >= pathSnodeCount else { + throw SnodeAPIError.insufficientSnodes + } + + /// Don't test path snodes as this would reveal the user's IP to them + return guardSnodes + .subtracting(details.reusablePaths.compactMap { $0.first }) + .map { (guardSnode: Snode) -> [Snode] in + let additionalSnodes: [Snode] = (0..<(OnionRequestAPI.pathSize - 1)).map { _ in + /// randomElement() uses the system's default random generator, which is cryptographically secure, the + /// force-unwrap here is safe because of the `pathSnodeCount` check above + unusedSnodes.popRandomElement()! + } + let result: [Snode] = [guardSnode].appending(contentsOf: additionalSnodes) + SNLog("[BuildPathsJob] Built new onion request path: \(result.prettifiedDescription).") + return result + } + } + .subscribe(on: queue, using: dependencies) + .receive(on: queue, using: dependencies) + .sinkUntilComplete( + receiveCompletion: { result in + switch result { + case .finished: break + case .failure(let error): + SNLog("[BuildPathsJob] Failed due to error: \(error)") + failure(job, error, false, dependencies) + } + }, + receiveValue: { (output: [[Snode]]) in + OnionRequestAPI.paths = (output + details.reusablePaths) + + dependencies.storage.write(using: dependencies) { db in + SNLog("[BuildPathsJob] Persisting onion request paths to database.") + try? output.save(db) + } + + DispatchQueue.main.async { + NotificationCenter.default.post(name: .pathsBuilt, object: nil) + } + + SNLog("[BuildPathsJob] Complete.") + success(job, false, dependencies) + } + ) + } + + private static func getGuardSnodes( + reusableGuardSnodes: [Snode], + ed25519SecretKey: [UInt8], + queue: DispatchQueue, + using dependencies: Dependencies + ) -> AnyPublisher, Error> { + guard OnionRequestAPI.guardSnodes.wrappedValue.count < targetGuardSnodeCount else { + return Just(OnionRequestAPI.guardSnodes.wrappedValue) + .setFailureType(to: Error.self) + .eraseToAnyPublisher() + } + + return Deferred { + Future<(unusedSnodes: Set, requiredGuardNodes: Int), Error> { resolver in + SNLog("[BuildPathsJob] Populating guard snode cache.") + let unusedSnodes: Set = SnodeAPI.snodePool.wrappedValue.subtracting(reusableGuardSnodes) + let requiredGuardNodes: Int = (targetGuardSnodeCount - reusableGuardSnodes.count) + + guard unusedSnodes.count >= requiredGuardNodes else { + return resolver(Result.failure(SnodeAPIError.insufficientSnodes)) + } + + resolver(Result.success((unusedSnodes, requiredGuardNodes))) + } + } + .flatMap { originalUnusedSnodes, requiredGuardNodes -> AnyPublisher, Error> in + var unusedSnodes: Set = originalUnusedSnodes + + func getGuardSnode() -> AnyPublisher { + // randomElement() uses the system's default random generator, which + // is cryptographically secure + guard let candidate = unusedSnodes.randomElement() else { + return Fail(error: SnodeAPIError.insufficientSnodes) + .eraseToAnyPublisher() + } + + unusedSnodes.remove(candidate) // All used snodes should be unique + SNLog("[BuildPathsJob] Testing guard snode: \(candidate).") + + // Loop until a reliable guard snode is found + return SnodeAPI + .testSnode( + snode: candidate, + ed25519SecretKey: ed25519SecretKey, + using: dependencies + ) + .map { _ in candidate } + .catch { _ in + return Just(()) + .setFailureType(to: Error.self) + .delay(for: .milliseconds(100), scheduler: queue) + .flatMap { _ in getGuardSnode() } + } + .eraseToAnyPublisher() + } + + return Publishers + .MergeMany((0.. AnyPublisher { + let paths: [[Snode]] = OnionRequestAPI.paths + + // Ensure the `guardSnodes` is up to date + if !paths.isEmpty { + OnionRequestAPI.guardSnodes.mutate { + $0.formUnion([ paths[0][0] ]) + + if paths.count >= 2 { + $0.formUnion([ paths[1][0] ]) + } + } + } + + // If we have enough paths then no need to do anything + guard paths.count < targetPathCount else { + return Just(()) + .setFailureType(to: Error.self) + .eraseToAnyPublisher() + } + + return Deferred { + Future { resolver in + let hasValidPath: Bool = snodeToExclude + .map { snode in paths.contains { !$0.contains(snode) } } + .defaulting(to: true) + + let targetJob: Job? = dependencies.storage.write(using: dependencies) { db in + // Fetch an existing job if there is one (if there are multiple it doesn't matter which we select) + if let existingJob: Job = try? Job.filter(Job.Columns.variant == Job.Variant.buildPaths).fetchOne(db) { + return existingJob + } + + return dependencies.jobRunner.add( + db, + job: Job( + variant: .buildPaths, + shouldBeUnique: true, + details: Details(reusablePaths: paths, ed25519SecretKey: ed25519SecretKey) + ), + canStartJob: true, + using: dependencies + ) + } + + guard let job: Job = targetJob else { + SNLog("[BuildPathsJob] Failed to retrieve existing job or schedule a new one.") + return resolver(Result.failure(JobRunnerError.generic)) + } + + // If we don't have a valid path then we should block this request until we have rebuilt + // the paths + guard hasValidPath else { + dependencies.jobRunner.afterJob(job) { result in + switch result { + case .succeeded: resolver(Result.success(())) + case .failed(let error, _): resolver(Result.failure(error ?? JobRunnerError.generic)) + case .deferred, .notFound: resolver(Result.failure(JobRunnerError.generic)) + } + } + return + } + + // Otherwise we can let the `BuildPathsJob` run in the background and should just return + // immediately + resolver(Result.success(())) + } + }.eraseToAnyPublisher() + } +} + +// MARK: - BuildPathsJob.Details + +extension BuildPathsJob { + public struct Details: Codable, UniqueHashable { + private enum CodingKeys: String, CodingKey { + case reusablePaths + case ed25519SecretKey + } + + fileprivate let reusablePaths: [[Snode]] + fileprivate let ed25519SecretKey: [UInt8]? + + // MARK: - UniqueHashable + + /// We want the `BuildPathsJob` to be unique regardless of what data is given to it + public var customHash: Int { + var hasher: Hasher = Hasher() + "BuildPathsJob.Details".hash(into: &hasher) + return hasher.finalize() + } + } +} diff --git a/SessionSnodeKit/Jobs/GetSnodePoolJob.swift b/SessionSnodeKit/Jobs/GetSnodePoolJob.swift index dbd5e5ef5..26bcfe6a0 100644 --- a/SessionSnodeKit/Jobs/GetSnodePoolJob.swift +++ b/SessionSnodeKit/Jobs/GetSnodePoolJob.swift @@ -41,7 +41,7 @@ public enum GetSnodePoolJob: JobExecutor { // If we don't have the snode pool cached then we should also try to build the path (this will // speed up the onboarding process for new users because it can run before the user is created) SnodeAPI.getSnodePool(ed25519SecretKey: ed25519SecretKey, using: dependencies) - .flatMap { _ in OnionRequestAPI.getPath(excluding: nil, ed25519SecretKey: ed25519SecretKey, using: dependencies) } + .flatMap { _ in BuildPathsJob.runIfNeeded(ed25519SecretKey: ed25519SecretKey, using: dependencies) } .subscribe(on: queue) .receive(on: queue) .sinkUntilComplete( diff --git a/SessionSnodeKit/Networking/OnionRequestAPI.swift b/SessionSnodeKit/Networking/OnionRequestAPI.swift index 30ee62519..65e6523da 100644 --- a/SessionSnodeKit/Networking/OnionRequestAPI.swift +++ b/SessionSnodeKit/Networking/OnionRequestAPI.swift @@ -80,7 +80,6 @@ public extension Network.RequestType { /// See the "Onion Requests" section of [The Session Whitepaper](https://arxiv.org/pdf/2002.04609.pdf) for more information. public enum OnionRequestAPI { - private static var buildPathsPublisher: Atomic?> = Atomic(nil) internal static var pathFailureCount: Atomic<[[Snode]: UInt]> = Atomic([:]) internal static var guardSnodes: Atomic> = Atomic([]) @@ -119,274 +118,12 @@ public enum OnionRequestAPI { private static let pathFailureThreshold: UInt = 3 /// The number of times a snode can fail before it's replaced. private static let snodeFailureThreshold: UInt = 3 - /// The number of paths to maintain. - public static let targetPathCount: UInt = 2 - - /// The number of guard snodes required to maintain `targetPathCount` paths. - private static var targetGuardSnodeCount: UInt { return targetPathCount } // One per path // MARK: - Onion Building Result private typealias OnionBuildingResult = (guardSnode: Snode, finalEncryptionResult: AES.GCM.EncryptionResult, destinationSymmetricKey: Data) // MARK: - Private API - - /// Finds `targetGuardSnodeCount` guard snodes to use for path building. The returned promise errors out with - /// `Error.insufficientSnodes` if not enough (reliable) snodes are available. - private static func getGuardSnodes( - reusing reusableGuardSnodes: [Snode], - ed25519SecretKey: [UInt8]?, - using dependencies: Dependencies - ) -> AnyPublisher, Error> { - guard guardSnodes.wrappedValue.count < targetGuardSnodeCount else { - return Just(guardSnodes.wrappedValue) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - - SNLog("Populating guard snode cache.") - // Sync on LokiAPI.workQueue - var unusedSnodes = SnodeAPI.snodePool.wrappedValue.subtracting(reusableGuardSnodes) - let reusableGuardSnodeCount = UInt(reusableGuardSnodes.count) - - guard unusedSnodes.count >= (targetGuardSnodeCount - reusableGuardSnodeCount) else { - return Fail(error: SnodeAPIError.insufficientSnodes) - .eraseToAnyPublisher() - } - - func getGuardSnode() -> AnyPublisher { - // randomElement() uses the system's default random generator, which - // is cryptographically secure - guard let candidate = unusedSnodes.randomElement() else { - return Fail(error: SnodeAPIError.insufficientSnodes) - .eraseToAnyPublisher() - } - - unusedSnodes.remove(candidate) // All used snodes should be unique - SNLog("Testing guard snode: \(candidate).") - - // Loop until a reliable guard snode is found - return SnodeAPI - .testSnode( - snode: candidate, - ed25519SecretKey: ed25519SecretKey, - using: dependencies - ) - .map { _ in candidate } - .catch { _ in - return Just(()) - .setFailureType(to: Error.self) - .delay(for: .milliseconds(100), scheduler: Threading.workQueue) - .flatMap { _ in getGuardSnode() } - } - .eraseToAnyPublisher() - } - - let publishers = (0..<(targetGuardSnodeCount - reusableGuardSnodeCount)) - .map { _ in getGuardSnode() } - - return Publishers.MergeMany(publishers) - .collect() - .map { output in Set(output) } - .handleEvents( - receiveOutput: { output in - OnionRequestAPI.guardSnodes.mutate { $0 = output } - } - ) - .eraseToAnyPublisher() - } - - /// Builds and returns `targetPathCount` paths. The returned promise errors out with `Error.insufficientSnodes` - /// if not enough (reliable) snodes are available. - @discardableResult - private static func buildPaths( - reusing reusablePaths: [[Snode]], - ed25519SecretKey: [UInt8]?, - using dependencies: Dependencies - ) -> AnyPublisher<[[Snode]], Error> { - if let existingBuildPathsPublisher = buildPathsPublisher.wrappedValue { - return existingBuildPathsPublisher - } - - return buildPathsPublisher.mutate { result in - /// It was possible for multiple threads to call this at the same time resulting in duplicate promises getting created, while - /// this should no longer be possible (as the `wrappedValue` should now properly be blocked) this is a sanity check - /// to make sure we don't create an additional promise when one already exists - if let previouslyBlockedPublisher: AnyPublisher<[[Snode]], Error> = result { - return previouslyBlockedPublisher - } - - SNLog("Building onion request paths.") - DispatchQueue.main.async { - NotificationCenter.default.post(name: .buildingPaths, object: nil) - } - - /// Need to include the post-request code and a `shareReplay` within the publisher otherwise it can still be executed - /// multiple times as a result of multiple subscribers - let reusableGuardSnodes = reusablePaths.map { $0[0] } - let publisher: AnyPublisher<[[Snode]], Error> = getGuardSnodes(reusing: reusableGuardSnodes, ed25519SecretKey: ed25519SecretKey, using: dependencies) - .flatMap { (guardSnodes: Set) -> AnyPublisher<[[Snode]], Error> in - var unusedSnodes: Set = SnodeAPI.snodePool.wrappedValue - .subtracting(guardSnodes) - .subtracting(reusablePaths.flatMap { $0 }) - let reusableGuardSnodeCount: UInt = UInt(reusableGuardSnodes.count) - let pathSnodeCount: UInt = (targetGuardSnodeCount - reusableGuardSnodeCount) * pathSize - (targetGuardSnodeCount - reusableGuardSnodeCount) - - guard unusedSnodes.count >= pathSnodeCount else { - return Fail<[[Snode]], Error>(error: SnodeAPIError.insufficientSnodes) - .eraseToAnyPublisher() - } - - // Don't test path snodes as this would reveal the user's IP to them - let paths: [[Snode]] = guardSnodes - .subtracting(reusableGuardSnodes) - .map { (guardSnode: Snode) in - let result: [Snode] = [guardSnode] - .appending( - contentsOf: (0..<(pathSize - 1)) - .map { _ in - // randomElement() uses the system's default random generator, - // which is cryptographically secure - let pathSnode: Snode = unusedSnodes.randomElement()! // Safe because of the pathSnodeCount check above - unusedSnodes.remove(pathSnode) // All used snodes should be unique - return pathSnode - } - ) - - SNLog("Built new onion request path: \(result.prettifiedDescription).") - return result - } - - return Just(paths) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - .handleEvents( - receiveOutput: { output in - OnionRequestAPI.paths = (output + reusablePaths) - - Storage.shared.write { db in - SNLog("Persisting onion request paths to database.") - try? output.save(db) - } - - DispatchQueue.main.async { - NotificationCenter.default.post(name: .pathsBuilt, object: nil) - } - }, - receiveCompletion: { _ in buildPathsPublisher.mutate { $0 = nil } } - ) - .shareReplay(1) - .eraseToAnyPublisher() - - /// Actually assign the atomic value - result = publisher - - return publisher - } - } - - /// Returns a `Path` to be used for building an onion request. Builds new paths as needed. - internal static func getPath( - excluding snode: Snode?, - ed25519SecretKey: [UInt8]?, - using dependencies: Dependencies - ) -> AnyPublisher<[Snode], Error> { - guard pathSize >= 1 else { preconditionFailure("Can't build path of size zero.") } - - let paths: [[Snode]] = OnionRequestAPI.paths - var cancellable: [AnyCancellable] = [] - - if !paths.isEmpty { - guardSnodes.mutate { - $0.formUnion([ paths[0][0] ]) - - if paths.count >= 2 { - $0.formUnion([ paths[1][0] ]) - } - } - } - - // randomElement() uses the system's default random generator, which is cryptographically secure - if - paths.count >= targetPathCount, - let targetPath: [Snode] = paths - .filter({ snode == nil || !$0.contains(snode!) }) - .randomElement() - { - return Just(targetPath) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - else if !paths.isEmpty { - if let snode = snode { - if let path = paths.first(where: { !$0.contains(snode) }) { - buildPaths(reusing: paths, ed25519SecretKey: ed25519SecretKey, using: dependencies) // Re-build paths in the background - .subscribe(on: DispatchQueue.global(qos: .background), using: dependencies) - .sink(receiveCompletion: { _ in cancellable = [] }, receiveValue: { _ in }) - .store(in: &cancellable) - - return Just(path) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - else { - return buildPaths(reusing: paths, ed25519SecretKey: ed25519SecretKey, using: dependencies) - .flatMap { paths in - guard let path: [Snode] = paths.filter({ !$0.contains(snode) }).randomElement() else { - return Fail<[Snode], Error>(error: SnodeAPIError.insufficientSnodes) - .eraseToAnyPublisher() - } - - return Just(path) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - .eraseToAnyPublisher() - } - } - else { - buildPaths(reusing: paths, ed25519SecretKey: ed25519SecretKey, using: dependencies) // Re-build paths in the background - .subscribe(on: DispatchQueue.global(qos: .background)) - .sink(receiveCompletion: { _ in cancellable = [] }, receiveValue: { _ in }) - .store(in: &cancellable) - - guard let path: [Snode] = paths.randomElement() else { - return Fail<[Snode], Error>(error: SnodeAPIError.insufficientSnodes) - .eraseToAnyPublisher() - } - - return Just(path) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - } - else { - return buildPaths(reusing: [], ed25519SecretKey: ed25519SecretKey, using: dependencies) - .flatMap { paths in - if let snode = snode { - if let path = paths.filter({ !$0.contains(snode) }).randomElement() { - return Just(path) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - - return Fail<[Snode], Error>(error: SnodeAPIError.insufficientSnodes) - .eraseToAnyPublisher() - } - - guard let path: [Snode] = paths.randomElement() else { - return Fail<[Snode], Error>(error: SnodeAPIError.insufficientSnodes) - .eraseToAnyPublisher() - } - - return Just(path) - .setFailureType(to: Error.self) - .eraseToAnyPublisher() - } - .eraseToAnyPublisher() - } - } internal static func dropGuardSnode(_ snode: Snode?) { guardSnodes.mutate { snodes in snodes = snodes.filter { $0 != snode } } @@ -394,7 +131,7 @@ public enum OnionRequestAPI { private static func drop(_ snode: Snode) throws { // We repair the path here because we can do it sync. In the case where we drop a whole - // path we leave the re-building up to getPath(excluding:using:) because re-building the path + // path we leave the re-building up to the `BuildPathsJob` because re-building the path // in that case is async. SnodeAPI.snodeFailureCount.mutate { $0[snode] = 0 } var oldPaths = paths @@ -451,8 +188,13 @@ public enum OnionRequestAPI { }() let ed25519SecretKey: [UInt8]? = Identity.fetchUserEd25519KeyPair()?.secretKey - return getPath(excluding: snodeToExclude, ed25519SecretKey: ed25519SecretKey, using: dependencies) - .tryFlatMap { path -> AnyPublisher<(ResponseInfoType, Data?), Error> in + return BuildPathsJob + .runIfNeeded( + excluding: snodeToExclude, + ed25519SecretKey: ed25519SecretKey, + using: dependencies + ) + .tryFlatMap { _ -> AnyPublisher<(ResponseInfoType, Data?), Error> in LibSession.sendOnionRequest( to: destination, body: body, diff --git a/SessionUtilitiesKit/General/Logging.swift b/SessionUtilitiesKit/General/Logging.swift index cf5208648..7e11e6e01 100644 --- a/SessionUtilitiesKit/General/Logging.swift +++ b/SessionUtilitiesKit/General/Logging.swift @@ -22,7 +22,7 @@ private extension DispatchQueue { } } -public func SNLog(_ message: String) { +public func SNLog(_ message: String, forceNSLog: Bool = false) { let logPrefixes: String = [ "Session", (Thread.isMainThread ? "Main" : nil), @@ -35,6 +35,10 @@ public func SNLog(_ message: String) { print("[\(logPrefixes)] \(message)") #endif OWSLogger.info("[\(logPrefixes)] \(message)") + + if forceNSLog { + NSLog(message) + } } public func SNLogNotTests(_ message: String) { diff --git a/SessionUtilitiesKit/JobRunner/JobRunner.swift b/SessionUtilitiesKit/JobRunner/JobRunner.swift index fb83b973e..265b65bd4 100644 --- a/SessionUtilitiesKit/JobRunner/JobRunner.swift +++ b/SessionUtilitiesKit/JobRunner/JobRunner.swift @@ -255,7 +255,8 @@ public final class JobRunner: JobRunnerType { jobVariants.remove(.notifyPushServer), jobVariants.remove(.sendReadReceipts), jobVariants.remove(.groupLeaving), - jobVariants.remove(.configurationSync) + jobVariants.remove(.configurationSync), + jobVariants.remove(.buildPaths) ].compactMap { $0 } ),