Fixed a few issues uncovered while testing and some cleanup

Fixed an incorrect optional in RoomPollInfo
Fixed an incorrect parameter name in the ClosedGroupRequestBody
Fixed a crash due to a change in the ContactUtilities
Cleaned up the duplicate code in the OnionRequestAPI, HTTP and SnodeAPI to all use 'Data' response types
Updated the SnodeAPI to casting types to Any (made it hard to catch breaking changes with HTTP and OnionRequestAPI)

Session/Utilities/BackgroundPoller.swift

@@ -50,8 +50,8 @@ public final class BackgroundPoller: NSObject {
                 
                 return attempt(maxRetryCount: 4, recoveringOn: DispatchQueue.main) {
                     return SnodeAPI.getRawMessages(from: snode, associatedWith: publicKey)
-                        .then(on: DispatchQueue.main) { rawResponse -> Promise<Void> in
-                            let messages = SnodeAPI.parseRawMessagesResponse(rawResponse, from: snode, associatedWith: publicKey)
+                        .then(on: DispatchQueue.main) { responseData -> Promise<Void> in
+                            let messages = SnodeAPI.parseRawMessagesResponse(responseData, from: snode, associatedWith: publicKey)
                             let promises = messages
                                 .compactMap { json -> Promise<Void>? in
                                     // Use a best attempt approach here; we don't want to fail

Session/Utilities/ContactUtilities.swift

@@ -32,8 +32,10 @@ enum ContactUtilities {
         
         // Sort alphabetically
         return result
-            .map { contact -> String in (contact.displayName(for: .regular) ?? contact.sessionID) }
-            .sorted()
+            .sorted(by: { lhs, rhs in
+                (lhs.displayName(for: .regular) ?? lhs.sessionID) < (rhs.displayName(for: .regular) ?? rhs.sessionID)
+            })
+            .map { $0.sessionID }
     }
     
     static func enumerateApprovedContactThreads(with block: @escaping (TSContactThread, Contact, UnsafeMutablePointer<ObjCBool>) -> ()) {

SessionMessagingKit/Open Groups/Models/RoomPollInfo.swift

@@ -27,7 +27,7 @@ extension OpenGroupAPI {
         }
         
         /// The room token as used in a URL, e.g. "sudoku"
-        public let token: String?
+        public let token: String
         
         /// Number of recently active users in the room over a recent time period (as given in the active_users_cutoff value)
         ///

SessionMessagingKit/Open Groups/OpenGroupAPI.swift

@@ -549,7 +549,6 @@ public final class OpenGroupAPI: NSObject {
         )
         
         return send(request, using: dependencies)
-            .decoded(as: [DirectMessage].self, on: OpenGroupAPI.workQueue, error: Error.parsingFailed, using: dependencies)
     }
     
     // MARK: - Users

SessionMessagingKit/Sending & Receiving/Notifications/PushNotificationAPI.swift

@@ -9,7 +9,7 @@ public final class PushNotificationAPI : NSObject {
     }
     
     struct ClosedGroupRequestBody: Codable {
-        let token: String
+        let closedGroupPublicKey: String
         let pubKey: String
     }
 
@@ -51,7 +51,6 @@ public final class PushNotificationAPI : NSObject {
         request.httpBody = body
         
         let promise: Promise<Void> = attempt(maxRetryCount: maxRetryCount, recoveringOn: DispatchQueue.global()) {
-            // TODO: Update this to use the V4 union requests once supported
             OnionRequestAPI.sendOnionRequest(request, to: server, using: .v2, with: serverPublicKey)
                 .map2 { _, response in
                     guard let response: UnregisterResponse = try? response?.decoded(as: UnregisterResponse.self) else {
@@ -101,7 +100,6 @@ public final class PushNotificationAPI : NSObject {
         request.httpBody = body
         
         let promise: Promise<Void> = attempt(maxRetryCount: maxRetryCount, recoveringOn: DispatchQueue.global()) {
-            // TODO: Update this to use the V4 union requests once supported
             OnionRequestAPI.sendOnionRequest(request, to: server, using: .v2, with: serverPublicKey)
                 .map2 { _, response in
                     guard let response: RegisterResponse = try? response?.decoded(as: RegisterResponse.self) else {
@@ -134,7 +132,10 @@ public final class PushNotificationAPI : NSObject {
     @discardableResult
     public static func performOperation(_ operation: ClosedGroupOperation, for closedGroupPublicKey: String, publicKey: String) -> Promise<Void> {
         let isUsingFullAPNs = UserDefaults.standard[.isUsingFullAPNs]
-        let requestBody: ClosedGroupRequestBody = ClosedGroupRequestBody(token: closedGroupPublicKey, pubKey: publicKey)
+        let requestBody: ClosedGroupRequestBody = ClosedGroupRequestBody(
+            closedGroupPublicKey: closedGroupPublicKey,
+            pubKey: publicKey
+        )
         
         guard isUsingFullAPNs else { return Promise<Void> { $0.fulfill(()) } }
         guard let body: Data = try? JSONEncoder().encode(requestBody) else {
@@ -148,7 +149,6 @@ public final class PushNotificationAPI : NSObject {
         request.httpBody = body
         
         let promise: Promise<Void> = attempt(maxRetryCount: maxRetryCount, recoveringOn: DispatchQueue.global()) {
-            // TODO: Update this to use the V4 union requests once supported
             OnionRequestAPI.sendOnionRequest(request, to: server, using: .v2, with: serverPublicKey)
                 .map2 { _, response in
                     guard let response: RegisterResponse = try? response?.decoded(as: RegisterResponse.self) else {

SessionMessagingKit/Sending & Receiving/Pollers/Poller.swift

@@ -87,9 +87,9 @@ public final class Poller : NSObject {
     private func poll(_ snode: Snode, seal longTermSeal: Resolver<Void>) -> Promise<Void> {
         guard isPolling else { return Promise { $0.fulfill(()) } }
         let userPublicKey = getUserHexEncodedPublicKey()
-        return SnodeAPI.getRawMessages(from: snode, associatedWith: userPublicKey).then(on: DispatchQueue.main) { [weak self] rawResponse -> Promise<Void> in
+        return SnodeAPI.getRawMessages(from: snode, associatedWith: userPublicKey).then(on: DispatchQueue.main) { [weak self] responseData -> Promise<Void> in
             guard let strongSelf = self, strongSelf.isPolling else { return Promise { $0.fulfill(()) } }
-            let messages = SnodeAPI.parseRawMessagesResponse(rawResponse, from: snode, associatedWith: userPublicKey)
+            let messages = SnodeAPI.parseRawMessagesResponse(responseData, from: snode, associatedWith: userPublicKey)
             if !messages.isEmpty {
                 SNLog("Received \(messages.count) new message(s).")
             }

SessionSnodeKit/Models/Error.swift

@@ -5,7 +5,7 @@ import SessionUtilitiesKit
 
 extension OnionRequestAPI {
     public enum Error: LocalizedError {
-        case httpRequestFailedAtDestination(statusCode: UInt, json: JSON, destination: Destination)
+        case httpRequestFailedAtDestination(statusCode: UInt, data: Data, destination: Destination)
         case insufficientSnodes
         case invalidURL
         case missingSnodeVersion

SessionSnodeKit/OnionRequestAPI.swift

@@ -9,10 +9,6 @@ public protocol OnionRequestAPIType {
 }
 
 public extension OnionRequestAPIType {
-    static func sendOnionRequest(to snode: Snode, invoking method: Snode.Method, with parameters: JSON, using version: OnionRequestAPI.Version = .v3) -> Promise<Data> {
-        return sendOnionRequest(to: snode, invoking: method, with: parameters, using: version, associatedWith: nil)
-    }
-    
     static func sendOnionRequest(_ request: URLRequest, to server: String, with x25519PublicKey: String) -> Promise<(OnionRequestResponseInfoType, Data?)> {
         sendOnionRequest(request, to: server, using: .v4, with: x25519PublicKey)
     }
@@ -50,24 +46,32 @@ public enum OnionRequestAPI: OnionRequestAPIType {
     // MARK: Onion Building Result
     private typealias OnionBuildingResult = (guardSnode: Snode, finalEncryptionResult: AESGCM.EncryptionResult, destinationSymmetricKey: Data)
 
-    // MARK: Private API
+    // MARK: - Private API
     /// Tests the given snode. The returned promise errors out if the snode is faulty; the promise is fulfilled otherwise.
     private static func testSnode(_ snode: Snode) -> Promise<Void> {
         let (promise, seal) = Promise<Void>.pending()
         DispatchQueue.global(qos: .userInitiated).async {
             let url = "\(snode.address):\(snode.port)/get_stats/v1"
             let timeout: TimeInterval = 3 // Use a shorter timeout for testing
-            HTTP.execute(.get, url, timeout: timeout).done2 { json in
-                guard let version = json["version"] as? String else { return seal.reject(Error.missingSnodeVersion) }
-                if version >= "2.0.7" {
-                    seal.fulfill(())
-                } else {
-                    SNLog("Unsupported snode version: \(version).")
-                    seal.reject(Error.unsupportedSnodeVersion(version))
+            
+            HTTP.execute(.get, url, timeout: timeout)
+                .done2 { responseData in
+                    guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+                        throw HTTP.Error.invalidJSON
+                    }
+                    guard let version = responseJson["version"] as? String else { return seal.reject(Error.missingSnodeVersion) }
+                    
+                    if version >= "2.0.7" {
+                        seal.fulfill(())
+                    }
+                    else {
+                        SNLog("Unsupported snode version: \(version).")
+                        seal.reject(Error.unsupportedSnodeVersion(version))
+                    }
+                }
+                .catch2 { error in
+                    seal.reject(error)
                 }
-            }.catch2 { error in
-                seal.reject(error)
-            }
         }
         return promise
     }
@@ -280,7 +284,7 @@ public enum OnionRequestAPI: OnionRequestAPIType {
     // MARK: - Public API
     
     /// Sends an onion request to `snode`. Builds new paths as needed.
-    public static func sendOnionRequest(to snode: Snode, invoking method: Snode.Method, with parameters: JSON, using version: Version = .v3, associatedWith publicKey: String? = nil) -> Promise<Data> {
+    public static func sendOnionRequest(to snode: Snode, invoking method: Snode.Method, with parameters: JSON, using version: Version, associatedWith publicKey: String?) -> Promise<Data> {
         let payloadJson: JSON = [ "method": method.rawValue, "params": parameters ]
         
         guard let jsonData: Data = try? JSONSerialization.data(withJSONObject: payloadJson, options: []), let payload: String = String(data: jsonData, encoding: .utf8) else {
@@ -294,11 +298,11 @@ public enum OnionRequestAPI: OnionRequestAPIType {
                 return data
             }
             .recover2 { error -> Promise<Data> in
-                guard case OnionRequestAPI.Error.httpRequestFailedAtDestination(let statusCode, let json, _) = error else {
+                guard case OnionRequestAPI.Error.httpRequestFailedAtDestination(let statusCode, let data, _) = error else {
                     throw error
                 }
                 
-                throw SnodeAPI.handleError(withStatusCode: statusCode, json: json, forSnode: snode, associatedWith: publicKey) ?? error
+                throw SnodeAPI.handleError(withStatusCode: statusCode, data: data, forSnode: snode, associatedWith: publicKey) ?? error
             }
     }
 
@@ -347,7 +351,7 @@ public enum OnionRequestAPI: OnionRequestAPIType {
                 }
                 let destinationSymmetricKey = intermediate.destinationSymmetricKey
                 
-                HTTP.updatedExecute(.post, url, body: body)
+                HTTP.execute(.post, url, body: body)
                     .done2 { responseData in
                         handleResponse(
                             responseData: responseData,
@@ -366,7 +370,7 @@ public enum OnionRequestAPI: OnionRequestAPIType {
         }
         
         promise.catch2 { error in // Must be invoked on Threading.workQueue
-            guard case HTTP.Error.httpRequestFailed(let statusCode, let json) = error, let guardSnode = guardSnode else {
+            guard case HTTP.Error.httpRequestFailed(let statusCode, let data) = error, let guardSnode = guardSnode else {
                 return
             }
             
@@ -381,7 +385,7 @@ public enum OnionRequestAPI: OnionRequestAPIType {
                 if pathFailureCount >= pathFailureThreshold {
                     dropGuardSnode(guardSnode)
                     path.forEach { snode in
-                        SnodeAPI.handleError(withStatusCode: statusCode, json: json, forSnode: snode) // Intentionally don't throw
+                        SnodeAPI.handleError(withStatusCode: statusCode, data: data, forSnode: snode) // Intentionally don't throw
                     }
                     
                     drop(path)
@@ -392,6 +396,17 @@ public enum OnionRequestAPI: OnionRequestAPIType {
             }
             
             let prefix = "Next node not found: "
+            let json: JSON?
+            
+            if let data: Data = data, let processedJson = try? JSONSerialization.jsonObject(with: data, options: [ .fragmentsAllowed ]) as? JSON {
+                json = processedJson
+            }
+            else if let data: Data = data, let result: String = String(data: data, encoding: .utf8) {
+                json = [ "result": result ]
+            }
+            else {
+                json = nil
+            }
             
             if let message = json?["result"] as? String, message.hasPrefix(prefix) {
                 let ed25519PublicKey = message[message.index(message.startIndex, offsetBy: prefix.count)..<message.endIndex]
@@ -401,7 +416,7 @@ public enum OnionRequestAPI: OnionRequestAPIType {
                     snodeFailureCount += 1
                     
                     if snodeFailureCount >= snodeFailureThreshold {
-                        SnodeAPI.handleError(withStatusCode: statusCode, json: json, forSnode: snode) // Intentionally don't throw
+                        SnodeAPI.handleError(withStatusCode: statusCode, data: data, forSnode: snode) // Intentionally don't throw
                         do {
                             try drop(snode)
                         }
@@ -483,7 +498,6 @@ public enum OnionRequestAPI: OnionRequestAPIType {
             case .v4:
                 // Note: We need to remove the leading forward slash unless we are explicitly hitting a legacy
                 // endpoint (in which case we need it to ensure the request signing works correctly
-                // TODO: Confirm the 'removingPrefix' isn't going to break the request signing on non-legacy endpoints
                 let endpoint: String = url.path
                     .appending(url.query.map { value in "?\(value)" })
                 
@@ -493,9 +507,9 @@ public enum OnionRequestAPI: OnionRequestAPIType {
                     headers: (request.allHTTPHeaderFields ?? [:])
                         .setting(
                             "Content-Type",
-                            // TODO: Determine what 'Content-Type' 'httpBodyStream' should have???.
                             (request.httpBody == nil && request.httpBodyStream == nil ? nil :
-                                ((request.allHTTPHeaderFields ?? [:])["Content-Type"] ?? "application/json")    // Default to JSON if not defined
+                                // Default to JSON if not defined
+                                ((request.allHTTPHeaderFields ?? [:])["Content-Type"] ?? "application/json")
                             )
                         )
                         .removingValue(forKey: "User-Agent")
@@ -573,14 +587,14 @@ public enum OnionRequestAPI: OnionRequestAPIType {
                         }
                         
                         guard 200...299 ~= statusCode else {
-                            return seal.reject(Error.httpRequestFailedAtDestination(statusCode: UInt(statusCode), json: body, destination: destination))
+                            return seal.reject(Error.httpRequestFailedAtDestination(statusCode: UInt(statusCode), data: bodyAsData, destination: destination))
                         }
                         
                         return seal.fulfill((OnionRequestAPI.ResponseInfo(code: statusCode, headers: [:]), bodyAsData))
                     }
                     
                     guard 200...299 ~= statusCode else {
-                        return seal.reject(Error.httpRequestFailedAtDestination(statusCode: UInt(statusCode), json: json, destination: destination))
+                        return seal.reject(Error.httpRequestFailedAtDestination(statusCode: UInt(statusCode), data: data, destination: destination))
                     }
                     
                     return seal.fulfill((OnionRequestAPI.ResponseInfo(code: statusCode, headers: [:]), data))
@@ -628,7 +642,7 @@ public enum OnionRequestAPI: OnionRequestAPIType {
                         return seal.reject(
                             Error.httpRequestFailedAtDestination(
                                 statusCode: UInt(responseInfo.code),
-                                json: [:],  // TODO: Remove the 'json' value??
+                                data: data,
                                 destination: destination
                             )
                         )

SessionSnodeKit/SnodeAPI.swift

@@ -60,11 +60,6 @@ public final class SnodeAPI : NSObject {
         }
     }
 
-    // MARK: Type Aliases
-    public typealias MessageListPromise = Promise<[JSON]>
-    public typealias RawResponse = Any
-    public typealias RawResponsePromise = Promise<RawResponse>
-    
     // MARK: Snode Pool Interaction
     private static func loadSnodePoolIfNeeded() {
         guard !hasLoadedSnodePool else { return }
@@ -129,30 +124,26 @@ public final class SnodeAPI : NSObject {
     }
     
     // MARK: Internal API
-    internal static func invoke(_ method: Snode.Method, on snode: Snode, associatedWith publicKey: String? = nil, parameters: JSON) -> RawResponsePromise {
+    internal static func invoke(_ method: Snode.Method, on snode: Snode, associatedWith publicKey: String? = nil, parameters: JSON) -> Promise<Data> {
         if Features.useOnionRequests {
             return OnionRequestAPI.sendOnionRequest(to: snode, invoking: method, with: parameters, using: .v3, associatedWith: publicKey)
-                .map2 { responseData in
-                    guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
-                        throw Error.generic
-                    }
-                    
-                    // FIXME: Would be nice to change this to not send 'Any'
-                    return responseJson as Any
-                }
-        } else {
+        }
+        else {
             let url = "\(snode.address):\(snode.port)/storage_rpc/v1"
-            return HTTP.execute(.post, url, parameters: parameters).map2 { $0 as Any }.recover2 { error -> Promise<Any> in
-                guard case HTTP.Error.httpRequestFailed(let statusCode, let json) = error else { throw error }
-                throw SnodeAPI.handleError(withStatusCode: statusCode, json: json, forSnode: snode, associatedWith: publicKey) ?? error
-            }
+            return HTTP.execute(.post, url, parameters: parameters)
+                .recover2 { error -> Promise<Data> in
+                    guard case HTTP.Error.httpRequestFailed(let statusCode, let data) = error else { throw error }
+                    throw SnodeAPI.handleError(withStatusCode: statusCode, data: data, forSnode: snode, associatedWith: publicKey) ?? error
+                }
         }
     }
     
     private static func getNetworkTime(from snode: Snode) -> Promise<UInt64> {
-        return invoke(.getInfo, on: snode, parameters: [:]).map2 { rawResponse in
-            guard let json = rawResponse as? JSON,
-                let timestamp = json["timestamp"] as? UInt64 else { throw HTTP.Error.invalidJSON }
+        return invoke(.getInfo, on: snode, parameters: [:]).map2 { responseData in
+            guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+                throw HTTP.Error.invalidJSON
+            }
+            guard let timestamp = responseJson["timestamp"] as? UInt64 else { throw HTTP.Error.invalidJSON }
             return timestamp
         }
     }
@@ -179,21 +170,27 @@ public final class SnodeAPI : NSObject {
         let (promise, seal) = Promise<Set<Snode>>.pending()
         Threading.workQueue.async {
             attempt(maxRetryCount: 4, recoveringOn: Threading.workQueue) {
-                HTTP.execute(.post, url, parameters: parameters, useSeedNodeURLSession: true).map2 { json -> Set<Snode> in
-                    guard let intermediate = json["result"] as? JSON, let rawSnodes = intermediate["service_node_states"] as? [JSON] else { throw Error.snodePoolUpdatingFailed }
-                    return Set(rawSnodes.compactMap { rawSnode in
-                        guard let address = rawSnode["public_ip"] as? String, let port = rawSnode["storage_port"] as? Int,
-                            let ed25519PublicKey = rawSnode["pubkey_ed25519"] as? String, let x25519PublicKey = rawSnode["pubkey_x25519"] as? String, address != "0.0.0.0" else {
-                            SNLog("Failed to parse snode from: \(rawSnode).")
-                            return nil
+                HTTP.execute(.post, url, parameters: parameters, useSeedNodeURLSession: true)
+                    .map2 { responseData -> Set<Snode> in
+                        guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+                            throw HTTP.Error.invalidJSON
                         }
-                        return Snode(address: "https://\(address)", port: UInt16(port), publicKeySet: Snode.KeySet(ed25519Key: ed25519PublicKey, x25519Key: x25519PublicKey))
-                    })
-                }
-            }.done2 { snodePool in
+                        guard let intermediate = responseJson["result"] as? JSON, let rawSnodes = intermediate["service_node_states"] as? [JSON] else { throw Error.snodePoolUpdatingFailed }
+                        return Set(rawSnodes.compactMap { rawSnode in
+                            guard let address = rawSnode["public_ip"] as? String, let port = rawSnode["storage_port"] as? Int,
+                                let ed25519PublicKey = rawSnode["pubkey_ed25519"] as? String, let x25519PublicKey = rawSnode["pubkey_x25519"] as? String, address != "0.0.0.0" else {
+                                SNLog("Failed to parse snode from: \(rawSnode).")
+                                return nil
+                            }
+                            return Snode(address: "https://\(address)", port: UInt16(port), publicKeySet: Snode.KeySet(ed25519Key: ed25519PublicKey, x25519Key: x25519PublicKey))
+                        })
+                    }
+            }
+            .done2 { snodePool in
                 SNLog("Got snode pool from seed node: \(target).")
                 seal.fulfill(snodePool)
-            }.catch2 { error in
+            }
+            .catch2 { error in
                 SNLog("Failed to contact seed node at: \(target).")
                 seal.reject(error)
             }
@@ -223,20 +220,24 @@ public final class SnodeAPI : NSObject {
                         ]
                     ]
                 ]
-                return invoke(.oxenDaemonRPCCall, on: snode, parameters: parameters).map2 { rawResponse in
-                    guard let json = rawResponse as? JSON, let intermediate = json["result"] as? JSON,
-                        let rawSnodes = intermediate["service_node_states"] as? [JSON] else {
-                        throw Error.snodePoolUpdatingFailed
-                    }
-                    return Set(rawSnodes.compactMap { rawSnode in
-                        guard let address = rawSnode["public_ip"] as? String, let port = rawSnode["storage_port"] as? Int,
-                            let ed25519PublicKey = rawSnode["pubkey_ed25519"] as? String, let x25519PublicKey = rawSnode["pubkey_x25519"] as? String, address != "0.0.0.0" else {
-                            SNLog("Failed to parse snode from: \(rawSnode).")
-                            return nil
+                return invoke(.oxenDaemonRPCCall, on: snode, parameters: parameters)
+                    .map2 { responseData in
+                        guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+                            throw HTTP.Error.invalidJSON
                         }
-                        return Snode(address: "https://\(address)", port: UInt16(port), publicKeySet: Snode.KeySet(ed25519Key: ed25519PublicKey, x25519Key: x25519PublicKey))
-                    })
-                }
+                        guard let intermediate = responseJson["result"] as? JSON,
+                            let rawSnodes = intermediate["service_node_states"] as? [JSON] else {
+                            throw Error.snodePoolUpdatingFailed
+                        }
+                        return Set(rawSnodes.compactMap { rawSnode in
+                            guard let address = rawSnode["public_ip"] as? String, let port = rawSnode["storage_port"] as? Int,
+                                let ed25519PublicKey = rawSnode["pubkey_ed25519"] as? String, let x25519PublicKey = rawSnode["pubkey_x25519"] as? String, address != "0.0.0.0" else {
+                                SNLog("Failed to parse snode from: \(rawSnode).")
+                                return nil
+                            }
+                            return Snode(address: "https://\(address)", port: UInt16(port), publicKeySet: Snode.KeySet(ed25519Key: ed25519PublicKey, x25519Key: x25519PublicKey))
+                        })
+                    }
             }
         }
         let promise = when(fulfilled: snodePoolPromises).map2 { results -> Set<Snode> in
@@ -336,8 +337,11 @@ public final class SnodeAPI : NSObject {
             for result in results {
                 switch result {
                 case .rejected(let error): return seal.reject(error)
-                case .fulfilled(let rawResponse):
-                    guard let json = rawResponse as? JSON, let intermediate = json["result"] as? JSON,
+                case .fulfilled(let responseData):
+                    guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+                        throw HTTP.Error.invalidJSON
+                    }
+                    guard let intermediate = responseJson["result"] as? JSON,
                         let hexEncodedCiphertext = intermediate["encrypted_value"] as? String else { return seal.reject(HTTP.Error.invalidJSON) }
                     let ciphertext = [UInt8](Data(hex: hexEncodedCiphertext))
                     let isArgon2Based = (intermediate["nonce"] == nil)
@@ -390,37 +394,23 @@ public final class SnodeAPI : NSObject {
                 attempt(maxRetryCount: 4, recoveringOn: Threading.workQueue) {
                     invoke(.getSwarm, on: snode, associatedWith: publicKey, parameters: parameters)
                 }
-            }.map2 { rawSnodes in
-                let swarm = parseSnodes(from: rawSnodes)
+            }.map2 { responseData in
+                let swarm = parseSnodes(from: responseData)
                 setSwarm(to: swarm, for: publicKey)
                 return swarm
             }
         }
     }
     
-    public static func getRawMessages(from snode: Snode, associatedWith publicKey: String) -> RawResponsePromise {
-        let (promise, seal) = RawResponsePromise.pending()
+    public static func getRawMessages(from snode: Snode, associatedWith publicKey: String) -> Promise<Data> {
+        let (promise, seal) = Promise<Data>.pending()
         Threading.workQueue.async {
             getMessagesInternal(from: snode, associatedWith: publicKey).done2 { seal.fulfill($0) }.catch2 { seal.reject($0) }
         }
         return promise
     }
-
-    public static func getMessages(for publicKey: String) -> Promise<Set<MessageListPromise>> {
-        let (promise, seal) = Promise<Set<MessageListPromise>>.pending()
-        Threading.workQueue.async {
-            attempt(maxRetryCount: maxRetryCount, recoveringOn: Threading.workQueue) {
-                getTargetSnodes(for: publicKey).mapValues2 { targetSnode in
-                    return getMessagesInternal(from: targetSnode, associatedWith: publicKey).map2 { rawResponse in
-                        parseRawMessagesResponse(rawResponse, from: targetSnode, associatedWith: publicKey)
-                    }
-                }.map2 { Set($0) }
-            }.done2 { seal.fulfill($0) }.catch2 { seal.reject($0) }
-        }
-        return promise
-    }
     
-    private static func getMessagesInternal(from snode: Snode, associatedWith publicKey: String) -> RawResponsePromise {
+    private static func getMessagesInternal(from snode: Snode, associatedWith publicKey: String) -> Promise<Data> {
         let storage = SNSnodeKitConfiguration.shared.storage
         
         // NOTE: All authentication logic is currently commented out, the reason being that we can't currently support
@@ -447,18 +437,21 @@ public final class SnodeAPI : NSObject {
         return invoke(.getMessages, on: snode, associatedWith: publicKey, parameters: parameters)
     }
 
-    public static func sendMessage(_ message: SnodeMessage) -> Promise<Set<RawResponsePromise>> {
-        let (promise, seal) = Promise<Set<RawResponsePromise>>.pending()
+    public static func sendMessage(_ message: SnodeMessage) -> Promise<Set<Promise<Data>>> {
+        let (promise, seal) = Promise<Set<Promise<Data>>>.pending()
         let publicKey = Features.useTestnet ? message.recipient.removingIdPrefixIfNeeded() : message.recipient
         Threading.workQueue.async {
-            getTargetSnodes(for: publicKey).map2 { targetSnodes in
-                let parameters = message.toJSON()
-                return Set(targetSnodes.map { targetSnode in
-                    attempt(maxRetryCount: maxRetryCount, recoveringOn: Threading.workQueue) {
-                        invoke(.sendMessage, on: targetSnode, associatedWith: publicKey, parameters: parameters)
-                    }
-                })
-            }.done2 { seal.fulfill($0) }.catch2 { seal.reject($0) }
+            getTargetSnodes(for: publicKey)
+                .map2 { targetSnodes in
+                    let parameters = message.toJSON()
+                    return Set(targetSnodes.map { targetSnode in
+                        attempt(maxRetryCount: maxRetryCount, recoveringOn: Threading.workQueue) {
+                            invoke(.sendMessage, on: targetSnode, associatedWith: publicKey, parameters: parameters)
+                        }
+                    })
+                }
+                .done2 { seal.fulfill($0) }
+                .catch2 { seal.reject($0) }
         }
         return promise
     }
@@ -485,29 +478,34 @@ public final class SnodeAPI : NSObject {
                     "signature": signature.toBase64()
                 ]
                 return attempt(maxRetryCount: maxRetryCount, recoveringOn: Threading.workQueue) {
-                    invoke(.deleteMessage, on: snode, associatedWith: publicKey, parameters: parameters).map2{ rawResponse -> [String:Bool] in
-                        guard let json = rawResponse as? JSON, let swarm = json["swarm"] as? JSON else { throw HTTP.Error.invalidJSON }
-                        var result: [String:Bool] = [:]
-                        for (snodePublicKey, rawJSON) in swarm {
-                            guard let json = rawJSON as? JSON else { throw HTTP.Error.invalidJSON }
-                            let isFailed = json["failed"] as? Bool ?? false
-                            if !isFailed {
-                                guard let hashes = json["deleted"] as? [String], let signature = json["signature"] as? String else { throw HTTP.Error.invalidJSON }
-                                // The signature format is ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] )
-                                let verificationData = (userX25519PublicKey + serverHashes.joined(separator: "") + hashes.joined(separator: "")).data(using: String.Encoding.utf8)!
-                                let isValid = sodium.sign.verify(message: Bytes(verificationData), publicKey: Bytes(Data(hex: snodePublicKey)), signature: Bytes(Data(base64Encoded: signature)!))
-                                result[snodePublicKey] = isValid
-                            } else {
-                                if let reason = json["reason"] as? String, let statusCode = json["code"] as? String {
-                                    SNLog("Couldn't delete data from: \(snodePublicKey) due to error: \(reason) (\(statusCode)).")
+                    invoke(.deleteMessage, on: snode, associatedWith: publicKey, parameters: parameters)
+                        .map2 { responseData -> [String: Bool] in
+                            guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+                                throw HTTP.Error.invalidJSON
+                            }
+                            guard let swarm = responseJson["swarm"] as? JSON else { throw HTTP.Error.invalidJSON }
+                            
+                            var result: [String: Bool] = [:]
+                            for (snodePublicKey, rawJSON) in swarm {
+                                guard let json = rawJSON as? JSON else { throw HTTP.Error.invalidJSON }
+                                let isFailed = json["failed"] as? Bool ?? false
+                                if !isFailed {
+                                    guard let hashes = json["deleted"] as? [String], let signature = json["signature"] as? String else { throw HTTP.Error.invalidJSON }
+                                    // The signature format is ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] )
+                                    let verificationData = (userX25519PublicKey + serverHashes.joined(separator: "") + hashes.joined(separator: "")).data(using: String.Encoding.utf8)!
+                                    let isValid = sodium.sign.verify(message: Bytes(verificationData), publicKey: Bytes(Data(hex: snodePublicKey)), signature: Bytes(Data(base64Encoded: signature)!))
+                                    result[snodePublicKey] = isValid
                                 } else {
-                                    SNLog("Couldn't delete data from: \(snodePublicKey).")
+                                    if let reason = json["reason"] as? String, let statusCode = json["code"] as? String {
+                                        SNLog("Couldn't delete data from: \(snodePublicKey) due to error: \(reason) (\(statusCode)).")
+                                    } else {
+                                        SNLog("Couldn't delete data from: \(snodePublicKey).")
+                                    }
+                                    result[snodePublicKey] = false
                                 }
-                                result[snodePublicKey] = false
                             }
+                            return result
                         }
-                        return result
-                    }
                 }
             }
         }
@@ -532,29 +530,36 @@ public final class SnodeAPI : NSObject {
                             "signature" : signature.toBase64()
                         ]
                         return attempt(maxRetryCount: maxRetryCount, recoveringOn: Threading.workQueue) {
-                            invoke(.clearAllData, on: snode, parameters: parameters).map2 { rawResponse -> [String:Bool] in
-                                guard let json = rawResponse as? JSON, let swarm = json["swarm"] as? JSON else { throw HTTP.Error.invalidJSON }
-                                var result: [String:Bool] = [:]
-                                for (snodePublicKey, rawJSON) in swarm {
-                                    guard let json = rawJSON as? JSON else { throw HTTP.Error.invalidJSON }
-                                    let isFailed = json["failed"] as? Bool ?? false
-                                    if !isFailed {
-                                        guard let hashes = json["deleted"] as? [String], let signature = json["signature"] as? String else { throw HTTP.Error.invalidJSON }
-                                        // The signature format is ( PUBKEY_HEX || TIMESTAMP || DELETEDHASH[0] || ... || DELETEDHASH[N] )
-                                        let verificationData = (userX25519PublicKey + String(timestamp) + hashes.joined(separator: "")).data(using: String.Encoding.utf8)!
-                                        let isValid = sodium.sign.verify(message: Bytes(verificationData), publicKey: Bytes(Data(hex: snodePublicKey)), signature: Bytes(Data(base64Encoded: signature)!))
-                                        result[snodePublicKey] = isValid
-                                    } else {
-                                        if let reason = json["reason"] as? String, let statusCode = json["code"] as? String {
-                                            SNLog("Couldn't delete data from: \(snodePublicKey) due to error: \(reason) (\(statusCode)).")
+                            invoke(.clearAllData, on: snode, parameters: parameters)
+                                .map2 { responseData -> [String: Bool] in
+                                    guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+                                        throw HTTP.Error.invalidJSON
+                                    }
+                                    guard let swarm = responseJson["swarm"] as? JSON else { throw HTTP.Error.invalidJSON }
+                                    
+                                    var result: [String: Bool] = [:]
+                                    
+                                    for (snodePublicKey, rawJSON) in swarm {
+                                        guard let json = rawJSON as? JSON else { throw HTTP.Error.invalidJSON }
+                                        let isFailed = json["failed"] as? Bool ?? false
+                                        if !isFailed {
+                                            guard let hashes = json["deleted"] as? [String], let signature = json["signature"] as? String else { throw HTTP.Error.invalidJSON }
+                                            // The signature format is ( PUBKEY_HEX || TIMESTAMP || DELETEDHASH[0] || ... || DELETEDHASH[N] )
+                                            let verificationData = (userX25519PublicKey + String(timestamp) + hashes.joined(separator: "")).data(using: String.Encoding.utf8)!
+                                            let isValid = sodium.sign.verify(message: Bytes(verificationData), publicKey: Bytes(Data(hex: snodePublicKey)), signature: Bytes(Data(base64Encoded: signature)!))
+                                            result[snodePublicKey] = isValid
                                         } else {
-                                            SNLog("Couldn't delete data from: \(snodePublicKey).")
+                                            if let reason = json["reason"] as? String, let statusCode = json["code"] as? String {
+                                                SNLog("Couldn't delete data from: \(snodePublicKey) due to error: \(reason) (\(statusCode)).")
+                                            } else {
+                                                SNLog("Couldn't delete data from: \(snodePublicKey).")
+                                            }
+                                            result[snodePublicKey] = false
                                         }
-                                        result[snodePublicKey] = false
                                     }
+                                    
+                                    return result
                                 }
-                                return result
-                            }
                         }
                     }
                 }
@@ -566,9 +571,13 @@ public final class SnodeAPI : NSObject {
     
     // The parsing utilities below use a best attempt approach to parsing; they warn for parsing failures but don't throw exceptions.
 
-    private static func parseSnodes(from rawResponse: Any) -> Set<Snode> {
-        guard let json = rawResponse as? JSON, let rawSnodes = json["snodes"] as? [JSON] else {
-            SNLog("Failed to parse snodes from: \(rawResponse).")
+    private static func parseSnodes(from responseData: Data) -> Set<Snode> {
+        guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+            SNLog("Failed to parse snodes from response data.")
+            return []
+        }
+        guard let rawSnodes = responseJson["snodes"] as? [JSON] else {
+            SNLog("Failed to parse snodes from: \(responseJson).")
             return []
         }
         return Set(rawSnodes.compactMap { rawSnode in
@@ -581,8 +590,11 @@ public final class SnodeAPI : NSObject {
         })
     }
 
-    public static func parseRawMessagesResponse(_ rawResponse: Any, from snode: Snode, associatedWith publicKey: String) -> [JSON] {
-        guard let json = rawResponse as? JSON, let rawMessages = json["messages"] as? [JSON] else { return [] }
+    public static func parseRawMessagesResponse(_ responseData: Data, from snode: Snode, associatedWith publicKey: String) -> [JSON] {
+        guard let responseJson: JSON = try? JSONSerialization.jsonObject(with: responseData, options: [ .fragmentsAllowed ]) as? JSON else {
+            return []
+        }
+        guard let rawMessages = responseJson["messages"] as? [JSON] else { return [] }
         updateLastMessageHashValueIfPossible(for: snode, associatedWith: publicKey, from: rawMessages)
         return removeDuplicates(from: rawMessages, associatedWith: publicKey)
     }
@@ -622,7 +634,7 @@ public final class SnodeAPI : NSObject {
     // MARK: Error Handling
     /// - Note: Should only be invoked from `Threading.workQueue` to avoid race conditions.
     @discardableResult
-    internal static func handleError(withStatusCode statusCode: UInt, json: JSON?, forSnode snode: Snode, associatedWith publicKey: String? = nil) -> Error? {
+    internal static func handleError(withStatusCode statusCode: UInt, data: Data?, forSnode snode: Snode, associatedWith publicKey: String? = nil) -> Error? {
         #if DEBUG
         dispatchPrecondition(condition: .onQueue(Threading.workQueue))
         #endif
@@ -641,37 +653,47 @@ public final class SnodeAPI : NSObject {
                 SnodeAPI.snodeFailureCount[snode] = 0
             }
         }
+        
         switch statusCode {
-        case 500, 502, 503:
-            // The snode is unreachable
-            handleBadSnode()
-        case 406:
-            SNLog("The user's clock is out of sync with the service node network.")
-            return Error.clockOutOfSync
-        case 421:
-            // The snode isn't associated with the given public key anymore
-            if let publicKey = publicKey {
-                func invalidateSwarm() {
-                    SNLog("Invalidating swarm for: \(publicKey).")
-                    SnodeAPI.dropSnodeFromSwarmIfNeeded(snode, publicKey: publicKey)
-                }
-                if let json = json {
-                    let snodes = parseSnodes(from: json)
-                    if !snodes.isEmpty {
-                        setSwarm(to: snodes, for: publicKey)
-                    } else {
+            case 500, 502, 503:
+                // The snode is unreachable
+                handleBadSnode()
+                
+            case 406:
+                SNLog("The user's clock is out of sync with the service node network.")
+                return Error.clockOutOfSync
+                
+            case 421:
+                // The snode isn't associated with the given public key anymore
+                if let publicKey = publicKey {
+                    func invalidateSwarm() {
+                        SNLog("Invalidating swarm for: \(publicKey).")
+                        SnodeAPI.dropSnodeFromSwarmIfNeeded(snode, publicKey: publicKey)
+                    }
+                    
+                    if let data: Data = data {
+                        let snodes = parseSnodes(from: data)
+                        
+                        if !snodes.isEmpty {
+                            setSwarm(to: snodes, for: publicKey)
+                        }
+                        else {
+                            invalidateSwarm()
+                        }
+                    }
+                    else {
                         invalidateSwarm()
                     }
-                } else {
-                    invalidateSwarm()
                 }
-            } else {
-                SNLog("Got a 421 without an associated public key.")
-            }
-        default:
-            handleBadSnode()
-            SNLog("Unhandled response code: \(statusCode).")
+                else {
+                    SNLog("Got a 421 without an associated public key.")
+                }
+                
+            default:
+                handleBadSnode()
+                SNLog("Unhandled response code: \(statusCode).")
         }
+        
         return nil
     }
 }

SessionUtilitiesKit/Networking/HTTP.swift

@@ -67,7 +67,8 @@ public enum HTTP {
         }
     }
 
-    // MARK: Verb
+    // MARK: - Verb
+    
     public enum Verb: String, Codable {
         case get = "GET"
         case put = "PUT"
@@ -75,92 +76,47 @@ public enum HTTP {
         case delete = "DELETE"
     }
 
-    // MARK: Error
+    // MARK: - Error
+    
     public enum Error : LocalizedError {
         case generic
-        case httpRequestFailed(statusCode: UInt, json: JSON?)
+        case httpRequestFailed(statusCode: UInt, data: Data?)
         case invalidJSON
         case invalidResponse
 
         public var errorDescription: String? {
             switch self {
-            case .generic: return "An error occurred."
-            case .httpRequestFailed(let statusCode, _): return "HTTP request failed with status code: \(statusCode)."
-            case .invalidJSON: return "Invalid JSON."
-            case .invalidResponse: return "Invalid Response"
+                case .generic: return "An error occurred."
+                case .httpRequestFailed(let statusCode, _): return "HTTP request failed with status code: \(statusCode)."
+                case .invalidJSON: return "Invalid JSON."
+                case .invalidResponse: return "Invalid Response"
             }
         }
     }
 
-    // MARK: Main
-    public static func execute(_ verb: Verb, _ url: String, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<JSON> {
+    // MARK: - Main
+    
+    public static func execute(_ verb: Verb, _ url: String, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<Data> {
         return execute(verb, url, body: nil, timeout: timeout, useSeedNodeURLSession: useSeedNodeURLSession)
     }
 
-    public static func execute(_ verb: Verb, _ url: String, parameters: JSON?, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<JSON> {
+    public static func execute(_ verb: Verb, _ url: String, parameters: JSON?, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<Data> {
         if let parameters = parameters {
             do {
                 guard JSONSerialization.isValidJSONObject(parameters) else { return Promise(error: Error.invalidJSON) }
                 let body = try JSONSerialization.data(withJSONObject: parameters, options: [ .fragmentsAllowed ])
                 return execute(verb, url, body: body, timeout: timeout, useSeedNodeURLSession: useSeedNodeURLSession)
-            } catch (let error) {
+            }
+            catch (let error) {
                 return Promise(error: error)
             }
-        } else {
+        }
+        else {
             return execute(verb, url, body: nil, timeout: timeout, useSeedNodeURLSession: useSeedNodeURLSession)
         }
     }
 
-    public static func execute(_ verb: Verb, _ url: String, body: Data?, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<JSON> {
-        var request = URLRequest(url: URL(string: url)!)
-        request.httpMethod = verb.rawValue
-        request.httpBody = body
-        request.timeoutInterval = timeout
-        request.allHTTPHeaderFields?.removeValue(forKey: "User-Agent")
-        request.setValue("WhatsApp", forHTTPHeaderField: "User-Agent") // Set a fake value
-        request.setValue("en-us", forHTTPHeaderField: "Accept-Language") // Set a fake value
-        let (promise, seal) = Promise<JSON>.pending()
-        let urlSession = useSeedNodeURLSession ? seedNodeURLSession : snodeURLSession
-        let task = urlSession.dataTask(with: request) { data, response, error in
-            guard let data = data, let response = response as? HTTPURLResponse else {
-                if let error = error {
-                    SNLog("\(verb.rawValue) request to \(url) failed due to error: \(error).")
-                } else {
-                    SNLog("\(verb.rawValue) request to \(url) failed.")
-                }
-                // Override the actual error so that we can correctly catch failed requests in sendOnionRequest(invoking:on:with:)
-                return seal.reject(Error.httpRequestFailed(statusCode: 0, json: nil))
-            }
-            if let error = error {
-                SNLog("\(verb.rawValue) request to \(url) failed due to error: \(error).")
-                // Override the actual error so that we can correctly catch failed requests in sendOnionRequest(invoking:on:with:)
-                return seal.reject(Error.httpRequestFailed(statusCode: 0, json: nil))
-            }
-            let statusCode = UInt(response.statusCode)
-            var json: JSON? = nil
-            if let j = try? JSONSerialization.jsonObject(with: data, options: [ .fragmentsAllowed ]) as? JSON {
-                json = j
-            } else if let result = String(data: data, encoding: .utf8) {
-                json = [ "result" : result ]
-            }
-            guard 200...299 ~= statusCode else {
-                let jsonDescription = json?.prettifiedDescription ?? "no debugging info provided"
-                SNLog("\(verb.rawValue) request to \(url) failed with status code: \(statusCode) (\(jsonDescription)).")
-                return seal.reject(Error.httpRequestFailed(statusCode: statusCode, json: json))
-            }
-            if let json = json {
-                seal.fulfill(json)
-            } else {
-                SNLog("Couldn't parse JSON returned by \(verb.rawValue) request to \(url).")
-                return seal.reject(Error.invalidJSON)
-            }
-        }
-        task.resume()
-        return promise
-    }
-    
-    // TODO: Consilidate the above and this method
-    public static func updatedExecute(_ verb: Verb, _ url: String, body: Data?, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<Data> {
+    public static func execute(_ verb: Verb, _ url: String, body: Data?, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<Data> {
         var request = URLRequest(url: URL(string: url)!)
         request.httpMethod = verb.rawValue
         request.httpBody = body
@@ -178,21 +134,27 @@ public enum HTTP {
                     SNLog("\(verb.rawValue) request to \(url) failed.")
                 }
                 // Override the actual error so that we can correctly catch failed requests in sendOnionRequest(invoking:on:with:)
-                return seal.reject(Error.httpRequestFailed(statusCode: 0, json: nil))
+                return seal.reject(Error.httpRequestFailed(statusCode: 0, data: nil))
             }
             if let error = error {
                 SNLog("\(verb.rawValue) request to \(url) failed due to error: \(error).")
                 // Override the actual error so that we can correctly catch failed requests in sendOnionRequest(invoking:on:with:)
-                return seal.reject(Error.httpRequestFailed(statusCode: 0, json: nil))
+                return seal.reject(Error.httpRequestFailed(statusCode: 0, data: data))
             }
             let statusCode = UInt(response.statusCode)
 
             guard 200...299 ~= statusCode else {
-//                let jsonDescription = json?.prettifiedDescription ?? "no debugging info provided"
-//                SNLog("\(verb.rawValue) request to \(url) failed with status code: \(statusCode) (\(jsonDescription)).")
-//                return seal.reject(Error.httpRequestFailed(statusCode: statusCode, json: json))
-                // TODO: Provide error from backend here
-                return seal.reject(Error.httpRequestFailed(statusCode: statusCode, json: [:]))
+                var json: JSON? = nil
+                if let processedJson: JSON = try? JSONSerialization.jsonObject(with: data, options: [ .fragmentsAllowed ]) as? JSON {
+                    json = processedJson
+                }
+                else if let result: String = String(data: data, encoding: .utf8) {
+                    json = [ "result": result ]
+                }
+                
+                let jsonDescription: String = (json?.prettifiedDescription ?? "no debugging info provided")
+                SNLog("\(verb.rawValue) request to \(url) failed with status code: \(statusCode) (\(jsonDescription)).")
+                return seal.reject(Error.httpRequestFailed(statusCode: statusCode, data: data))
             }
             
             seal.fulfill(data)