From 82a71f23851a9c71ca06065f584203c5109bcce5 Mon Sep 17 00:00:00 2001 From: gmbnt Date: Mon, 6 Apr 2020 13:26:26 +1000 Subject: [PATCH] Implement onion request error handling --- SignalServiceKit/src/Loki/API/HTTP/HTTP.swift | 6 ++- .../src/Loki/API/LokiAPI+SwarmAPI.swift | 2 +- SignalServiceKit/src/Loki/API/LokiAPI.swift | 2 +- .../API/Onion Requests/OnionRequestAPI.swift | 52 +++++++++++++++++-- .../Onion Requests/OnionRequestAPITests.swift | 2 +- 5 files changed, 56 insertions(+), 8 deletions(-) diff --git a/SignalServiceKit/src/Loki/API/HTTP/HTTP.swift b/SignalServiceKit/src/Loki/API/HTTP/HTTP.swift index 5c143343a..18475b278 100644 --- a/SignalServiceKit/src/Loki/API/HTTP/HTTP.swift +++ b/SignalServiceKit/src/Loki/API/HTTP/HTTP.swift @@ -60,11 +60,13 @@ internal enum HTTP { } else { print("[Loki] \(verb.rawValue) request to \(url) failed.") } - return seal.reject(error ?? Error.generic) + // Override the actual error so that we can correctly catch failed requests in sendOnionRequest(invoking:on:with:) + return seal.reject(Error.httpRequestFailed(statusCode: 0, json: nil)) } if let error = error { print("[Loki] \(verb.rawValue) request to \(url) failed due to error: \(error).") - return seal.reject(error) + // Override the actual error so that we can correctly catch failed requests in sendOnionRequest(invoking:on:with:) + return seal.reject(Error.httpRequestFailed(statusCode: 0, json: nil)) } let statusCode = UInt(response.statusCode) var json: JSON? = nil diff --git a/SignalServiceKit/src/Loki/API/LokiAPI+SwarmAPI.swift b/SignalServiceKit/src/Loki/API/LokiAPI+SwarmAPI.swift index fda68accf..23743e78c 100644 --- a/SignalServiceKit/src/Loki/API/LokiAPI+SwarmAPI.swift +++ b/SignalServiceKit/src/Loki/API/LokiAPI+SwarmAPI.swift @@ -10,7 +10,7 @@ public extension LokiAPI { private static let minimumSnodeCount = 2 private static let targetSnodeCount = 3 - fileprivate static let failureThreshold = 2 + internal static let failureThreshold = 2 // MARK: Caching internal static var swarmCache: [String:[LokiAPITarget]] = [:] diff --git a/SignalServiceKit/src/Loki/API/LokiAPI.swift b/SignalServiceKit/src/Loki/API/LokiAPI.swift index 5873d8114..028658fd3 100644 --- a/SignalServiceKit/src/Loki/API/LokiAPI.swift +++ b/SignalServiceKit/src/Loki/API/LokiAPI.swift @@ -101,7 +101,7 @@ public final class LokiAPI : NSObject { if let headers = headers { request.allHTTPHeaderFields = headers } request.timeoutInterval = timeout ?? defaultTimeout if useOnionRequests { - return OnionRequestAPI.sendOnionRequest(invoking: method, on: target, with: parameters).map { $0 as Any } + return OnionRequestAPI.sendOnionRequest(invoking: method, on: target, with: parameters, associatedWith: hexEncodedPublicKey).map { $0 as Any } } else { return TSNetworkManager.shared().perform(request, withCompletionQueue: DispatchQueue.global()) .map { $0.responseObject } diff --git a/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPI.swift b/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPI.swift index 1e8ae4e97..bd338941a 100644 --- a/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPI.swift +++ b/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPI.swift @@ -24,6 +24,7 @@ internal enum OnionRequestAPI { // MARK: Error internal enum Error : LocalizedError { + case httpRequestFailedAtTargetSnode(statusCode: UInt, json: JSON) case insufficientSnodes case missingSnodeVersion case randomDataGenerationFailed @@ -32,6 +33,7 @@ internal enum OnionRequestAPI { var errorDescription: String? { switch self { + case .httpRequestFailedAtTargetSnode(let statusCode): return "HTTP request failed at target snode with status code: \(statusCode)." case .insufficientSnodes: return "Couldn't find enough snodes to build a path." case .missingSnodeVersion: return "Missing snode version." case .randomDataGenerationFailed: return "Couldn't generate random data." @@ -178,12 +180,13 @@ internal enum OnionRequestAPI { // MARK: Internal API /// Sends an onion request to `snode`. Builds new paths as needed. - internal static func sendOnionRequest(invoking method: LokiAPITarget.Method, on snode: LokiAPITarget, with parameters: JSON) -> Promise { + internal static func sendOnionRequest(invoking method: LokiAPITarget.Method, on snode: LokiAPITarget, with parameters: JSON, associatedWith hexEncodedPublicKey: String) -> Promise { let (promise, seal) = Promise.pending() + var guardSnode: LokiAPITarget! workQueue.async { let payload: JSON = [ "method" : method.rawValue, "params" : parameters ] buildOnion(around: payload, targetedAt: snode).done(on: workQueue) { intermediate in - let guardSnode = intermediate.guardSnode + guardSnode = intermediate.guardSnode let url = "\(guardSnode.address):\(guardSnode.port)/onion_req" let finalEncryptionResult = intermediate.finalEncryptionResult let onion = finalEncryptionResult.ciphertext @@ -203,7 +206,9 @@ internal enum OnionRequestAPI { let data = Data(try aes.decrypt(ciphertext.bytes)) guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? JSON, let bodyAsString = json["body"] as? String, let bodyAsData = bodyAsString.data(using: .utf8), - let body = try JSONSerialization.jsonObject(with: bodyAsData, options: []) as? JSON else { return seal.reject(HTTP.Error.invalidJSON) } + let body = try JSONSerialization.jsonObject(with: bodyAsData, options: []) as? JSON, + let statusCode = json["status"] as? Int else { return seal.reject(HTTP.Error.invalidJSON) } + guard 200...299 ~= statusCode else { return seal.reject(Error.httpRequestFailedAtTargetSnode(statusCode: UInt(statusCode), json: body)) } seal.fulfill(body) } catch (let error) { seal.reject(error) @@ -215,6 +220,47 @@ internal enum OnionRequestAPI { seal.reject(error) } } + promise.catch(on: workQueue) { error in // Must be invoked on workQueue + guard case HTTP.Error.httpRequestFailed(_, _) = error else { return } + dropPath(containing: guardSnode) // A snode in the path is bad; retry with a different path + } + promise.recover(on: LokiAPI.errorHandlingQueue) { error -> Promise in // Must be invoked on LokiAPI.errorHandlingQueue + // The code below is very similar to that in LokiAPI.handlingSnodeErrorsIfNeeded(for:associatedWith:), but unfortunately slightly + // different due to the fact that OnionRequestAPI uses the newer HTTP API, whereas LokiAPI still uses TSNetworkManager + guard case Error.httpRequestFailedAtTargetSnode(let statusCode, let json) = error else { throw error } + switch statusCode { + case 0, 400, 500, 503: + // The snode is unreachable + let oldFailureCount = LokiAPI.failureCount[snode] ?? 0 + let newFailureCount = oldFailureCount + 1 + LokiAPI.failureCount[snode] = newFailureCount + print("[Loki] Couldn't reach snode at: \(snode); setting failure count to \(newFailureCount).") + if newFailureCount >= LokiAPI.failureThreshold { + print("[Loki] Failure threshold reached for: \(snode); dropping it.") + LokiAPI.dropIfNeeded(snode, hexEncodedPublicKey: hexEncodedPublicKey) // Remove it from the swarm cache associated with the given public key + LokiAPI.randomSnodePool.remove(snode) // Remove it from the random snode pool + LokiAPI.failureCount[snode] = 0 + } + case 406: + print("[Loki] The user's clock is out of sync with the service node network.") + throw LokiAPI.LokiAPIError.clockOutOfSync + case 421: + // The snode isn't associated with the given public key anymore + print("[Loki] Invalidating swarm for: \(hexEncodedPublicKey).") + LokiAPI.dropIfNeeded(snode, hexEncodedPublicKey: hexEncodedPublicKey) + case 432: + // The proof of work difficulty is too low + if let powDifficulty = json["difficulty"] as? Int { + print("[Loki] Setting proof of work difficulty to \(powDifficulty).") + LokiAPI.powDifficulty = UInt(powDifficulty) + } else { + print("[Loki] Failed to update proof of work difficulty.") + } + break + default: break + } + throw error + } return promise } } diff --git a/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPITests.swift b/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPITests.swift index 72761b1a0..652b34f4e 100644 --- a/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPITests.swift +++ b/SignalServiceKit/src/Loki/API/Onion Requests/OnionRequestAPITests.swift @@ -18,7 +18,7 @@ class OnionRequestAPITests : XCTestCase { let mockSessionID = "0582bc30f11e8a9736407adcaca03b049f4acd4af3ae7eb6b6608d30f0b1e6a20e" let parameters: JSON = [ "pubKey" : mockSessionID ] let (promise, seal) = Promise.pending() - OnionRequestAPI.invoke(.getSwarm, on: snode, with: parameters).done(on: OnionRequestAPI.workQueue) { json in + OnionRequestAPI.sendOnionRequest(invoking: .getSwarm, on: snode, with: parameters).done(on: OnionRequestAPI.workQueue) { json in successCount += 1 print("[Loki] [Onion Request API] Onion request succeeded with result: \(json.prettifiedDescription).") seal.fulfill(())