From 482f169df1f5fde73624624de86504f92ccd83a1 Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 02:46:42 +0930 Subject: [PATCH] Refactor SnodeApi --- .../messaging/jobs/ConfigurationSyncJob.kt | 1 - .../org/session/libsession/snode/SnodeAPI.kt | 728 +++++++----------- .../session/libsignal/utilities/Base64.java | 4 +- 3 files changed, 269 insertions(+), 464 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt b/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt index 4a3299d197..a9f076b106 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt @@ -63,7 +63,6 @@ data class ConfigurationSyncJob(val destination: Destination): Job { // return a list of batch request objects val snodeMessage = MessageSender.buildConfigMessageToSnode(destination.destinationPublicKey(), message) val authenticated = SnodeAPI.buildAuthenticatedStoreBatchInfo( - destination.destinationPublicKey(), config.configNamespace(), snodeMessage ) ?: return@map null // this entry will be null otherwise diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 9ceefe8386..8e19234b0d 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -9,9 +9,9 @@ import com.goterl.lazysodium.interfaces.PwHash import com.goterl.lazysodium.interfaces.SecretBox import com.goterl.lazysodium.interfaces.Sign import com.goterl.lazysodium.utils.Key +import com.goterl.lazysodium.utils.KeyPair import nl.komponents.kovenant.Promise import nl.komponents.kovenant.all -import nl.komponents.kovenant.deferred import nl.komponents.kovenant.functional.bind import nl.komponents.kovenant.functional.map import nl.komponents.kovenant.task @@ -30,7 +30,6 @@ import org.session.libsignal.utilities.JsonUtil import org.session.libsignal.utilities.Log import org.session.libsignal.utilities.Namespace import org.session.libsignal.utilities.Snode -import org.session.libsignal.utilities.ThreadUtils import org.session.libsignal.utilities.prettifiedDescription import org.session.libsignal.utilities.retryIfNeeded import java.security.SecureRandom @@ -46,7 +45,7 @@ object SnodeAPI { private val broadcaster: Broadcaster get() = SnodeModule.shared.broadcaster - internal var snodeFailureCount: MutableMap = mutableMapOf() + private var snodeFailureCount: MutableMap = mutableMapOf() internal var snodePool: Set get() = database.getSnodePool() set(newValue) { database.setSnodePool(newValue) } @@ -57,7 +56,7 @@ object SnodeAPI { internal var clockOffset = 0L @JvmStatic - public val nowWithOffset + val nowWithOffset get() = System.currentTimeMillis() + clockOffset internal var forkInfo by observable(database.getForkInfo()) { _, oldValue, newValue -> @@ -68,32 +67,32 @@ object SnodeAPI { } // Settings - private val maxRetryCount = 6 - private val minimumSnodePoolCount = 12 - private val minimumSwarmSnodeCount = 3 + private const val maxRetryCount = 6 + private const val minimumSnodePoolCount = 12 + private const val minimumSwarmSnodeCount = 3 // Use port 4433 if the API level can handle the network security configuration and enforce pinned certificates private val seedNodePort = if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) 443 else 4443 - private val seedNodePool by lazy { - if (useTestnet) { - setOf( "http://public.loki.foundation:38157" ) - } else { - setOf( - "https://seed1.getsession.org:$seedNodePort", - "https://seed2.getsession.org:$seedNodePort", - "https://seed3.getsession.org:$seedNodePort", - ) - } - } + private const val snodeFailureThreshold = 3 private const val useOnionRequests = true - const val useTestnet = false + private const val useTestnet = false + + private val seedNodePool = if (useTestnet) { + setOf( "http://public.loki.foundation:38157" ) + } else { + setOf( + "https://seed1.getsession.org:$seedNodePort", + "https://seed2.getsession.org:$seedNodePort", + "https://seed3.getsession.org:$seedNodePort", + ) + } - const val KEY_IP = "public_ip" - const val KEY_PORT = "storage_port" - const val KEY_X25519 = "pubkey_x25519" - const val KEY_ED25519 = "pubkey_ed25519" - const val KEY_VERSION = "storage_server_version" + private const val KEY_IP = "public_ip" + private const val KEY_PORT = "storage_port" + private const val KEY_X25519 = "pubkey_x25519" + private const val KEY_ED25519 = "pubkey_ed25519" + private const val KEY_VERSION = "storage_server_version" // Error sealed class Error(val description: String) : Exception(description) { @@ -122,39 +121,28 @@ object SnodeAPI { parameters: Map, publicKey: String? = null, version: Version = Version.V3 - ): RawResponsePromise { - val url = "${snode.address}:${snode.port}/storage_rpc/v1" - val deferred = deferred, Exception>() - if (useOnionRequests) { - OnionRequestAPI.sendOnionRequest(method, parameters, snode, version, publicKey).map { - val body = it.body ?: throw Error.Generic - deferred.resolve(JsonUtil.fromJson(body, Map::class.java)) - }.fail { deferred.reject(it) } - } else { - ThreadUtils.queue { - val payload = mapOf( "method" to method.rawValue, "params" to parameters ) - try { - val response = HTTP.execute(HTTP.Verb.POST, url, payload).toString() - val json = JsonUtil.fromJson(response, Map::class.java) - deferred.resolve(json) - } catch (exception: Exception) { - val httpRequestFailedException = exception as? HTTP.HTTPRequestFailedException - if (httpRequestFailedException != null) { - val error = handleSnodeError(httpRequestFailedException.statusCode, httpRequestFailedException.json, snode, publicKey) - if (error != null) { return@queue deferred.reject(exception) } - } - Log.d("Loki", "Unhandled exception: $exception.") - deferred.reject(exception) + ): RawResponsePromise = if (useOnionRequests) OnionRequestAPI.sendOnionRequest(method, parameters, snode, version, publicKey).map { + val body = it.body ?: throw Error.Generic + JsonUtil.fromJson(body, Map::class.java) + } else task { + val payload = mapOf( "method" to method.rawValue, "params" to parameters ) + try { + val url = "${snode.address}:${snode.port}/storage_rpc/v1" + val response = HTTP.execute(HTTP.Verb.POST, url, payload).toString() + JsonUtil.fromJson(response, Map::class.java) + } catch (exception: Exception) { + (exception as? HTTP.HTTPRequestFailedException)?.run { + handleSnodeError(statusCode, json, snode, publicKey) + // TODO Check if we meant to throw the error returned by handleSnodeError + throw exception } + Log.d("Loki", "Unhandled exception: $exception.") + throw exception } } - return deferred.promise - } - internal fun getRandomSnode(): Promise { - val snodePool = this.snodePool - - if (snodePool.count() < minimumSnodePoolCount) { + internal fun getRandomSnode(): Promise = + snodePool.takeIf { it.size >= minimumSnodePoolCount }?.let { Promise.of(it.getRandomElement()) } ?: task { val target = seedNodePool.random() val url = "$target/json_rpc" Log.d("Loki", "Populating snode pool using: $target.") @@ -169,73 +157,48 @@ object SnodeAPI { ) ) ) - val deferred = deferred() - deferred() - ThreadUtils.queue { - try { - val response = HTTP.execute(HTTP.Verb.POST, url, parameters, useSeedNodeConnection = true) - val json = try { - JsonUtil.fromJson(response, Map::class.java) - } catch (exception: Exception) { - mapOf( "result" to response.toString()) - } - val intermediate = json["result"] as? Map<*, *> - val rawSnodes = intermediate?.get("service_node_states") as? List<*> - if (rawSnodes != null) { - val snodePool = rawSnodes.mapNotNull { rawSnode -> - val rawSnodeAsJSON = rawSnode as? Map<*, *> - val address = rawSnodeAsJSON?.get(KEY_IP) as? String - val port = rawSnodeAsJSON?.get(KEY_PORT) as? Int - val ed25519Key = rawSnodeAsJSON?.get(KEY_ED25519) as? String - val x25519Key = rawSnodeAsJSON?.get(KEY_X25519) as? String - val version = (rawSnodeAsJSON?.get(KEY_VERSION) as? ArrayList<*>) - ?.filterIsInstance() // get the array as Integers - ?.let(Snode::Version) // turn it int a version - - if (address != null && port != null && ed25519Key != null && x25519Key != null - && address != "0.0.0.0" && version != null) { - Snode( - address = "https://$address", - port = port, - publicKeySet = Snode.KeySet(ed25519Key, x25519Key), - version = version - ) - } else { - Log.d("Loki", "Failed to parse: ${rawSnode?.prettifiedDescription()}.") - null - } - }.toMutableSet() - Log.d("Loki", "Persisting snode pool to database.") - this.snodePool = snodePool - try { - deferred.resolve(snodePool.getRandomElement()) - } catch (exception: Exception) { - Log.d("Loki", "Got an empty snode pool from: $target.") - deferred.reject(SnodeAPI.Error.Generic) - } - } else { - Log.d("Loki", "Failed to update snode pool from: ${(rawSnodes as List<*>?)?.prettifiedDescription()}.") - deferred.reject(SnodeAPI.Error.Generic) - } - } catch (exception: Exception) { - deferred.reject(exception) - } + val response = HTTP.execute(HTTP.Verb.POST, url, parameters, useSeedNodeConnection = true) + val json = try { + JsonUtil.fromJson(response, Map::class.java) + } catch (exception: Exception) { + mapOf( "result" to response.toString()) } - return deferred.promise - } else { - return Promise.of(snodePool.getRandomElement()) - } - } - - private fun extractVersionString(jsonVersion: String): String{ - return jsonVersion.removeSurrounding("[", "]").split(", ").joinToString(separator = ".") + val intermediate = json["result"] as? Map<*, *> ?: throw Error.Generic + .also { Log.d("Loki", "Failed to update snode pool, intermediate was null.") } + val rawSnodes = intermediate["service_node_states"] as? List<*> ?: throw Error.Generic + .also { Log.d("Loki", "Failed to update snode pool, rawSnodes was null.") } + + rawSnodes.asSequence().mapNotNull { it as? Map<*, *> }.mapNotNull { rawSnode -> + createSnode( + address = rawSnode[KEY_IP] as? String, + port = rawSnode[KEY_PORT] as? Int, + ed25519Key = rawSnode[KEY_ED25519] as? String, + x25519Key = rawSnode[KEY_X25519] as? String, + version = (rawSnode[KEY_VERSION] as? List<*>) + ?.filterIsInstance() + ?.let(Snode::Version) + ).also { if (it == null) Log.d("Loki", "Failed to parse: ${rawSnode.prettifiedDescription()}.") } + }.toSet().also { + Log.d("Loki", "Persisting snode pool to database.") + this.snodePool = it + }.runCatching { getRandomElement() }.onFailure { + Log.d("Loki", "Got an empty snode pool from: $target.") + throw SnodeAPI.Error.Generic + }.getOrThrow() + } + + private fun createSnode(address: String?, port: Int?, ed25519Key: String?, x25519Key: String?, version: Snode.Version? = Snode.Version.ZERO): Snode? { + return Snode( + address?.takeUnless { it == "0.0.0.0" }?.let { "https://$it" } ?: return null, + port ?: return null, + Snode.KeySet(ed25519Key ?: return null, x25519Key ?: return null), + version ?: return null + ) } internal fun dropSnodeFromSwarmIfNeeded(snode: Snode, publicKey: String) { - val swarm = database.getSwarm(publicKey)?.toMutableSet() - if (swarm != null && swarm.contains(snode)) { - swarm.remove(snode) - database.setSwarm(publicKey, swarm) + database.getSwarm(publicKey)?.takeIf { snode in it }?.let { + database.setSwarm(publicKey, it - snode) } } @@ -246,8 +209,6 @@ object SnodeAPI { // Public API fun getAccountID(onsName: String): Promise { - val deferred = deferred() - val promise = deferred.promise val validationCount = 3 val accountIDByteCount = 33 // Hash the ONS name using BLAKE2b @@ -255,96 +216,79 @@ object SnodeAPI { val nameAsData = onsName.toByteArray() val nameHash = ByteArray(GenericHash.BYTES) if (!sodium.cryptoGenericHash(nameHash, nameHash.size, nameAsData, nameAsData.size.toLong())) { - deferred.reject(Error.HashingFailed) - return promise + throw Error.HashingFailed } val base64EncodedNameHash = Base64.encodeBytes(nameHash) // Ask 3 different snodes for the Account ID associated with the given name hash val parameters = mapOf( - "endpoint" to "ons_resolve", - "params" to mapOf( "type" to 0, "name_hash" to base64EncodedNameHash ) + "endpoint" to "ons_resolve", + "params" to mapOf( "type" to 0, "name_hash" to base64EncodedNameHash ) ) - val promises = (1..validationCount).map { + val promises = List(validationCount) { getRandomSnode().bind { snode -> retryIfNeeded(maxRetryCount) { invoke(Snode.Method.OxenDaemonRPCCall, snode, parameters) } } } - all(promises).success { results -> + return all(promises).map { results -> val accountIDs = mutableListOf() for (json in results) { val intermediate = json["result"] as? Map<*, *> - val hexEncodedCiphertext = intermediate?.get("encrypted_value") as? String - if (hexEncodedCiphertext != null) { - val ciphertext = Hex.fromStringCondensed(hexEncodedCiphertext) - val isArgon2Based = (intermediate["nonce"] == null) - if (isArgon2Based) { - // Handle old Argon2-based encryption used before HF16 - val salt = ByteArray(PwHash.SALTBYTES) - val key: ByteArray - val nonce = ByteArray(SecretBox.NONCEBYTES) - val accountIDAsData = ByteArray(accountIDByteCount) - try { - key = Key.fromHexString(sodium.cryptoPwHash(onsName, SecretBox.KEYBYTES, salt, PwHash.OPSLIMIT_MODERATE, PwHash.MEMLIMIT_MODERATE, PwHash.Alg.PWHASH_ALG_ARGON2ID13)).asBytes - } catch (e: SodiumException) { - deferred.reject(Error.HashingFailed) - return@success - } - if (!sodium.cryptoSecretBoxOpenEasy(accountIDAsData, ciphertext, ciphertext.size.toLong(), nonce, key)) { - deferred.reject(Error.DecryptionFailed) - return@success - } - accountIDs.add(Hex.toStringCondensed(accountIDAsData)) - } else { - val hexEncodedNonce = intermediate["nonce"] as? String - if (hexEncodedNonce == null) { - deferred.reject(Error.Generic) - return@success - } - val nonce = Hex.fromStringCondensed(hexEncodedNonce) - val key = ByteArray(GenericHash.BYTES) - if (!sodium.cryptoGenericHash(key, key.size, nameAsData, nameAsData.size.toLong(), nameHash, nameHash.size)) { - deferred.reject(Error.HashingFailed) - return@success - } - val accountIDAsData = ByteArray(accountIDByteCount) - if (!sodium.cryptoAeadXChaCha20Poly1305IetfDecrypt(accountIDAsData, null, null, ciphertext, ciphertext.size.toLong(), null, 0, nonce, key)) { - deferred.reject(Error.DecryptionFailed) - return@success - } - accountIDs.add(Hex.toStringCondensed(accountIDAsData)) + val hexEncodedCiphertext = intermediate?.get("encrypted_value") as? String ?: throw Error.Generic + val ciphertext = Hex.fromStringCondensed(hexEncodedCiphertext) + val isArgon2Based = (intermediate["nonce"] == null) + if (isArgon2Based) { + // Handle old Argon2-based encryption used before HF16 + val salt = ByteArray(PwHash.SALTBYTES) + val nonce = ByteArray(SecretBox.NONCEBYTES) + val accountIDAsData = ByteArray(accountIDByteCount) + val key = try { + Key.fromHexString(sodium.cryptoPwHash(onsName, SecretBox.KEYBYTES, salt, PwHash.OPSLIMIT_MODERATE, PwHash.MEMLIMIT_MODERATE, PwHash.Alg.PWHASH_ALG_ARGON2ID13)).asBytes + } catch (e: SodiumException) { + throw Error.HashingFailed + } + if (!sodium.cryptoSecretBoxOpenEasy(accountIDAsData, ciphertext, ciphertext.size.toLong(), nonce, key)) { + throw Error.DecryptionFailed } + accountIDs.add(Hex.toStringCondensed(accountIDAsData)) } else { - deferred.reject(Error.Generic) - return@success + val hexEncodedNonce = intermediate["nonce"] as? String ?: throw Error.Generic + val nonce = Hex.fromStringCondensed(hexEncodedNonce) + val key = ByteArray(GenericHash.BYTES) + if (!sodium.cryptoGenericHash(key, key.size, nameAsData, nameAsData.size.toLong(), nameHash, nameHash.size)) { + throw Error.HashingFailed + } + val accountIDAsData = ByteArray(accountIDByteCount) + if (!sodium.cryptoAeadXChaCha20Poly1305IetfDecrypt(accountIDAsData, null, null, ciphertext, ciphertext.size.toLong(), null, 0, nonce, key)) { + throw Error.DecryptionFailed + } + accountIDs.add(Hex.toStringCondensed(accountIDAsData)) } } - if (accountIDs.size == validationCount && accountIDs.toSet().size == 1) { - deferred.resolve(accountIDs.first()) - } else { - deferred.reject(Error.ValidationFailed) - } + accountIDs.takeIf { it.size == validationCount && it.toSet().size == 1 }?.first() + ?: throw Error.ValidationFailed } - return promise } - fun getSwarm(publicKey: String): Promise, Exception> { - val cachedSwarm = database.getSwarm(publicKey) - return if (cachedSwarm != null && cachedSwarm.size >= minimumSwarmSnodeCount) { - val cachedSwarmCopy = mutableSetOf() // Workaround for a Kotlin compiler issue - cachedSwarmCopy.addAll(cachedSwarm) - task { cachedSwarmCopy } - } else { - val parameters = mapOf( "pubKey" to publicKey ) - getRandomSnode().bind { - invoke(Snode.Method.GetSwarm, it, parameters, publicKey) + fun getSwarm(publicKey: String): Promise, Exception> = + database.getSwarm(publicKey)?.takeIf { it.size >= minimumSwarmSnodeCount }?.let(Promise.Companion::of) + ?: getRandomSnode().bind { + invoke(Snode.Method.GetSwarm, it, parameters = mapOf( "pubKey" to publicKey ), publicKey) }.map { parseSnodes(it).toSet() }.success { database.setSwarm(publicKey, it) } - } + + private fun signAndEncode(data: ByteArray, userED25519KeyPair: KeyPair) = sign(data, userED25519KeyPair).let(Base64::encodeBytes) + private fun sign(data: ByteArray, userED25519KeyPair: KeyPair): ByteArray = ByteArray(Sign.BYTES).also { + sodium.cryptoSignDetached( + it, + data, + data.size.toLong(), + userED25519KeyPair.secretKey.asBytes + ) } fun getRawMessages(snode: Snode, publicKey: String, requiresAuth: Boolean = true, namespace: Int = 0): RawResponsePromise { @@ -365,23 +309,19 @@ object SnodeAPI { } val timestamp = System.currentTimeMillis() + clockOffset val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - val verificationData = - if (namespace != 0) "retrieve$namespace$timestamp".toByteArray() - else "retrieve$timestamp".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userED25519KeyPair.secretKey.asBytes - ) + val verificationData = buildString { + append("retrieve") + if (namespace != 0) append(namespace) + append(timestamp) + }.toByteArray() + val signature = try { + signAndEncode(verificationData, userED25519KeyPair) } catch (exception: Exception) { return Promise.ofFail(Error.SigningFailed) } parameters["timestamp"] = timestamp parameters["pubkey_ed25519"] = ed25519PublicKey - parameters["signature"] = Base64.encodeBytes(signature) + parameters["signature"] = signature } // If the namespace is default (0) here it will be implicitly read as 0 on the storage server @@ -394,42 +334,34 @@ object SnodeAPI { return invoke(Snode.Method.Retrieve, snode, parameters, publicKey) } - fun buildAuthenticatedStoreBatchInfo(publicKey: String, namespace: Int, message: SnodeMessage): SnodeBatchRequestInfo? { - val params = mutableMapOf() - // load the message data params into the sub request - // currently loads: - // pubKey - // data - // ttl - // timestamp - params.putAll(message.toJSON()) - params["namespace"] = namespace - + fun buildAuthenticatedStoreBatchInfo(namespace: Int, message: SnodeMessage): SnodeBatchRequestInfo? { // used for sig generation since it is also the value used in timestamp parameter val messageTimestamp = message.timestamp - val userEd25519KeyPair = try { - MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null - } catch (e: Exception) { - return null - } + val userED25519KeyPair = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() ?: return null - val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) val verificationData = "store$namespace$messageTimestamp".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val signature = try { + signAndEncode(verificationData, userED25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) + return null } - // timestamp already set - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) + + val params = buildMap { + // load the message data params into the sub request + // currently loads: + // pubKey + // data + // ttl + // timestamp + putAll(message.toJSON()) + this["namespace"] = namespace + // timestamp already set + this["pubkey_ed25519"] = userED25519KeyPair.publicKey.asHexString + this["signature"] = signature + } + return SnodeBatchRequestInfo( Snode.Method.SendMessage.rawValue, params, @@ -444,32 +376,26 @@ object SnodeAPI { * @param required indicates that *at least one* message in the list is deleted from the server, otherwise it will return 404 */ fun buildAuthenticatedDeleteBatchInfo(publicKey: String, messageHashes: List, required: Boolean = false): SnodeBatchRequestInfo? { - val params = mutableMapOf( - "pubkey" to publicKey, - "required" to required, // could be omitted technically but explicit here - "messages" to messageHashes - ) val userEd25519KeyPair = try { MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null } catch (e: Exception) { return null } val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - val verificationData = "delete${messageHashes.joinToString("")}".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val verificationData = sequenceOf("delete").plus(messageHashes).toByteArray() + val signature = try { + signAndEncode(verificationData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return null } - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) + val params = buildMap { + this["pubkey"] = publicKey + this["required"] = required // could be omitted technically but explicit here + this["messages"] = messageHashes + this["pubkey_ed25519"] = ed25519PublicKey + this["signature"] = signature + } return SnodeBatchRequestInfo( Snode.Method.DeleteMessage.rawValue, params, @@ -479,39 +405,25 @@ object SnodeAPI { fun buildAuthenticatedRetrieveBatchRequest(snode: Snode, publicKey: String, namespace: Int = 0, maxSize: Int? = null): SnodeBatchRequestInfo? { val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: "" - val params = mutableMapOf( - "pubkey" to publicKey, - "last_hash" to lastHashValue, - ) - val userEd25519KeyPair = try { - MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null - } catch (e: Exception) { - return null - } + val userEd25519KeyPair = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() ?: return null val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString val timestamp = System.currentTimeMillis() + clockOffset - val signature = ByteArray(Sign.BYTES) val verificationData = if (namespace == 0) "retrieve$timestamp".toByteArray() else "retrieve$namespace$timestamp".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val signature = try { + signAndEncode(verificationData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return null } - params["timestamp"] = timestamp - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) - if (namespace != 0) { - params["namespace"] = namespace - } - if (maxSize != null) { - params["max_size"] = maxSize + val params = buildMap { + this["pubkey"] = publicKey + this["last_hash"] = lastHashValue + this["timestamp"] = timestamp + this["pubkey_ed25519"] = ed25519PublicKey + this["signature"] = signature + if (namespace != 0) this["namespace"] = namespace + if (maxSize != null) this["max_size"] = maxSize } return SnodeBatchRequestInfo( Snode.Method.Retrieve.rawValue, @@ -535,13 +447,12 @@ object SnodeAPI { } fun getRawBatchResponse(snode: Snode, publicKey: String, requests: List, sequence: Boolean = false): RawResponsePromise { - val parameters = mutableMapOf( - "requests" to requests - ) + val parameters = buildMap { this["requests"] = requests } return invoke(if (sequence) Snode.Method.Sequence else Snode.Method.Batch, snode, parameters, publicKey).success { rawResponses -> - val responseList = (rawResponses["results"] as List) - responseList.forEachIndexed { index, response -> - if (response["code"] as? Int != 200) { + rawResponses["results"].let { it as List } + .asSequence() + .filter { it["code"] as? Int != 200 } + .forEach { response -> Log.w("Loki", "response code was not 200") handleSnodeError( response["code"] as? Int ?: 0, @@ -550,7 +461,6 @@ object SnodeAPI { publicKey ) } - } } } @@ -562,14 +472,8 @@ object SnodeAPI { val signData = "${Snode.Method.GetExpiries.rawValue}$timestamp${hashes.joinToString(separator = "")}".toByteArray() val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - try { - sodium.cryptoSignDetached( - signature, - signData, - signData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val signature = try { + signAndEncode(signData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return@retryIfNeeded Promise.ofFail(e) @@ -579,7 +483,7 @@ object SnodeAPI { "messages" to hashes, "timestamp" to timestamp, "pubkey_ed25519" to ed25519PublicKey, - "signature" to Base64.encodeBytes(signature) + "signature" to signature ) getSingleTargetSnode(publicKey) bind { snode -> invoke(Snode.Method.GetExpiries, snode, params, publicKey) @@ -587,8 +491,8 @@ object SnodeAPI { } } - fun alterTtl(messageHashes: List, newExpiry: Long, publicKey: String, extend: Boolean = false, shorten: Boolean = false): RawResponsePromise { - return retryIfNeeded(maxRetryCount) { + fun alterTtl(messageHashes: List, newExpiry: Long, publicKey: String, extend: Boolean = false, shorten: Boolean = false): RawResponsePromise = + retryIfNeeded(maxRetryCount) { val params = buildAlterTtlParams(messageHashes, newExpiry, publicKey, extend, shorten) ?: return@retryIfNeeded Promise.ofFail( Exception("Couldn't build signed params for alterTtl request for newExpiry=$newExpiry, extend=$extend, shorten=$shorten") @@ -597,111 +501,96 @@ object SnodeAPI { invoke(Snode.Method.Expire, snode, params, publicKey) } } - } private fun buildAlterTtlParams( // TODO: in future this will probably need to use the closed group subkeys / admin keys for group swarms messageHashes: List, newExpiry: Long, publicKey: String, extend: Boolean = false, - shorten: Boolean = false): Map? { + shorten: Boolean = false + ): Map? { val userEd25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null - val params = mutableMapOf( - "expiry" to newExpiry, - "messages" to messageHashes, - ) - if (extend) { - params["extend"] = true - } else if (shorten) { - params["shorten"] = true - } + val shortenOrExtend = if (extend) "extend" else if (shorten) "shorten" else "" val signData = "${Snode.Method.Expire.rawValue}$shortenOrExtend$newExpiry${messageHashes.joinToString(separator = "")}".toByteArray() - val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - try { - sodium.cryptoSignDetached( - signature, - signData, - signData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val signature = try { + signAndEncode(signData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return null } - params["pubkey"] = publicKey - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) - return params + return buildMap { + this["expiry"] = newExpiry + this["messages"] = messageHashes + when { + extend -> this["extend"] = true + shorten -> this["shorten"] = true + } + this["pubkey"] = publicKey + this["pubkey_ed25519"] = userEd25519KeyPair.publicKey.asHexString + this["signature"] = signature + } } - fun getMessages(publicKey: String): MessageListPromise { - return retryIfNeeded(maxRetryCount) { - getSingleTargetSnode(publicKey).bind { snode -> - getRawMessages(snode, publicKey).map { parseRawMessagesResponse(it, snode, publicKey) } - } + fun getMessages(publicKey: String): MessageListPromise = retryIfNeeded(maxRetryCount) { + getSingleTargetSnode(publicKey).bind { snode -> + getRawMessages(snode, publicKey).map { parseRawMessagesResponse(it, snode, publicKey) } } } - private fun getNetworkTime(snode: Snode): Promise, Exception> { - return invoke(Snode.Method.Info, snode, emptyMap()).map { rawResponse -> + private fun getNetworkTime(snode: Snode): Promise, Exception> = + invoke(Snode.Method.Info, snode, emptyMap()).map { rawResponse -> val timestamp = rawResponse["timestamp"] as? Long ?: -1 snode to timestamp } - } - fun sendMessage(message: SnodeMessage, requiresAuth: Boolean = false, namespace: Int = 0): RawResponsePromise { - val destination = message.recipient - return retryIfNeeded(maxRetryCount) { + fun sendMessage(message: SnodeMessage, requiresAuth: Boolean = false, namespace: Int = 0): RawResponsePromise = + retryIfNeeded(maxRetryCount) { val module = MessagingModuleConfiguration.shared val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val parameters = message.toJSON().toMutableMap() + val parameters = message.toJSON().toMutableMap() // Construct signature if (requiresAuth) { val sigTimestamp = nowWithOffset val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) // assume namespace here is non-zero, as zero namespace doesn't require auth val verificationData = "store$namespace$sigTimestamp".toByteArray() - try { - sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) + val signature = try { + signAndEncode(verificationData, userED25519KeyPair) } catch (exception: Exception) { return@retryIfNeeded Promise.ofFail(Error.SigningFailed) } parameters["sig_timestamp"] = sigTimestamp parameters["pubkey_ed25519"] = ed25519PublicKey - parameters["signature"] = Base64.encodeBytes(signature) + parameters["signature"] = signature } // If the namespace is default (0) here it will be implicitly read as 0 on the storage server // we only need to specify it explicitly if we want to (in future) or if it is non-zero if (namespace != 0) { parameters["namespace"] = namespace } + val destination = message.recipient getSingleTargetSnode(destination).bind { snode -> invoke(Snode.Method.SendMessage, snode, parameters, destination) } } - } - fun deleteMessage(publicKey: String, serverHashes: List): Promise, Exception> { - return retryIfNeeded(maxRetryCount) { + fun deleteMessage(publicKey: String, serverHashes: List): Promise, Exception> = + retryIfNeeded(maxRetryCount) { val module = MessagingModuleConfiguration.shared val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) getSingleTargetSnode(publicKey).bind { snode -> retryIfNeeded(maxRetryCount) { - val signature = ByteArray(Sign.BYTES) val verificationData = sequenceOf(Snode.Method.DeleteMessage.rawValue).plus(serverHashes).toByteArray() - sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) val deleteMessageParams = mapOf( "pubkey" to userPublicKey, "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, "messages" to serverHashes, - "signature" to Base64.encodeBytes(signature) + "signature" to signAndEncode(verificationData, userED25519KeyPair) ) invoke(Snode.Method.DeleteMessage, snode, deleteMessageParams, publicKey).map { rawResponse -> val swarms = rawResponse["swarm"] as? Map ?: return@map mapOf() @@ -729,44 +618,36 @@ object SnodeAPI { } } } - } // Parsing private fun parseSnodes(rawResponse: Any): List = (rawResponse as? Map<*, *>) ?.run { get("snodes") as? List<*> } - ?.mapNotNull { rawSnode -> - val rawSnodeAsJSON = rawSnode as? Map<*, *> - val address = rawSnodeAsJSON?.get("ip") as? String - val portAsString = rawSnodeAsJSON?.get("port") as? String - val port = portAsString?.toInt() - val ed25519Key = rawSnodeAsJSON?.get(KEY_ED25519) as? String - val x25519Key = rawSnodeAsJSON?.get(KEY_X25519) as? String - - if (address != null && port != null && ed25519Key != null && x25519Key != null && address != "0.0.0.0") { - Snode("https://$address", port, Snode.KeySet(ed25519Key, x25519Key), Snode.Version.ZERO) - } else { - Log.d("Loki", "Failed to parse snode from: ${rawSnode?.prettifiedDescription()}.") - null - } - } ?: listOf().also { Log.d("Loki", "Failed to parse snodes from: ${rawResponse.prettifiedDescription()}.") } - - fun deleteAllMessages(): Promise, Exception> { - return retryIfNeeded(maxRetryCount) { + ?.asSequence() + ?.mapNotNull { it as? Map<*, *> } + ?.mapNotNull { + createSnode( + address = it["ip"] as? String, + port = (it["port"] as? String)?.toInt(), + ed25519Key = it[KEY_ED25519] as? String, + x25519Key = it[KEY_X25519] as? String + ).apply { if (this == null) Log.d("Loki", "Failed to parse snode from: ${it.prettifiedDescription()}.") } + }?.toList() ?: listOf().also { Log.d("Loki", "Failed to parse snodes from: ${rawResponse.prettifiedDescription()}.") } + + fun deleteAllMessages(): Promise, Exception> = + retryIfNeeded(maxRetryCount) { val module = MessagingModuleConfiguration.shared val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) getSingleTargetSnode(userPublicKey).bind { snode -> retryIfNeeded(maxRetryCount) { getNetworkTime(snode).bind { (_, timestamp) -> - val signature = ByteArray(Sign.BYTES) val verificationData = (Snode.Method.DeleteAll.rawValue + Namespace.ALL + timestamp.toString()).toByteArray() - sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) val deleteMessageParams = mapOf( "pubkey" to userPublicKey, "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, "timestamp" to timestamp, - "signature" to Base64.encodeBytes(signature), + "signature" to signAndEncode(verificationData, userED25519KeyPair), "namespace" to Namespace.ALL, ) invoke(Snode.Method.DeleteAll, snode, deleteMessageParams, userPublicKey).map { @@ -778,75 +659,14 @@ object SnodeAPI { } } } - } - fun updateExpiry(updatedExpiryMs: Long, serverHashes: List): Promise, Long>>, Exception> { - return retryIfNeeded(maxRetryCount) { - val module = MessagingModuleConfiguration.shared - val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val updatedExpiryMsWithNetworkOffset = updatedExpiryMs + clockOffset - getSingleTargetSnode(userPublicKey).bind { snode -> - retryIfNeeded(maxRetryCount) { - // "expire" || expiry || messages[0] || ... || messages[N] - val verificationData = sequenceOf(Snode.Method.Expire.rawValue, "$updatedExpiryMsWithNetworkOffset").plus(serverHashes).toByteArray() - val signature = ByteArray(Sign.BYTES) - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userED25519KeyPair.secretKey.asBytes - ) - val params = mapOf( - "pubkey" to userPublicKey, - "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, - "expiry" to updatedExpiryMs, - "messages" to serverHashes, - "signature" to Base64.encodeBytes(signature) - ) - invoke(Snode.Method.Expire, snode, params, userPublicKey).map { rawResponse -> - val swarms = rawResponse["swarm"] as? Map ?: return@map mapOf() - val result = swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> - val json = rawJSON as? Map ?: return@mapNotNull null - val isFailed = json["failed"] as? Boolean ?: false - val statusCode = json["code"] as? String - val reason = json["reason"] as? String - hexSnodePublicKey to if (isFailed) { - Log.e("Loki", "Failed to update expiry for: $hexSnodePublicKey due to error: $reason ($statusCode).") - listOf() to 0L - } else { - val hashes = json["updated"] as List - val expiryApplied = json["expiry"] as Long - val signature = json["signature"] as String - val snodePublicKey = Key.fromHexString(hexSnodePublicKey) - // The signature looks like ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] ) - val message = sequenceOf(userPublicKey).plus(serverHashes).plus(hashes).toByteArray() - if (sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes)) { - hashes to expiryApplied - } else listOf() to 0L - } - } - return@map result.toMap() - }.fail { e -> - Log.e("Loki", "Failed to update expiry", e) - } - } - } - } - } - - fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String, namespace: Int = 0, updateLatestHash: Boolean = true, updateStoredHashes: Boolean = true): List> { - val messages = rawResponse["messages"] as? List<*> - return if (messages != null) { + fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String, namespace: Int = 0, updateLatestHash: Boolean = true, updateStoredHashes: Boolean = true): List> = + (rawResponse["messages"] as? List<*>)?.let { messages -> if (updateLatestHash) { updateLastMessageHashValueIfPossible(snode, publicKey, messages, namespace) } - val newRawMessages = removeDuplicates(publicKey, messages, namespace, updateStoredHashes) - return parseEnvelopes(newRawMessages) - } else { - listOf() - } - } + removeDuplicates(publicKey, messages, namespace, updateStoredHashes).let(::parseEnvelopes) + } ?: listOf() fun updateLastMessageHashValueIfPossible(snode: Snode, publicKey: String, rawMessages: List<*>, namespace: Int) { val lastMessageAsJSON = rawMessages.lastOrNull() as? Map<*, *> @@ -859,34 +679,33 @@ object SnodeAPI { } fun removeDuplicates(publicKey: String, rawMessages: List<*>, namespace: Int, updateStoredHashes: Boolean): List<*> { - val originalMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace)?.toMutableSet() ?: mutableSetOf() + val originalMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace) ?: emptySet() val receivedMessageHashValues = originalMessageHashValues.toMutableSet() val result = rawMessages.filter { rawMessage -> - val rawMessageAsJSON = rawMessage as? Map<*, *> - val hashValue = rawMessageAsJSON?.get("hash") as? String - if (hashValue != null) { - val isDuplicate = receivedMessageHashValues.contains(hashValue) - receivedMessageHashValues.add(hashValue) - !isDuplicate - } else { - Log.d("Loki", "Missing hash value for message: ${rawMessage?.prettifiedDescription()}.") - false - } + (rawMessage as? Map<*, *>) + ?.let { it["hash"] as? String } + ?.let { receivedMessageHashValues.add(it) } + ?: false.also { Log.d("Loki", "Missing hash value for message: ${rawMessage?.prettifiedDescription()}.") } } - if (originalMessageHashValues != receivedMessageHashValues && updateStoredHashes) { + if (updateStoredHashes && originalMessageHashValues.containsAll(receivedMessageHashValues)) { database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues, namespace) } return result } - private fun parseEnvelopes(rawMessages: List<*>): List> { - return rawMessages.mapNotNull { rawMessage -> + private fun parseEnvelopes(rawMessages: List<*>): List> = + rawMessages.mapNotNull { rawMessage -> val rawMessageAsJSON = rawMessage as? Map<*, *> val base64EncodedData = rawMessageAsJSON?.get("data") as? String val data = base64EncodedData?.let { Base64.decode(it) } + + data?.runCatching(MessageWrapper::unwrap) + ?.map { it to rawMessageAsJSON["hash"] as? String } + ?.onFailure { Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") } + if (data != null) { try { - Pair(MessageWrapper.unwrap(data), rawMessageAsJSON.get("hash") as? String) + MessageWrapper.unwrap(data) to rawMessageAsJSON["hash"] as? String } catch (e: Exception) { Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") null @@ -896,12 +715,11 @@ object SnodeAPI { null } } - } @Suppress("UNCHECKED_CAST") private fun parseDeletions(userPublicKey: String, timestamp: Long, rawResponse: RawResponse): Map { val swarms = rawResponse["swarm"] as? Map ?: return mapOf() - val result = swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> + return swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> val json = rawJSON as? Map ?: return@mapNotNull null val isFailed = json["failed"] as? Boolean ?: false val statusCode = json["code"] as? String @@ -917,14 +735,13 @@ object SnodeAPI { val message = sequenceOf(userPublicKey, "$timestamp").plus(hashes).toByteArray() sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes) } - } - return result.toMap() + }.toMap() } // endregion // Error Handling - internal fun handleSnodeError(statusCode: Int, json: Map<*, *>?, snode: Snode, publicKey: String? = null): Exception? { + internal fun handleSnodeError(statusCode: Int, json: Map<*, *>?, snode: Snode, publicKey: String? = null): Throwable? = runCatching { fun handleBadSnode() { val oldFailureCount = snodeFailureCount[snode] ?: 0 val newFailureCount = oldFailureCount + 1 @@ -932,56 +749,43 @@ object SnodeAPI { Log.d("Loki", "Couldn't reach snode at $snode; setting failure count to $newFailureCount.") if (newFailureCount >= snodeFailureThreshold) { Log.d("Loki", "Failure threshold reached for: $snode; dropping it.") - if (publicKey != null) { - dropSnodeFromSwarmIfNeeded(snode, publicKey) - } - snodePool = snodePool.toMutableSet().minus(snode).toSet() + publicKey?.let { dropSnodeFromSwarmIfNeeded(snode, it) } + snodePool -= snode Log.d("Loki", "Snode pool count: ${snodePool.count()}.") snodeFailureCount[snode] = 0 } } when (statusCode) { - 400, 500, 502, 503 -> { // Usually indicates that the snode isn't up to date - handleBadSnode() - } + // Usually indicates that the snode isn't up to date + 400, 500, 502, 503 -> handleBadSnode() 406 -> { Log.d("Loki", "The user's clock is out of sync with the service node network.") broadcaster.broadcast("clockOutOfSync") - return Error.ClockOutOfSync + throw Error.ClockOutOfSync } 421 -> { // The snode isn't associated with the given public key anymore if (publicKey != null) { - fun invalidateSwarm() { - Log.d("Loki", "Invalidating swarm for: $publicKey.") - dropSnodeFromSwarmIfNeeded(snode, publicKey) - } - if (json != null) { - val snodes = parseSnodes(json) - if (snodes.isNotEmpty()) { - database.setSwarm(publicKey, snodes.toSet()) - } else { - invalidateSwarm() + json?.let(::parseSnodes) + ?.takeIf { it.isNotEmpty() } + ?.let { database.setSwarm(publicKey, it.toSet()) } + ?: run { + Log.d("Loki", "Invalidating swarm for: $publicKey.") + dropSnodeFromSwarmIfNeeded(snode, publicKey) } - } else { - invalidateSwarm() - } - } else { - Log.d("Loki", "Got a 421 without an associated public key.") - } + } else Log.d("Loki", "Got a 421 without an associated public key.") } 404 -> { Log.d("Loki", "404, probably no file found") - return Error.Generic + throw Error.Generic } else -> { handleBadSnode() Log.d("Loki", "Unhandled response code: ${statusCode}.") - return Error.Generic + throw Error.Generic } } - return null - } + }.exceptionOrNull() } // Type Aliases diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java b/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java index 3ff38f76d3..35ec22e0e0 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java @@ -1,5 +1,7 @@ package org.session.libsignal.utilities; +import androidx.annotation.NonNull; + /** *

Encodes and decodes to and from Base64 notation.

*

Homepage: http://iharder.net/base64.

@@ -714,7 +716,7 @@ public class Base64 * @throws NullPointerException if source array is null * @since 1.4 */ - public static String encodeBytes( byte[] source ) { + public static String encodeBytes(@NonNull byte[] source ) { // Since we're not going to have the GZIP encoding turned on, // we're not going to have an java.io.IOException thrown, so // we should not force the user to have to catch it.