Merge pull request #734 from neuroscr/fileproxy

ephemeral layering file proxy
pull/721/head
Maxim Shishmarev 5 years ago committed by GitHub
commit dc0571137c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,12 @@
{
"storageProfile": "swarm-testing",
"seedNodeList": [
{
"ip": "localhost",
"port": "22129"
}
],
"openDevTools": true,
"defaultPublicChatServer": "https://team-chat.lokinet.org/"
}

@ -0,0 +1,12 @@
{
"storageProfile": "swarm-testing2",
"seedNodeList": [
{
"ip": "localhost",
"port": "22129"
}
],
"openDevTools": true,
"defaultPublicChatServer": "https://team-chat.lokinet.org/"
}

@ -644,7 +644,7 @@
Whisper.events.on('registration_done', async () => {
window.log.info('handling registration event');
// Disable link previews as default per Kee 20/01/28
// Disable link previews as default per Kee
storage.onready(async () => {
storage.put('linkPreviews', false);
});

@ -1,6 +1,6 @@
/* global log, textsecure, libloki, Signal, Whisper, Headers, ConversationController,
clearTimeout, MessageController, libsignal, StringView, window, _,
dcodeIO, Buffer */
dcodeIO, Buffer, lokiSnodeAPI, TextDecoder */
const nodeFetch = require('node-fetch');
const { URL, URLSearchParams } = require('url');
const FormData = require('form-data');
@ -108,7 +108,7 @@ class LokiAppDotNetServerAPI {
// no big deal if it fails...
if (res.err || !res.response || !res.response.data) {
if (res.err) {
log.error(`Error ${res.err}`);
log.error(`setProfileName Error ${res.err}`);
}
return [];
}
@ -135,7 +135,7 @@ class LokiAppDotNetServerAPI {
if (res.err || !res.response || !res.response.data) {
if (res.err) {
log.error(`Error ${res.err}`);
log.error(`setHomeServer Error ${res.err}`);
}
return [];
}
@ -291,6 +291,102 @@ class LokiAppDotNetServerAPI {
}
}
async _sendToProxy(fetchOptions, method, headers, endpoint) {
const randSnode = await lokiSnodeAPI.getRandomSnodeAddress();
const url = `https://${randSnode.ip}:${randSnode.port}/file_proxy`;
const payloadObj = {
// I think this is a stream, we may need to collect it all?
body: fetchOptions.body, // might need to b64 if binary...
endpoint,
method,
headers,
};
// from https://github.com/sindresorhus/is-stream/blob/master/index.js
if (
payloadObj.body &&
typeof payloadObj.body === 'object' &&
typeof payloadObj.body.pipe === 'function'
) {
log.info('detected body is a stream');
const fData = payloadObj.body.getBuffer();
const fHeaders = payloadObj.body.getHeaders();
// update headers for boundary
payloadObj.headers = { ...payloadObj.headers, ...fHeaders };
// update body with base64 chunk
payloadObj.body = {
fileUpload: fData.toString('base64'),
};
}
// convert our payload to binary buffer
const payloadData = Buffer.from(
dcodeIO.ByteBuffer.wrap(JSON.stringify(payloadObj)).toArrayBuffer()
);
payloadObj.body = false; // free memory
// make temporary key for this request/response
const ephemeralKey = libsignal.Curve.generateKeyPair();
// mix server pub key with our priv key
const symKey = libsignal.Curve.calculateAgreement(
this.pubKey, // server's pubkey
ephemeralKey.privKey // our privkey
);
const ivAndCiphertext = await libloki.crypto.DHEncrypt(symKey, payloadData);
// convert final buffer to base64
const cipherText64 = dcodeIO.ByteBuffer.wrap(ivAndCiphertext).toString(
'base64'
);
const ephemeralPubKey64 = dcodeIO.ByteBuffer.wrap(
ephemeralKey.pubKey
).toString('base64');
const finalRequestHeader = {
'X-Loki-File-Server-Ephemeral-Key': ephemeralPubKey64,
};
const firstHopOptions = {
method: 'POST',
// not sure why I can't use anything but json...
// text/plain would be preferred...
body: JSON.stringify({ cipherText64 }),
headers: {
'Content-Type': 'application/json',
'X-Loki-File-Server-Target': '/loki/v1/secure_rpc',
'X-Loki-File-Server-Verb': 'POST',
'X-Loki-File-Server-Headers': JSON.stringify(finalRequestHeader),
},
};
const result = await nodeFetch(url, firstHopOptions);
const txtResponse = await result.text();
let response = JSON.parse(txtResponse);
if (response.meta && response.meta.code === 200) {
// convert base64 in response to binary
const ivAndCiphertextResponse = dcodeIO.ByteBuffer.wrap(
response.data,
'base64'
).toArrayBuffer();
const decrypted = await libloki.crypto.DHDecrypt(
symKey,
ivAndCiphertextResponse
);
const textDecoder = new TextDecoder();
const json = textDecoder.decode(decrypted);
// replace response
response = JSON.parse(json);
} else {
log.warn('file server secure_rpc gave an non-200 response');
}
return { result, txtResponse, response };
}
// make a request to the server
async serverRequest(endpoint, options = {}) {
const {
@ -300,14 +396,14 @@ class LokiAppDotNetServerAPI {
objBody,
forceFreshToken = false,
} = options;
const url = new URL(`${this.baseServerUrl}/${endpoint}`);
if (params) {
url.search = new URLSearchParams(params);
}
let result;
const fetchOptions = {};
const headers = {};
try {
const fetchOptions = {};
const headers = {};
if (forceFreshToken) {
await this.getOrRefreshServerToken(true);
}
@ -324,24 +420,42 @@ class LokiAppDotNetServerAPI {
fetchOptions.body = rawBody;
}
fetchOptions.headers = new Headers(headers);
result = await nodeFetch(url, fetchOptions || undefined);
} catch (e) {
log.info(`e ${e}`);
log.info('serverRequest set up error:', JSON.stringify(e));
return {
err: e,
};
}
let response = null;
let response;
let result;
let txtResponse;
let mode = 'nodeFetch';
try {
response = await result.json();
if (
window.lokiFeatureFlags.useSnodeProxy &&
(this.baseServerUrl === 'https://file-dev.lokinet.org' ||
this.baseServerUrl === 'https://file.lokinet.org')
) {
mode = '_sendToProxy';
// have to send headers because fetchOptions.headers isn't readable
({ response, txtResponse, result } = await this._sendToProxy(
fetchOptions,
method,
headers,
endpoint
));
} else {
result = await nodeFetch(url, fetchOptions || undefined);
txtResponse = await result.text();
response = JSON.parse(txtResponse);
}
} catch (e) {
log.warn(`serverRequest json parse ${e}`);
log.info(`serverRequest ${mode} error json: ${txtResponse}`);
return {
err: e,
statusCode: result.status,
};
}
// if it's a response style with a meta
if (result.status !== 200) {
if (!forceFreshToken && (!response.meta || response.meta.code === 401)) {
@ -378,7 +492,7 @@ class LokiAppDotNetServerAPI {
if (res.err || !res.response || !res.response.data) {
if (res.err) {
log.error(`Error ${res.err}`);
log.error(`getUserAnnotations Error ${res.err}`);
}
return [];
}
@ -497,7 +611,7 @@ class LokiAppDotNetServerAPI {
if (res.err || !res.response || !res.response.data) {
if (res.err) {
log.error(`Error ${res.err}`);
log.error(`getSubscribers Error ${res.err}`);
}
return [];
}
@ -527,7 +641,7 @@ class LokiAppDotNetServerAPI {
if (res.err || !res.response || !res.response.data) {
if (res.err) {
log.error(`Error ${res.err}`);
log.error(`getUsers Error ${res.err}`);
}
return [];
}
@ -620,6 +734,7 @@ class LokiAppDotNetServerAPI {
contentType: 'application/octet-stream',
name: 'content',
filename: 'attachment',
knownLength: buffer.byteLength,
});
return this.uploadData(formData);
@ -682,7 +797,7 @@ class LokiPublicChannelAPI {
if (res.err || !res.response || !res.response.data) {
if (res.err) {
log.error(`Error ${res.err}`);
log.error(`banUser Error ${res.err}`);
}
return false;
}
@ -798,7 +913,7 @@ class LokiPublicChannelAPI {
);
if (updateRes.err || !updateRes.response || !updateRes.response.data) {
if (updateRes.err) {
log.error(`Error ${updateRes.err}`);
log.error(`setChannelSettings Error ${updateRes.err}`);
}
return false;
}
@ -955,7 +1070,7 @@ class LokiPublicChannelAPI {
// if any problems, abort out
if (res.err || !res.response) {
if (res.err) {
log.error(`Error ${res.err}`);
log.error(`pollOnceForDeletions Error ${res.err}`);
}
break;
}
@ -1369,6 +1484,7 @@ class LokiPublicChannelAPI {
const primaryPubKey = slavePrimaryMap[slaveKey];
// send out remaining messages for this merged identity
/* eslint-disable no-param-reassign */
if (slavePrimaryMap[slaveKey]) {
// rewrite source, profile
messageData.source = primaryPubKey;
@ -1379,6 +1495,7 @@ class LokiPublicChannelAPI {
messageData.message.profile.avatar = avatar;
messageData.message.profileKey = profileKey;
}
/* eslint-enable no-param-reassign */
this.chatAPI.emit('publicMessage', {
message: messageData,
});

@ -1,4 +1,4 @@
/* global log, libloki */
/* global log, libloki, process, window */
/* global storage: false */
/* global Signal: false */
/* global log: false */
@ -8,11 +8,49 @@ const LokiAppDotNetAPI = require('./loki_app_dot_net_api');
const DEVICE_MAPPING_USER_ANNOTATION_TYPE =
'network.loki.messenger.devicemapping';
// const LOKIFOUNDATION_DEVFILESERVER_PUBKEY =
// 'BSZiMVxOco/b3sYfaeyiMWv/JnqokxGXkHoclEx8TmZ6';
const LOKIFOUNDATION_FILESERVER_PUBKEY =
'BWJQnVm97sQE3Q1InB4Vuo+U/T1hmwHBv0ipkiv8tzEc';
// can have multiple of these instances as each user can have a
// different home server
class LokiFileServerInstance {
constructor(ourKey) {
this.ourKey = ourKey;
// do we have their pubkey locally?
/*
// get remote pubKey
this._server.serverRequest('loki/v1/public_key').then(keyRes => {
// we don't need to delay to protect identity because the token request
// should only be done over lokinet-lite
this.delayToken = true;
if (keyRes.err || !keyRes.response || !keyRes.response.data) {
if (keyRes.err) {
log.error(`Error ${keyRes.err}`);
}
} else {
// store it
this.pubKey = dcodeIO.ByteBuffer.wrap(
keyRes.response.data,
'base64'
).toArrayBuffer();
// write it to a file
}
});
*/
// Hard coded
this.pubKey = window.Signal.Crypto.base64ToArrayBuffer(
LOKIFOUNDATION_FILESERVER_PUBKEY
);
if (this.pubKey.byteLength && this.pubKey.byteLength !== 33) {
log.error(
'FILESERVER PUBKEY is invalid, length:',
this.pubKey.byteLength
);
process.exit(1);
}
}
// FIXME: this is not file-server specific
@ -21,6 +59,10 @@ class LokiFileServerInstance {
async establishConnection(serverUrl) {
// why don't we extend this?
this._server = new LokiAppDotNetAPI(this.ourKey, serverUrl);
// configure proxy
this._server.pubKey = this.pubKey;
// get a token for multidevice
const gotToken = await this._server.getOrRefreshServerToken();
// TODO: Handle this failure gracefully
@ -220,6 +262,8 @@ class LokiHomeServerInstance extends LokiFileServerInstance {
return this._setOurDeviceMapping(authorisations, isPrimary);
}
// you only upload to your own home server
// you can download from any server...
uploadAvatar(data) {
return this._server.uploadAvatar(data);
}

@ -3,7 +3,7 @@
/* global log, dcodeIO, window, callWorker, lokiP2pAPI, lokiSnodeAPI, textsecure */
const _ = require('lodash');
const { rpc } = require('./loki_rpc');
const { lokiRpc } = require('./loki_rpc');
const DEFAULT_CONNECTIONS = 3;
const MAX_ACCEPTABLE_FAILURES = 1;
@ -47,7 +47,7 @@ const trySendP2p = async (pubKey, data64, isPing, messageEventData) => {
return false;
}
try {
await rpc(p2pDetails.address, p2pDetails.port, 'store', {
await lokiRpc(p2pDetails.address, p2pDetails.port, 'store', {
data: data64,
});
lokiP2pAPI.setContactOnline(pubKey);
@ -213,6 +213,7 @@ class LokiMessageAPI {
const successfulSend = await this.sendToNode(
snode.ip,
snode.port,
snode,
params
);
if (successfulSend) {
@ -237,12 +238,20 @@ class LokiMessageAPI {
return false;
}
async sendToNode(address, port, params) {
async sendToNode(address, port, targetNode, params) {
let successiveFailures = 0;
while (successiveFailures < MAX_ACCEPTABLE_FAILURES) {
await sleepFor(successiveFailures * 500);
try {
const result = await rpc(`https://${address}`, port, 'store', params);
const result = await lokiRpc(
`https://${address}`,
port,
'store',
params,
{},
'/storage_rpc/v1',
targetNode
);
// Make sure we aren't doing too much PoW
const currentDifficulty = window.storage.get('PoWDifficulty', null);
@ -365,12 +374,14 @@ class LokiMessageAPI {
},
};
const result = await rpc(
const result = await lokiRpc(
`https://${nodeUrl}`,
nodeData.port,
'retrieve',
params,
options
options,
'/storage_rpc/v1',
nodeData
);
return result.messages || [];
}
@ -386,10 +397,10 @@ class LokiMessageAPI {
const lastHash = await window.Signal.Data.getLastHashBySnode(
nodes[i].address
);
this.ourSwarmNodes[nodes[i].address] = {
...nodes[i],
lastHash,
ip: nodes[i].ip,
port: nodes[i].port,
};
}

@ -1,4 +1,5 @@
/* global log, libloki, textsecure, getStoragePubKey */
/* global log, libloki, textsecure, getStoragePubKey, lokiSnodeAPI, StringView,
libsignal, window, TextDecoder, TextEncoder, dcodeIO */
const nodeFetch = require('node-fetch');
const { parse } = require('url');
@ -21,8 +22,70 @@ const decryptResponse = async (response, address) => {
return {};
};
// TODO: Don't allow arbitrary URLs, only snodes and loki servers
const sendToProxy = async (options = {}, targetNode) => {
const randSnode = await lokiSnodeAPI.getRandomSnodeAddress();
const url = `https://${randSnode.ip}:${randSnode.port}/proxy`;
log.info(
`Proxy snode request to ${targetNode.pubkey_ed25519} via ${
randSnode.pubkey_ed25519
}`
);
const snPubkeyHex = StringView.hexToArrayBuffer(targetNode.pubkey_x25519);
const myKeys = window.libloki.crypto.snodeCipher._ephemeralKeyPair;
const symmetricKey = libsignal.Curve.calculateAgreement(
snPubkeyHex,
myKeys.privKey
);
const textEncoder = new TextEncoder();
const body = JSON.stringify(options);
const plainText = textEncoder.encode(body);
const ivAndCiphertext = await window.libloki.crypto.DHEncrypt(
symmetricKey,
plainText
);
const firstHopOptions = {
method: 'POST',
body: ivAndCiphertext,
headers: {
'X-Sender-Public-Key': StringView.arrayBufferToHex(myKeys.pubKey),
'X-Target-Snode-Key': targetNode.pubkey_ed25519,
},
};
const response = await nodeFetch(url, firstHopOptions);
const ciphertext = await response.text();
const ciphertextBuffer = dcodeIO.ByteBuffer.wrap(
ciphertext,
'base64'
).toArrayBuffer();
const plaintextBuffer = await window.libloki.crypto.DHDecrypt(
symmetricKey,
ciphertextBuffer
);
const textDecoder = new TextDecoder();
const plaintext = textDecoder.decode(plaintextBuffer);
const jsonRes = JSON.parse(plaintext);
jsonRes.json = () => JSON.parse(jsonRes.body);
return jsonRes;
};
// A small wrapper around node-fetch which deserializes response
const fetch = async (url, options = {}) => {
const lokiFetch = async (url, options = {}, targetNode = null) => {
const timeout = options.timeout || 10000;
const method = options.method || 'GET';
@ -47,12 +110,19 @@ const fetch = async (url, options = {}) => {
}
}
const fetchOptions = {
...options,
timeout,
method,
};
try {
const response = await nodeFetch(url, {
...options,
timeout,
method,
});
if (window.lokiFeatureFlags.useSnodeProxy && targetNode) {
const result = await sendToProxy(fetchOptions, targetNode);
return result.json();
}
const response = await nodeFetch(url, fetchOptions);
let result;
// Wrong swarm
@ -107,13 +177,14 @@ const fetch = async (url, options = {}) => {
};
// Wrapper for a JSON RPC request
const rpc = (
const lokiRpc = (
address,
port,
method,
params,
options = {},
endpoint = endpointBase
endpoint = endpointBase,
targetNode
) => {
const headers = options.headers || {};
const portString = port ? `:${port}` : '';
@ -144,9 +215,9 @@ const rpc = (
},
};
return fetch(url, fetchOptions);
return lokiFetch(url, fetchOptions, targetNode);
};
module.exports = {
rpc,
lokiRpc,
};

@ -1,10 +1,10 @@
/* eslint-disable class-methods-use-this */
/* global window, ConversationController, _ */
/* global window, ConversationController, _, log */
const is = require('@sindresorhus/is');
const dns = require('dns');
const process = require('process');
const { rpc } = require('./loki_rpc');
const { lokiRpc } = require('./loki_rpc');
const natUpnp = require('nat-upnp');
const resolve4 = url =>
@ -94,6 +94,8 @@ class LokiSnodeAPI {
fields: {
public_ip: true,
storage_port: true,
pubkey_x25519: true,
pubkey_ed25519: true,
},
};
const seedNode = seedNodes.splice(
@ -101,7 +103,7 @@ class LokiSnodeAPI {
1
)[0];
try {
const result = await rpc(
const result = await lokiRpc(
`http://${seedNode.ip}`,
seedNode.port,
'get_n_service_nodes',
@ -116,8 +118,11 @@ class LokiSnodeAPI {
this.randomSnodePool = snodes.map(snode => ({
ip: snode.public_ip,
port: snode.storage_port,
pubkey_x25519: snode.pubkey_x25519,
pubkey_ed25519: snode.pubkey_ed25519,
}));
} catch (e) {
log.warn('initialiseRandomPool error', JSON.stringify(e));
window.mixpanel.track('Seed Node Failed');
if (seedNodes.length === 0) {
throw new window.textsecure.SeedNodeError(
@ -191,17 +196,27 @@ class LokiSnodeAPI {
async getSwarmNodes(pubKey) {
// TODO: Hit multiple random nodes and merge lists?
const { ip, port } = await this.getRandomSnodeAddress();
const snode = await this.getRandomSnodeAddress();
try {
const result = await rpc(`https://${ip}`, port, 'get_snodes_for_pubkey', {
pubKey,
});
const snodes = result.snodes.filter(snode => snode.ip !== '0.0.0.0');
const result = await lokiRpc(
`https://${snode.ip}`,
snode.port,
'get_snodes_for_pubkey',
{
pubKey,
},
{},
'/storage_rpc/v1',
snode
);
const snodes = result.snodes.filter(tSnode => tSnode.ip !== '0.0.0.0');
return snodes;
} catch (e) {
log.error('getSwarmNodes', JSON.stringify(e));
//
this.randomSnodePool = _.without(
this.randomSnodePool,
_.find(this.randomSnodePool, { ip })
_.find(this.randomSnodePool, { ip: snode.ip })
);
return this.getSwarmNodes(pubKey);
}

@ -17,6 +17,8 @@
"start-multi2": "cross-env NODE_APP_INSTANCE=2 electron .",
"start-prod": "cross-env NODE_ENV=production NODE_APP_INSTANCE=devprod LOKI_DEV=1 electron .",
"start-prod-multi": "cross-env NODE_ENV=production NODE_APP_INSTANCE=devprod1 LOKI_DEV=1 electron .",
"start-swarm-test": "cross-env NODE_ENV=swarm-testing NODE_APP_INSTANCE=test1 LOKI_DEV=1 electron .",
"start-swarm-test-2": "cross-env NODE_ENV=swarm-testing2 NODE_APP_INSTANCE=test2 LOKI_DEV=1 electron .",
"grunt": "grunt",
"icon-gen": "electron-icon-maker --input=images/icon_1024.png --output=./build",
"generate": "yarn icon-gen && yarn grunt",
@ -78,7 +80,7 @@
"emoji-panel": "https://github.com/scottnonnenberg-signal/emoji-panel.git#v0.5.5",
"filesize": "3.6.1",
"firstline": "1.2.1",
"form-data": "2.3.2",
"form-data": "^3.0.0",
"fs-extra": "5.0.0",
"glob": "7.1.2",
"google-libphonenumber": "3.2.2",

@ -498,6 +498,7 @@ window.SMALL_GROUP_SIZE_LIMIT = 10;
window.lokiFeatureFlags = {
multiDeviceUnpairing: true,
privateGroupChats: false,
useSnodeProxy: false,
};
// eslint-disable-next-line no-extend-native,func-names

@ -3824,16 +3824,16 @@ forever-agent@~0.6.1:
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=
form-data@2.3.2:
version "2.3.2"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099"
integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=
form-data@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682"
integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==
dependencies:
asynckit "^0.4.0"
combined-stream "1.0.6"
combined-stream "^1.0.8"
mime-types "^2.1.12"
form-data@~2.3.1, form-data@~2.3.2:
form-data@~2.3.1:
version "2.3.3"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"
integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==
@ -3842,6 +3842,15 @@ form-data@~2.3.1, form-data@~2.3.2:
combined-stream "^1.0.6"
mime-types "^2.1.12"
form-data@~2.3.2:
version "2.3.2"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099"
integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=
dependencies:
asynckit "^0.4.0"
combined-stream "1.0.6"
mime-types "^2.1.12"
forwarded@~0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"

Loading…
Cancel
Save