Merge remote-tracking branch 'upstream/clearnet' into react-refactor

config/swarm-testing.json

@@ -6,5 +6,6 @@
     }
   ],
   "openDevTools": true,
-  "defaultPublicChatServer": "https://team-chat.lokinet.org/"
+  "defaultPublicChatServer": "https://team-chat.lokinet.org/",
+  "defaultFileServer": "https://file-dev.getsession.org"
 }

js/modules/loki_app_dot_net_api.d.ts

@@ -9,6 +9,11 @@ interface UploadResponse {
   id?: number;
 }
 
+interface DownloadResponse {
+  statucCode: number;
+  reponse: any;
+}
+
 export interface LokiAppDotNetServerInterface {
   findOrCreateChannel(
     api: LokiPublicChatFactoryAPI,
@@ -19,6 +24,7 @@ export interface LokiAppDotNetServerInterface {
   uploadAvatar(data: FormData): Promise<UploadResponse>;
   putAttachment(data: ArrayBuffer): Promise<UploadResponse>;
   putAvatar(data: ArrayBuffer): Promise<UploadResponse>;
+  downloadAttachment(url: String): Promise<DownloadResponse>; // todo: add return type
 }
 
 export interface LokiPublicChannelAPI {

js/modules/loki_app_dot_net_api.js

@@ -1240,6 +1240,14 @@ class LokiAppDotNetServerAPI {
     });
     return this.uploadAvatar(formData);
   }
+
+  async downloadAttachment(url) {
+    const endpoint = new URL(url).pathname;
+
+    return this.serverRequest(`loki/v1${endpoint}`, {
+      method: 'GET',
+    });
+  }
 }
 
 // functions to a specific ADN channel on an ADN server

js/modules/loki_file_server_api.d.ts

@@ -13,4 +13,5 @@ interface DeviceMappingAnnotation {
 interface LokiFileServerInstance {
   getUserDeviceMapping(pubKey: string): Promise<DeviceMappingAnnotation | null>;
   clearOurDeviceMappingAnnotations(): Promise<void>;
+  downloadAttachment(url: string): Promise<any>;
 }

js/modules/loki_file_server_api.js

@@ -198,6 +198,11 @@ class LokiFileServerInstance {
     result.slaveMap = newSlavePrimaryMap;
     return result;
   }
+
+  // for files
+  async downloadAttachment(url) {
+    return this._server.downloadAttachment(url);
+  }
 }
 
 // extends LokiFileServerInstance with functions we'd only perform on our own home server

libtextsecure/sendmessage.js

@@ -185,8 +185,8 @@ Message.prototype = {
 };
 
 function MessageSender() {
+  // Currently only used for getProxiedSize() and makeProxiedRequest(), which are only used for fetching previews
   this.server = WebAPI.connect();
-  this.pendingMessages = {};
 }
 
 MessageSender.prototype = {

preload.js

@@ -47,8 +47,8 @@ window.getCommitHash = () => config.commitHash;
 window.getNodeVersion = () => config.node_version;
 window.getHostName = () => config.hostname;
 window.getServerTrustRoot = () => config.serverTrustRoot;
-window.isBehindProxy = () => Boolean(config.proxyUrl);
 window.JobQueue = JobQueue;
+window.isBehindProxy = () => Boolean(config.proxyUrl);
 window.getStoragePubKey = key =>
   window.isDev() ? key.substring(0, key.length - 2) : key;
 window.getDefaultFileServer = () => config.defaultFileServer;
@@ -453,6 +453,7 @@ window.lokiFeatureFlags = {
   privateGroupChats: true,
   useSnodeProxy: !process.env.USE_STUBBED_NETWORK,
   useOnionRequests: true,
+  useOnionRequestsV2: false,
   useFileOnionRequests: true,
   enableSenderKeys: true,
   onionRequestHops: 3,

ts/receiver/attachments.ts

@@ -3,16 +3,35 @@ import _ from 'lodash';
 
 import * as Data from '../../js/modules/data';
 
-// TODO: Might convert it to a class later
-let webAPI: any;
-
 export async function downloadAttachment(attachment: any) {
-  if (!webAPI) {
-    webAPI = window.WebAPI.connect();
+  const serverUrl = new URL(attachment.url).origin;
+
+  // The fileserver adds the `-static` part for some reason
+  const defaultFileserver = _.includes(
+    ['https://file-static.lokinet.org', 'https://file.getsession.org'],
+    serverUrl
+  );
+
+  let res: any;
+
+  // TODO: we need attachments to remember which API should be used to retrieve them
+  if (!defaultFileserver) {
+    const serverAPI = await window.lokiPublicChatAPI.findOrCreateServer(
+      serverUrl
+    );
+
+    if (serverAPI) {
+      res = await serverAPI.downloadAttachment(attachment.url);
+    }
+  }
+
+  // Fallback to using the default fileserver
+  if (defaultFileserver || !res) {
+    res = await window.lokiFileServerAPI.downloadAttachment(attachment.url);
   }
 
   // The attachment id is actually just the absolute url of the attachment
-  let data = await webAPI.getAttachment(attachment.url);
+  let data = new Uint8Array(res.response.data).buffer;
   if (!attachment.isRaw) {
     const { key, digest, size } = attachment;
 

ts/session/snode_api/onions.ts

@@ -33,7 +33,7 @@ async function encryptForRelay(
   destination: any,
   ctx: any
 ) {
-  const { log, dcodeIO, StringView } = window;
+  const { log, StringView } = window;
 
   // ctx contains: ciphertext, symmetricKey, ephemeralKey
   const payload = ctx.ciphertext;
@@ -51,23 +51,80 @@ async function encryptForRelay(
   return encryptForPubKey(relayX25519hex, reqObj);
 }
 
-async function makeGuardPayload(guardCtx: any) {
+// `ctx` holds info used by `node` to relay further
+async function encryptForRelayV2(
+  relayX25519hex: string,
+  destination: any,
+  ctx: any
+) {
+  const { log, StringView } = window;
+
+  if (!destination.host && !destination.destination) {
+    log.warn('loki_rpc::encryptForRelay - no destination', destination);
+  }
+
+  const reqObj = {
+    ...destination,
+    ephemeral_key: StringView.arrayBufferToHex(ctx.ephemeralKey),
+  };
+
+  const plaintext = encodeCiphertextPlusJson(ctx.ciphertext, reqObj);
+
+  return window.libloki.crypto.encryptForPubkey(relayX25519hex, plaintext);
+}
+
+function makeGuardPayload(guardCtx: any): Uint8Array {
   const ciphertextBase64 = StringUtils.decode(guardCtx.ciphertext, 'base64');
 
-  const guardPayloadObj = {
+  const payloadObj = {
     ciphertext: ciphertextBase64,
     ephemeral_key: StringUtils.decode(guardCtx.ephemeralKey, 'hex'),
   };
-  return guardPayloadObj;
+
+  const payloadStr = JSON.stringify(payloadObj);
+
+  const buffer = ByteBuffer.wrap(payloadStr, 'utf8');
+
+  return buffer.buffer;
 }
 
-// we just need the targetNode.pubkey_ed25519 for the encryption
-// targetPubKey is ed25519 if snode is the target
-async function makeOnionRequest(
+/// Encode ciphertext as (len || binary) and append payloadJson as utf8
+function encodeCiphertextPlusJson(
+  ciphertext: any,
+  payloadJson: any
+): Uint8Array {
+  const payloadStr = JSON.stringify(payloadJson);
+
+  const bufferJson = ByteBuffer.wrap(payloadStr, 'utf8');
+
+  const len = ciphertext.length as number;
+  const arrayLen = bufferJson.buffer.length + 4 + len;
+  const littleEndian = true;
+  const buffer = new ByteBuffer(arrayLen, littleEndian);
+
+  buffer.writeInt32(len);
+  buffer.append(ciphertext);
+  buffer.append(bufferJson);
+
+  return new Uint8Array(buffer.buffer);
+}
+
+// New "semi-binary" encoding
+function makeGuardPayloadV2(guardCtx: any): Uint8Array {
+  const guardPayloadObj = {
+    ephemeral_key: StringUtils.decode(guardCtx.ephemeralKey, 'hex'),
+  };
+
+  return encodeCiphertextPlusJson(guardCtx.ciphertext, guardPayloadObj);
+}
+
+async function buildOnionCtxs(
   nodePath: Array<Snode>,
   destCtx: any,
   targetED25519Hex: string,
-  finalRelayOptions?: any,
+  // whether to use the new "semi-binary" protocol
+  useV2: boolean,
+  fileServerOptions?: any,
   id = ''
 ) {
   const { log } = window;
@@ -80,10 +137,12 @@ async function makeOnionRequest(
     let dest;
     const relayingToFinalDestination = i === firstPos; // if last position
 
-    if (relayingToFinalDestination && finalRelayOptions) {
+    if (relayingToFinalDestination && fileServerOptions) {
+      const target = useV2 ? '/loki/v2/lsrpc' : '/loki/v1/lsrpc';
+
       dest = {
-        host: finalRelayOptions.host,
-        target: '/loki/v1/lsrpc',
+        host: fileServerOptions.host,
+        target,
         method: 'POST',
       };
     } else {
@@ -107,8 +166,9 @@ async function makeOnionRequest(
       };
     }
     try {
+      const encryptFn = useV2 ? encryptForRelayV2 : encryptForRelay;
       // eslint-disable-next-line no-await-in-loop
-      const ctx = await encryptForRelay(
+      const ctx = await encryptFn(
         nodePath[i].pubkey_x25519,
         dest,
         ctxes[ctxes.length - 1]
@@ -123,12 +183,38 @@ async function makeOnionRequest(
       throw e;
     }
   }
+
+  return ctxes;
+}
+
+// we just need the targetNode.pubkey_ed25519 for the encryption
+// targetPubKey is ed25519 if snode is the target
+async function makeOnionRequest(
+  nodePath: Array<Snode>,
+  destCtx: any,
+  targetED25519Hex: string,
+  // whether to use the new (v2) protocol
+  useV2: boolean,
+  finalRelayOptions?: any,
+  id = ''
+) {
+  const ctxes = await buildOnionCtxs(
+    nodePath,
+    destCtx,
+    targetED25519Hex,
+    useV2,
+    finalRelayOptions,
+    id
+  );
+
   const guardCtx = ctxes[ctxes.length - 1]; // last ctx
 
-  const payloadObj = makeGuardPayload(guardCtx);
+  const payload = useV2
+    ? makeGuardPayloadV2(guardCtx)
+    : makeGuardPayload(guardCtx);
 
   // all these requests should use AesGcm
-  return payloadObj;
+  return payload;
 }
 
 // Process a response as it arrives from `fetch`, handling
@@ -276,14 +362,6 @@ const sendOnionRequest = async (
 ) => {
   const { log, StringView } = window;
 
-  // loki-storage may need this to function correctly
-  // but ADN calls will not always have a body
-  /*
-    if (!finalDestOptions.body) {
-      finalDestOptions.body = '';
-    }
-    */
-
   let id = '';
   if (lsrpcIdx !== undefined) {
     id += `${lsrpcIdx}=>`;
@@ -315,9 +393,25 @@ const sendOnionRequest = async (
     options.headers = '';
   }
 
+  const useV2 = window.lokiFeatureFlags.useOnionRequestsV2;
+
   let destCtx;
   try {
-    destCtx = await encryptForPubKey(destX25519hex, options);
+    if (useV2 && !finalRelayOptions) {
+      const body = options.body || '';
+      delete options.body;
+
+      const textEncoder = new TextEncoder();
+      const bodyEncoded = textEncoder.encode(body);
+
+      const plaintext = encodeCiphertextPlusJson(bodyEncoded, options);
+      destCtx = await window.libloki.crypto.encryptForPubkey(
+        destX25519hex,
+        plaintext
+      );
+    } else {
+      destCtx = await encryptForPubKey(destX25519hex, options);
+    }
   } catch (e) {
     log.error(
       `loki_rpc::sendOnionRequest ${id} - encryptForPubKey failure [`,
@@ -333,22 +427,27 @@ const sendOnionRequest = async (
     throw e;
   }
 
-  const payloadObj = await makeOnionRequest(
+  const payload = await makeOnionRequest(
     nodePath,
     destCtx,
     targetEd25519hex,
+    useV2,
     finalRelayOptions,
     id
   );
 
+  log.debug('Onion payload size: ', payload.length);
+
   const guardFetchOptions = {
     method: 'POST',
-    body: JSON.stringify(payloadObj),
+    body: payload,
     // we are talking to a snode...
     agent: snodeHttpsAgent,
   };
 
-  const guardUrl = `https://${nodePath[0].ip}:${nodePath[0].port}/onion_req`;
+  const target = useV2 ? '/onion_req/v2' : '/onion_req';
+
+  const guardUrl = `https://${nodePath[0].ip}:${nodePath[0].port}${target}`;
   const response = await fetch(guardUrl, guardFetchOptions);
 
   return processOnionResponse(

ts/window.d.ts

@@ -61,6 +61,7 @@ declare global {
       privateGroupChats: boolean;
       useSnodeProxy: boolean;
       useOnionRequests: boolean;
+      useOnionRequestsV2: boolean;
       useFileOnionRequests: boolean;
       enableSenderKeys: boolean;
       onionRequestHops: number;