fix: added batch requests for snode but signature fails

ts/node/database_utility.ts

@@ -79,6 +79,7 @@ const allowedKeysFormatRowOfConversation = [
   'identityPrivateKey',
 ];
 
+//tslint-disable cyclomatic-complexity
 export function formatRowOfConversation(row?: Record<string, any>): ConversationAttributes | null {
   if (!row) {
     return null;

ts/session/apis/open_group_api/sogsv3/sogsApiV3.ts

@@ -38,7 +38,6 @@ import { processMessagesUsingCache } from './sogsV3MutationCache';
 import { destroyMessagesAndUpdateRedux } from '../../../../util/expiringMessages';
 import { sogsRollingDeletions } from './sogsRollingDeletions';
 
-
 /**
  * Get the convo matching those criteria and make sure it is an opengroup convo, or return null.
  * If you get null, you most likely need to cancel the processing of whatever you are doing

ts/session/apis/snode_api/SnodeRequestTypes.ts

@@ -1,3 +1,4 @@
+
 export type SwarmForSubRequest = { method: 'get_swarm'; params: { pubkey: string } };
 
 type RetrieveMaxCountSize = { max_count?: number; max_size?: number };
@@ -18,6 +19,22 @@ export type RetrievePubkeySubRequestType = {
     RetrieveMaxCountSize;
 };
 
+/** Those namespaces do not require to be authenticated for storing messages.
+ *  -> 0 is used for our swarm, and anyone needs to be able to send message to us.
+ *  -> -10 is used for legacy closed group and we do not have authentication for them yet (but we will with the new closed groups)
+ *  -> others are currently unused
+ *
+ */
+// type UnauthenticatedStoreNamespaces = -30 | -20 | -10 | 0 | 10 | 20 | 30;
+
+export type RetrieveLegacyClosedGroupSubRequestType = {
+  method: 'retrieve';
+  params: {
+    namespace: -10; // legacy closed groups retrieve are not authenticated because the clients do not have a shared key
+  } & RetrieveAlwaysNeeded &
+    RetrieveMaxCountSize;
+};
+
 export type RetrieveSubKeySubRequestType = {
   method: 'retrieve';
   params: {
@@ -28,7 +45,10 @@ export type RetrieveSubKeySubRequestType = {
     RetrieveMaxCountSize;
 };
 
-export type RetrieveSubRequestType = RetrievePubkeySubRequestType | RetrieveSubKeySubRequestType;
+export type RetrieveSubRequestType =
+  | RetrieveLegacyClosedGroupSubRequestType
+  | RetrievePubkeySubRequestType
+  | RetrieveSubKeySubRequestType;
 
 // FIXME those store types are not right
 // type StoreAlwaysNeeded = { pubkey: string; timestamp: number; data: string };
@@ -38,7 +58,7 @@ export type RetrieveSubRequestType = RetrievePubkeySubRequestType | RetrieveSubK
 //   params: {
 //     ttl?: string; // required, unless expiry is given
 //     expiry?: number; // required, unless ttl is given
-//     namespace: -60 | -50 | -40 | -30 | -20 | -10 | 0 | 10 | 20 | 30 | 40 | 50 | 60; // we can only store without authentication on namespaces divisible by 10 (...-60...0...60...)
+//     namespace: UnauthenticatedNamespaces; // we can only store without authentication on namespaces divisible by 10 (...-60...0...60...)
 //   } & StoreAlwaysNeeded;
 // };
 
@@ -104,9 +124,11 @@ export type SnodeApiSubRequests =
   | StoreOnNodeSubRequest
   | NetworkTimeSubRequest;
 
+//tslint-disable: array-type
 export type NonEmptyArray<T> = [T, ...T[]];
 
 export type NotEmptyArrayOfBatchResults = NonEmptyArray<{
   code: number;
   body: Record<string, any>;
 }>;
+

ts/session/apis/snode_api/batchRequest.ts

@@ -10,8 +10,8 @@ import { NotEmptyArrayOfBatchResults, SnodeApiSubRequests } from './SnodeRequest
  * The body is already parsed from json and is enforced to be an Array of at least one element
  * @param subRequests the list of requests to do
  * @param targetNode the node to do the request to, once all the onion routing is done
- * @param timeout
+ * @param timeout the timeout at which we should cancel this request.
- * @param associatedWith
+ * @param associatedWith used mostly for handling 421 errors, we need the pubkey the change is associated to
  * @returns
  */
 export async function doSnodeBatchRequest(
@@ -45,13 +45,14 @@ export async function doSnodeBatchRequest(
  */
 function decodeBatchRequest(snodeResponse: SnodeResponse): NotEmptyArrayOfBatchResults {
   try {
+    console.warn('decodeBatch: ', snodeResponse);
     if (snodeResponse.status !== 200) {
       throw new Error(`decodeBatchRequest invalid status code: ${snodeResponse.status}`);
     }
     const parsed = JSON.parse(snodeResponse.body);
 
     if (!isArray(parsed.results)) {
-      throw new Error(`decodeBatchRequest results is not an array`);
+      throw new Error('decodeBatchRequest results is not an array');
     }
 
     if (!parsed.results.length) {

ts/session/apis/snode_api/onsResolve.ts

@@ -1,9 +1,9 @@
 import _, { range } from 'lodash';
 import { getSodiumRenderer } from '../../crypto';
 import {
-  stringToUint8Array,
-  fromUInt8ArrayToBase64,
   fromHexToArray,
+  fromUInt8ArrayToBase64,
+  stringToUint8Array,
   toHex,
 } from '../../utils/String';
 import { doSnodeBatchRequest } from './batchRequest';

ts/session/apis/snode_api/retrieveRequest.ts

@@ -1,35 +1,41 @@
-import { isEmpty } from 'lodash';
 import { Snode } from '../../../data/data';
 import { updateIsOnline } from '../../../state/ducks/onion';
 import { getSodiumRenderer } from '../../crypto';
-import { UserUtils, StringUtils } from '../../utils';
+import { StringUtils, UserUtils } from '../../utils';
 import { fromHexToArray, fromUInt8ArrayToBase64 } from '../../utils/String';
 import { doSnodeBatchRequest } from './batchRequest';
 import { GetNetworkTime } from './getNetworkTime';
-import { RetrievePubkeySubRequestType, RetrieveSubRequestType } from './SnodeRequestTypes';
+import {
+  RetrieveLegacyClosedGroupSubRequestType,
+  RetrieveSubRequestType,
+} from './SnodeRequestTypes';
 
 async function getRetrieveSignatureParams(params: {
   pubkey: string;
-  lastHash: string;
   namespace: number;
+  ourPubkey: string;
 }): Promise<{
   timestamp: number;
   signature: string;
   pubkey_ed25519: string;
   namespace: number;
-} | null> {
+}> {
-  const ourPubkey = UserUtils.getOurPubKeyFromCache();
   const ourEd25519Key = await UserUtils.getUserED25519KeyPair();
 
-  if (isEmpty(params?.pubkey) || ourPubkey.key !== params.pubkey || !ourEd25519Key) {
+  if (!ourEd25519Key) {
-    return null;
+    window.log.warn('getRetrieveSignatureParams: User has no getUserED25519KeyPair()');
+    throw new Error('getRetrieveSignatureParams: User has no getUserED25519KeyPair()');
   }
   const namespace = params.namespace || 0;
   const edKeyPrivBytes = fromHexToArray(ourEd25519Key?.privKey);
 
   const signatureTimestamp = GetNetworkTime.getNowWithNetworkOffset();
 
-  const verificationData = StringUtils.encode(`retrieve${namespace}${signatureTimestamp}`, 'utf8');
+  const verificationData =
+    namespace === 0
+      ? StringUtils.encode(`retrieve${signatureTimestamp}`, 'utf8')
+      : StringUtils.encode(`retrieve${namespace}${signatureTimestamp}`, 'utf8');
+
   const message = new Uint8Array(verificationData);
 
   const sodium = await getSodiumRenderer();
@@ -45,27 +51,56 @@ async function getRetrieveSignatureParams(params: {
     };
   } catch (e) {
     window.log.warn('getSignatureParams failed with: ', e.message);
-    return null;
+    throw e;
   }
 }
 
 async function buildRetrieveRequest(
   lastHashes: Array<string>,
   pubkey: string,
-  namespaces: Array<number>
+  namespaces: Array<number>,
+  ourPubkey: string
 ): Promise<Array<RetrieveSubRequestType>> {
   const retrieveRequestsParams = await Promise.all(
     namespaces.map(async (namespace, index) => {
       const retrieveParam = {
         pubkey,
-        lastHash: lastHashes.at(index) || '',
+        last_hash: lastHashes.at(index) || '',
         namespace,
+        timestamp: GetNetworkTime.getNowWithNetworkOffset(),
       };
-      const signatureBuilt = await getRetrieveSignatureParams(retrieveParam);
+
-      const signatureParams = signatureBuilt || {};
+      if (namespace === -10) {
-      const retrieve: RetrievePubkeySubRequestType = {
+        if (pubkey === ourPubkey || !pubkey.startsWith('05')) {
+          throw new Error(
+            'namespace -10 can only be used to retrieve messages from a legacy closed group'
+          );
+        }
+        const retrieveLegacyClosedGroup = {
+          ...retrieveParam,
+          namespace: namespace as -10,
+        };
+        const retrieveParamsLegacy: RetrieveLegacyClosedGroupSubRequestType = {
+          method: 'retrieve',
+          params: { ...retrieveLegacyClosedGroup },
+        };
+
+        return retrieveParamsLegacy;
+      }
+
+      // all legacy closed group retrieves are unauthenticated and run above.
+      // if we get here, this can only be a retrieve for our own swarm, which needs to be authenticated
+      if (namespace !== 0) {
+        throw new Error('not a legacy closed group. namespace can only be 0');
+      }
+      if (pubkey !== ourPubkey) {
+        throw new Error('not a legacy closed group. pubkey can only be our number');
+      }
+      const signatureArgs = { ...retrieveParam, ourPubkey };
+      const signatureBuilt = await getRetrieveSignatureParams(signatureArgs);
+      const retrieve: RetrieveSubRequestType = {
         method: 'retrieve',
-        params: { ...signatureParams, ...retrieveParam },
+        params: { ...retrieveParam, ...signatureBuilt },
       };
       return retrieve;
     })
@@ -79,16 +114,23 @@ async function retrieveNextMessages(
   targetNode: Snode,
   lastHashes: Array<string>,
   associatedWith: string,
-  namespaces: Array<number>
+  namespaces: Array<number>,
-): Promise<Array<any>> {
+  ourPubkey: string
+): Promise<Array<{ code: number; messages: Array<Record<string, any>> }>> {
   if (namespaces.length !== lastHashes.length) {
     throw new Error('namespaces and lasthashes does not match');
   }
 
-  const retrieveRequestsParams = await buildRetrieveRequest(lastHashes, associatedWith, namespaces);
+  const retrieveRequestsParams = await buildRetrieveRequest(
+    lastHashes,
+    associatedWith,
+    namespaces,
+    ourPubkey
+  );
   // let exceptions bubble up
   // no retry for this one as this a call we do every few seconds while polling for messages
 
+  console.warn('retrieveRequestsParams', retrieveRequestsParams);
   const results = await doSnodeBatchRequest(retrieveRequestsParams, targetNode, 4000);
 
   if (!results || !results.length) {
@@ -118,15 +160,18 @@ async function retrieveNextMessages(
     );
   }
 
+  console.warn('what should we do if we dont get a 200 on any of those fetches?');
+
   try {
-    const json = firstResult.body;
+    // we rely on the code of the
+    const bodyFirstResult = firstResult.body;
     if (!window.inboxStore?.getState().onionPaths.isOnline) {
       window.inboxStore?.dispatch(updateIsOnline(true));
     }
 
-    GetNetworkTime.handleTimestampOffsetFromNetwork('retrieve', json.t);
+    GetNetworkTime.handleTimestampOffsetFromNetwork('retrieve', bodyFirstResult.t);
 
-    return json.messages || [];
+    return results.map(result => ({ code: result.code, messages: result.body as Array<any> }));
   } catch (e) {
     window?.log?.warn('exception while parsing json of nextMessage:', e);
     if (!window.inboxStore?.getState().onionPaths.isOnline) {

ts/session/apis/snode_api/sessionRpc.ts

@@ -123,14 +123,14 @@ export async function snodeRpc(
 ): Promise<undefined | SnodeResponse> {
   const url = `https://${targetNode.ip}:${targetNode.port}/storage_rpc/v1`;
 
-
-
   const body = {
     jsonrpc: '2.0',
     method,
     params: clone(params),
   };
 
+  console.warn('snodeRPC', body);
+
   const fetchOptions: LokiFetchOptions = {
     method: 'POST',
     body: JSON.stringify(body),

ts/session/apis/snode_api/storeMessage.ts

@@ -17,7 +17,6 @@ async function storeOnNode(
 ): Promise<string | null | boolean> {
   try {
     const subRequests = buildStoreRequests(params);
-
     const result = await doSnodeBatchRequest(subRequests, targetNode, 4000, params.pubkey);
 
     if (!result || !result.length) {

ts/session/apis/snode_api/swarmPolling.ts

@@ -3,7 +3,7 @@ import * as snodePool from './snodePool';
 import { ERROR_CODE_NO_CONNECT } from './SNodeAPI';
 import { SignalService } from '../../../protobuf';
 import * as Receiver from '../../../receiver/receiver';
-import _, { concat } from 'lodash';
+import _, { concat, last } from 'lodash';
 import { Data, Snode } from '../../../data/data';
 
 import { StringUtils, UserUtils } from '../../utils';
@@ -142,6 +142,7 @@ export class SwarmPolling {
     }
     // we always poll as often as possible for our pubkey
     const ourPubkey = UserUtils.getOurPubKeyFromCache();
+    // const directPromises = Promise.resolve();
     const directPromises = Promise.all([this.pollOnceForKey(ourPubkey, false, [0])]).then(
       () => undefined
     );
@@ -243,6 +244,7 @@ export class SwarmPolling {
         return group;
       });
     } else if (isGroup) {
+      debugger;
       window?.log?.info(
         `Polled for group(${ed25519Str(
           pubkey.key
@@ -269,8 +271,11 @@ export class SwarmPolling {
     pubkey: PubKey,
     namespaces: Array<number>
   ): Promise<Array<any> | null> {
+    const namespaceLength = namespaces.length;
+    if (namespaceLength > 2 || namespaceLength <= 0) {
+      throw new Error('pollNodeForKey needs  1 or 2 namespaces to be given at all times');
+    }
     const edkey = node.pubkey_ed25519;
-
     const pkStr = pubkey.key;
 
     try {
@@ -279,27 +284,47 @@ export class SwarmPolling {
           const prevHashes = await Promise.all(
             namespaces.map(namespace => this.getLastHash(edkey, pkStr, namespace))
           );
-          const messages = await SnodeAPIRetrieve.retrieveNextMessages(
+          const results = await SnodeAPIRetrieve.retrieveNextMessages(
             node,
             prevHashes,
             pkStr,
-            namespaces
+            namespaces,
+            UserUtils.getOurPubKeyStrFromCache()
           );
-          if (!messages.length) {
+          debugger;
+          if (!results.length) {
             return [];
           }
 
-          // const lastMessage = _.last(messages);
+          if (results.length !== namespaceLength) {
-          // const newHashes = // TODO
+            window.log.error(
-
+              `pollNodeForKey asked for ${namespaceLength} namespaces but received only messages about ${results.length} namespaces`
-          // await this.updateLastHashes({
+            );
-          //   edkey: edkey,
+            throw new Error(
-          //   pubkey,
+              `pollNodeForKey asked for ${namespaceLength} namespaces but received only messages about ${results.length} namespaces`
-          //   namespaces,
+            );
-          //   hash: lastMessage.hash,
+          }
-          //   expiration: lastMessage.expiration,
+
-          // });
+          const lastMessages = results.map(r => {
-          return messages;
+            return last(r.messages);
+          });
+
+          await Promise.all(
+            lastMessages.map(async (lastMessage, index) => {
+              if (!lastMessage) {
+                return Promise.resolve();
+              }
+              return this.updateLastHash({
+                edkey: edkey,
+                pubkey,
+                namespace: namespaces[index],
+                hash: lastMessage.hash,
+                expiration: lastMessage.expiration,
+              });
+            })
+          );
+
+          return results;
         },
         {
           minTimeout: 100,
@@ -374,14 +399,17 @@ export class SwarmPolling {
     expiration: number;
   }): Promise<void> {
     const pkStr = pubkey.key;
-
+    const cached = await this.getLastHash(edkey, pubkey.key, namespace);
-    await Data.updateLastHash({
+
-      convoId: pkStr,
+    if (!cached || cached !== hash) {
-      snode: edkey,
+      await Data.updateLastHash({
-      hash,
+        convoId: pkStr,
-      expiresAt: expiration,
+        snode: edkey,
-      namespace,
+        hash,
-    });
+        expiresAt: expiration,
+        namespace,
+      });
+    }
 
     if (!this.lastHashes[edkey]) {
       this.lastHashes[edkey] = {};

ts/session/conversations/createClosedGroup.ts

@@ -13,8 +13,8 @@ import {
   generateGroupV3Keypair,
 } from '../crypto';
 import {
-  ClosedGroupNewMessageParams,
   ClosedGroupNewMessage,
+  ClosedGroupNewMessageParams,
 } from '../messages/outgoing/controlMessage/group/ClosedGroupNewMessage';
 import { PubKey } from '../types';
 import { UserUtils } from '../utils';

ts/session/types/PubKey.ts

@@ -1,6 +1,5 @@
 import { fromHexToArray } from '../utils/String';
 
-
 export enum KeyPrefixType {
   /**
    * Used for keys which have the blinding update and aren't using blinding

ts/session/utils/Bencoding.ts

@@ -66,7 +66,7 @@ export class BDecode {
     return parsed;
   }
 
-  parseList(): BencodeArrayType {
+  private parseList(): BencodeArrayType {
     const parsed: BencodeArrayType = [];
 
     if (this.currentParsingIndex >= this.content.length) {
@@ -134,7 +134,6 @@ export class BDecode {
       throw new Error('parseString: cannot parse string without length');
     }
 
-
     if (strLength === 0) {
       return '';
     }
@@ -193,7 +192,6 @@ export class BEncode {
   }
 
   private encodeItem(item: BencodeElementType): Uint8Array {
-
     if (isNumber(item) && isFinite(item)) {
       return from_string(`i${item}e`);
     }
@@ -216,6 +214,7 @@ export class BEncode {
 
     if (isArray(item)) {
       let content = new Uint8Array();
+      //tslint disable prefer-for-of
       for (let index = 0; index < item.length; index++) {
         const encodedItem = this.encodeItem(item[index]);
         const encodedItemLength = encodedItem.length;

ts/test/session/unit/messages/closed_groups/v3/GroupInviteMessage_test.ts

@@ -7,9 +7,7 @@ import { Constants } from '../../../../../../session';
 import { from_hex } from 'libsodium-wrappers-sumo';
 
 describe('GroupInviteMessage', () => {
-  beforeEach(async () => {});
+  it('can create valid message', () => {
-
-  it('can create valid message', async () => {
     const message = new GroupInviteMessage({
       timestamp: 12345,
       memberPrivateKey: '654321',

ts/test/session/unit/messages/closed_groups/v3/GroupMemberLeftMessage_test.ts

@@ -6,9 +6,7 @@ import { Constants } from '../../../../../../session';
 import { GroupMemberLeftMessage } from '../../../../../../session/messages/outgoing/controlMessage/group/v3/GroupMemberLeftMessage';
 
 describe('GroupMemberLeftMessage', () => {
-  beforeEach(async () => {});
+  it('can create valid message', () => {
-
-  it('can create valid message', async () => {
     const message = new GroupMemberLeftMessage({
       timestamp: 12345,
       identifier: v4(),
@@ -23,7 +21,7 @@ describe('GroupMemberLeftMessage', () => {
       .to.have.property('groupMessage')
       .to.have.property('memberLeftMessage').to.be.not.null;
 
-      expect(decoded.dataMessage)
+    expect(decoded.dataMessage)
       .to.have.property('groupMessage')
       .to.have.property('memberLeftMessage').to.be.empty;
     expect(message)
@@ -48,5 +46,4 @@ describe('GroupMemberLeftMessage', () => {
     expect(message.identifier).to.not.equal(null, 'identifier cannot be null');
     expect(message.identifier).to.not.equal(undefined, 'identifier cannot be undefined');
   });
-
 });

ts/test/session/unit/messages/closed_groups/v3/GroupPromoteMessage_test.ts

@@ -7,9 +7,7 @@ import { Constants } from '../../../../../../session';
 import { from_hex } from 'libsodium-wrappers-sumo';
 
 describe('GroupPromoteMessage', () => {
-  beforeEach(async () => {});
+  it('can create valid message', () => {
-
-  it('can create valid message', async () => {
     const message = new GroupPromoteMessage({
       timestamp: 12345,
       identifier: v4(),

ts/types/Util.ts

@@ -7,3 +7,5 @@ export type RenderTextCallbackType = (options: {
 }) => JSX.Element;
 
 export type LocalizerType = (key: LocalizerKeys, values?: Array<string>) => string;
+
+export type FixedLengthArray<T, Length extends number> = Array<T> & { length: Length };

tslint.json

@@ -25,6 +25,7 @@
     "object-literal-key-quotes": [true, "as-needed"],
     "object-literal-sort-keys": false,
     "no-async-without-await": true,
+    "no-empty-interface": false,
     "ordered-imports": [
       true,
       {