From 8618cf75e915a65e624822f9342ea9a886c72439 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 4 Nov 2021 11:36:39 +1100 Subject: [PATCH] send and handle uuid for multi device calls --- protos/SignalService.proto | 4 + .../calling/InConversationCallContainer.tsx | 2 +- .../outgoing/controlMessage/CallMessage.ts | 8 + ts/session/utils/CallManager.ts | 164 ++++++++++++++---- 4 files changed, 146 insertions(+), 32 deletions(-) diff --git a/protos/SignalService.proto b/protos/SignalService.proto index a2891f4fd..5234f108d 100644 --- a/protos/SignalService.proto +++ b/protos/SignalService.proto @@ -157,6 +157,7 @@ message DataMessage { message CallMessage { enum Type { + PRE_OFFER = 6; OFFER = 1; ANSWER = 2; PROVISIONAL_ANSWER = 3; @@ -170,6 +171,9 @@ message CallMessage { repeated uint32 sdpMLineIndexes = 3; repeated string sdpMids = 4; + // @required + required string uuid = 5; + } message ConfigurationMessage { diff --git a/ts/components/session/calling/InConversationCallContainer.tsx b/ts/components/session/calling/InConversationCallContainer.tsx index db9395d2d..bf92757ae 100644 --- a/ts/components/session/calling/InConversationCallContainer.tsx +++ b/ts/components/session/calling/InConversationCallContainer.tsx @@ -164,7 +164,7 @@ const HangUpButton = () => { const handleEndCall = async () => { // call method to end call connection if (ongoingCallPubkey) { - await CallManager.USER_rejectIncomingCallRequest(ongoingCallPubkey); + await CallManager.USER_hangup(ongoingCallPubkey); } }; diff --git a/ts/session/messages/outgoing/controlMessage/CallMessage.ts b/ts/session/messages/outgoing/controlMessage/CallMessage.ts index 307ea9991..eba624b44 100644 --- a/ts/session/messages/outgoing/controlMessage/CallMessage.ts +++ b/ts/session/messages/outgoing/controlMessage/CallMessage.ts @@ -8,6 +8,7 @@ interface CallMessageParams extends MessageParams { sdpMLineIndexes?: Array; sdpMids?: Array; sdps?: Array; + uuid: string; } export class CallMessage extends ContentMessage { @@ -15,6 +16,7 @@ export class CallMessage extends ContentMessage { public readonly sdpMLineIndexes?: Array; public readonly sdpMids?: Array; public readonly sdps?: Array; + public readonly uuid: string; constructor(params: CallMessageParams) { super({ timestamp: params.timestamp, identifier: params.identifier }); @@ -22,6 +24,8 @@ export class CallMessage extends ContentMessage { this.sdpMLineIndexes = params.sdpMLineIndexes; this.sdpMids = params.sdpMids; this.sdps = params.sdps; + this.uuid = params.uuid; + // this does not make any sense if ( this.type !== signalservice.CallMessage.Type.END_CALL && @@ -29,6 +33,9 @@ export class CallMessage extends ContentMessage { ) { throw new Error('sdps must be set unless this is a END_CALL type message'); } + if (this.uuid.length === 0) { + throw new Error('uuid must cannot be empty'); + } } public contentProto(): SignalService.Content { @@ -47,6 +54,7 @@ export class CallMessage extends ContentMessage { sdpMLineIndexes: this.sdpMLineIndexes, sdpMids: this.sdpMids, sdps: this.sdps, + uuid: this.uuid, }); } } diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index b1122aa46..0b34b770f 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -19,8 +19,12 @@ import { ed25519Str } from '../onions/onionPath'; import { getMessageQueue } from '../sending'; import { PubKey } from '../types'; +import { v4 as uuidv4 } from 'uuid'; + export type InputItem = { deviceId: string; label: string }; +let currentCallUUID: string | undefined; + // const VIDEO_WIDTH = 640; // const VIDEO_RATIO = 16 / 9; @@ -72,9 +76,9 @@ export function removeVideoEventsListener(uniqueId: string) { } /** - * This field stores all the details received by a sender about a call in separate messages. + * This field stores all the details received about a specific call with the same uuid. It is a per pubkey and per device cache. */ -const callCache = new Map>(); +const callCache = new Map>>(); let peerConnection: RTCPeerConnection | null; let dataChannel: RTCDataChannel | null; @@ -265,11 +269,17 @@ async function handleNegotiationNeededEvent(_event: Event, recipient: string) { } await peerConnection?.setLocalDescription(offer); + if (!currentCallUUID) { + window.log.warn('cannot send offer without a currentCallUUID'); + throw new Error('cannot send offer without a currentCallUUID'); + } + if (offer && offer.sdp) { const offerMessage = new CallMessage({ timestamp: Date.now(), type: SignalService.CallMessage.Type.OFFER, sdps: [offer.sdp], + uuid: currentCallUUID, }); window.log.info('sending OFFER MESSAGE'); @@ -349,12 +359,19 @@ export async function USER_callRecipient(recipient: string) { ToastUtils.pushVideoCallPermissionNeeded(); return; } + if (currentCallUUID) { + window.log.warn( + 'Looks like we are already in a call as in USER_callRecipient is not undefined' + ); + return; + } await updateInputLists(); window?.log?.info(`starting call with ${ed25519Str(recipient)}..`); window.inboxStore?.dispatch(startingCallWith({ pubkey: recipient })); if (peerConnection) { throw new Error('USER_callRecipient peerConnection is already initialized '); } + currentCallUUID = uuidv4(); peerConnection = createOrGetPeerConnection(recipient, true); await openMediaDevicesAndAddTracks(); } @@ -381,12 +398,17 @@ const iceSenderDebouncer = _.debounce(async (recipient: string) => { return null; }) ); + if (!currentCallUUID) { + window.log.warn('Cannot send ice candidates without a currentCallUUID'); + return; + } const callIceCandicates = new CallMessage({ timestamp: Date.now(), type: SignalService.CallMessage.Type.ICE_CANDIDATES, sdpMLineIndexes: validCandidates.map(c => c.sdpMLineIndex), sdpMids: validCandidates.map(c => c.sdpMid), sdps: validCandidates.map(c => c.candidate), + uuid: currentCallUUID, }); window.log.info('sending ICE CANDIDATES MESSAGE to ', recipient); @@ -395,11 +417,15 @@ const iceSenderDebouncer = _.debounce(async (recipient: string) => { }, 2000); const findLastMessageTypeFromSender = (sender: string, msgType: SignalService.CallMessage.Type) => { - const msgCacheFromSender = callCache.get(sender); - if (!msgCacheFromSender) { + const msgCacheFromSenderWithDevices = callCache.get(sender); + if (!msgCacheFromSenderWithDevices) { return undefined; } - const lastOfferMessage = _.findLast(msgCacheFromSender, m => m.type === msgType); + + // FIXME this does not sort by timestamp as we do not have a timestamp stored in the SignalService.CallMessage object... + const allMsg = _.flattenDeep([...msgCacheFromSenderWithDevices.values()]); + const allMsgFromType = allMsg.filter(m => m.type === msgType); + const lastOfferMessage = _.last(allMsgFromType); if (!lastOfferMessage) { return undefined; @@ -458,6 +484,7 @@ function closeVideoCall() { remoteStream = null; selectedCameraId = INPUT_DISABLED_DEVICE_ID; selectedAudioInputId = INPUT_DISABLED_DEVICE_ID; + currentCallUUID = undefined; callVideoListeners(); window.inboxStore?.dispatch(setFullScreenCall(false)); } @@ -480,6 +507,7 @@ function onDataChannelReceivedMessage(ev: MessageEvent) { return; } handleCallTypeEndCall(foundEntry.id); + return; } @@ -550,14 +578,14 @@ function createOrGetPeerConnection(withPubkey: string, createDataChannel: boolea // tslint:disable-next-line: function-name export async function USER_acceptIncomingCallRequest(fromSender: string) { - const msgCacheFromSender = callCache.get(fromSender); - await updateInputLists(); - if (!msgCacheFromSender) { - window?.log?.info( - 'incoming call request cannot be accepted as the corresponding message is not found' + if (currentCallUUID) { + window.log.warn( + 'Looks like we are already in a call as in USER_acceptIncomingCallRequest is not undefined' ); return; } + await updateInputLists(); + const lastOfferMessage = findLastMessageTypeFromSender( fromSender, SignalService.CallMessage.Type.OFFER @@ -578,6 +606,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { peerConnection = createOrGetPeerConnection(fromSender, false); await openMediaDevicesAndAddTracks(); + currentCallUUID = uuidv4(); const { sdps } = lastOfferMessage; if (!sdps || sdps.length === 0) { @@ -617,15 +646,49 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) { const endCallMessage = new CallMessage({ type: SignalService.CallMessage.Type.END_CALL, timestamp: Date.now(), + uuid: uuidv4(), // just send a random thing, we just want to reject the call + }); + // delete all msg not from that uuid only but from that sender pubkey + + window.inboxStore?.dispatch( + endCall({ + pubkey: fromSender, + }) + ); + window.log.info('USER_rejectIncomingCallRequest'); + clearCallCacheFromPubkey(fromSender); + + await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), endCallMessage); + + const convos = getConversationController().getConversations(); + const callingConvos = convos.filter(convo => convo.callState !== undefined); + if (callingConvos.length > 0) { + // we just got a new offer from someone we are already in a call with + if (callingConvos.length === 1 && callingConvos[0].id === fromSender) { + closeVideoCall(); + } + } +} + +// tslint:disable-next-line: function-name +export async function USER_hangup(fromSender: string) { + if (!currentCallUUID) { + window.log.warn('cannot hangup without a currentCallUUID'); + return; + } + const endCallMessage = new CallMessage({ + type: SignalService.CallMessage.Type.END_CALL, + timestamp: Date.now(), + uuid: currentCallUUID, }); - callCache.delete(fromSender); window.inboxStore?.dispatch(endCall({ pubkey: fromSender })); - window.log.info('sending END_CALL MESSAGE'); + window.log.info('sending hangup with an END_CALL MESSAGE'); sendHangupViaDataChannel(); await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), endCallMessage); + clearCallCacheFromPubkey(fromSender); const convos = getConversationController().getConversations(); const callingConvos = convos.filter(convo => convo.callState !== undefined); @@ -638,7 +701,8 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) { } export function handleCallTypeEndCall(sender: string) { - callCache.delete(sender); + clearCallCacheFromPubkey(sender); + window.log.info('handling callMessage END_CALL'); const convos = getConversationController().getConversations(); @@ -655,6 +719,11 @@ export function handleCallTypeEndCall(sender: string) { async function buildAnswerAndSendIt(sender: string) { if (peerConnection) { + if (!currentCallUUID) { + window.log.warn('cannot send answer without a currentCallUUID'); + return; + } + const answer = await peerConnection.createAnswer({ offerToReceiveAudio: true, offerToReceiveVideo: true, @@ -669,6 +738,7 @@ async function buildAnswerAndSendIt(sender: string) { timestamp: Date.now(), type: SignalService.CallMessage.Type.ANSWER, sdps: [answerSdp], + uuid: currentCallUUID, }); window.log.info('sending ANSWER MESSAGE'); @@ -683,7 +753,11 @@ export async function handleCallTypeOffer( incomingOfferTimestamp: number ) { try { - window.log.info('handling callMessage OFFER'); + const remoteCallUUID = callMessage.uuid; + if (!remoteCallUUID || remoteCallUUID.length === 0) { + throw new Error('incoming offer call has no valid uuid'); + } + window.log.info('handling callMessage OFFER with uuid: ', remoteCallUUID); const convos = getConversationController().getConversations(); const callingConvos = convos.filter(convo => convo.callState !== undefined); @@ -727,15 +801,10 @@ export async function handleCallTypeOffer( } window.inboxStore?.dispatch(incomingCall({ pubkey: sender })); - // don't need to do the sending here as we dispatch an answer in a + pushCallMessageToCallCache(sender, remoteCallUUID, callMessage); } catch (err) { window.log?.error(`Error handling offer message ${err}`); } - - if (!callCache.has(sender)) { - callCache.set(sender, new Array()); - } - callCache.get(sender)?.push(callMessage); } async function handleMissedCall( @@ -778,14 +847,15 @@ export async function handleCallTypeAnswer(sender: string, callMessage: SignalSe window.log.warn('cannot handle answered message without signal description protols'); return; } + const remoteCallUUID = callMessage.uuid; + if (!remoteCallUUID || remoteCallUUID.length === 0) { + window.log.warn('handleCallTypeAnswer has no valid uuid'); + return; + } window.log.info('handling callMessage ANSWER'); - if (!callCache.has(sender)) { - callCache.set(sender, new Array()); - } - - callCache.get(sender)?.push(callMessage); + pushCallMessageToCallCache(sender, remoteCallUUID, callMessage); if (!peerConnection) { window.log.info('handleCallTypeAnswer without peer connection. Dropping'); @@ -808,13 +878,14 @@ export async function handleCallTypeIceCandidates( window.log.warn('cannot handle iceCandicates message without candidates'); return; } - window.log.info('handling callMessage ICE_CANDIDATES'); - - if (!callCache.has(sender)) { - callCache.set(sender, new Array()); + const remoteCallUUID = callMessage.uuid; + if (!remoteCallUUID || remoteCallUUID.length === 0) { + window.log.warn('handleCallTypeIceCandidates has no valid uuid'); + return; } + window.log.info('handling callMessage ICE_CANDIDATES'); - callCache.get(sender)?.push(callMessage); + pushCallMessageToCallCache(sender, remoteCallUUID, callMessage); await addIceCandidateToExistingPeerConnection(callMessage); } @@ -841,5 +912,36 @@ async function addIceCandidateToExistingPeerConnection(callMessage: SignalServic // tslint:disable-next-line: no-async-without-await export async function handleOtherCallTypes(sender: string, callMessage: SignalService.CallMessage) { - callCache.get(sender)?.push(callMessage); + const remoteCallUUID = callMessage.uuid; + if (!remoteCallUUID || remoteCallUUID.length === 0) { + window.log.warn('handleOtherCallTypes has no valid uuid'); + return; + } + pushCallMessageToCallCache(sender, remoteCallUUID, callMessage); +} + +function clearCallCacheFromPubkey(sender: string) { + callCache.delete(sender); +} + +function createCallCacheForPubkeyAndUUID(sender: string, uuid: string) { + if (!callCache.has(sender)) { + callCache.set(sender, new Map()); + } + + if (!callCache.get(sender)?.has(uuid)) { + callCache.get(sender)?.set(uuid, new Array()); + } +} + +function pushCallMessageToCallCache( + sender: string, + uuid: string, + callMessage: SignalService.CallMessage +) { + createCallCacheForPubkeyAndUUID(sender, uuid); + callCache + .get(sender) + ?.get(uuid) + ?.push(callMessage); }