From a629985a4f8a32d86d0a58a751e85b7552a5b258 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Tue, 12 Oct 2021 13:51:05 +1100 Subject: [PATCH] fix toggling input source and off in video calls --- ts/receiver/callMessage.ts | 35 +-- ts/session/crypto/BufferPadding.ts | 2 +- ts/session/snode_api/SNodeAPI.ts | 2 +- ts/session/utils/CallManager.ts | 398 +++++++++++++++++------------ 4 files changed, 249 insertions(+), 188 deletions(-) diff --git a/ts/receiver/callMessage.ts b/ts/receiver/callMessage.ts index 7fa2ee413..d9310ea89 100644 --- a/ts/receiver/callMessage.ts +++ b/ts/receiver/callMessage.ts @@ -18,25 +18,14 @@ export async function handleCallMessage( const sentTimestamp = _.toNumber(envelope.timestamp); const { type } = callMessage; - switch (type) { - case SignalService.CallMessage.Type.END_CALL: - window.log.info('handling callMessage END_CALL'); - break; - case SignalService.CallMessage.Type.ANSWER: - window.log.info('handling callMessage ANSWER'); - break; - case SignalService.CallMessage.Type.ICE_CANDIDATES: - window.log.info('handling callMessage ICE_CANDIDATES'); - break; - case SignalService.CallMessage.Type.OFFER: - window.log.info('handling callMessage OFFER'); - break; - case SignalService.CallMessage.Type.PROVISIONAL_ANSWER: - window.log.info('handling callMessage PROVISIONAL_ANSWER'); - break; - default: - window.log.info('handling callMessage unknown'); + + if (type === SignalService.CallMessage.Type.PROVISIONAL_ANSWER) { + await removeFromCache(envelope); + + window.log.info('Skipping callMessage PROVISIONAL_ANSWER'); + return; } + if (type === SignalService.CallMessage.Type.OFFER) { if (Math.max(sentTimestamp - (Date.now() - currentOffset)) > TTL_DEFAULT.CALL_MESSAGE) { window?.log?.info('Dropping incoming OFFER callMessage sent a while ago: ', sentTimestamp); @@ -46,7 +35,7 @@ export async function handleCallMessage( } await removeFromCache(envelope); - await CallManager.handleOfferCallMessage(sender, callMessage, sentTimestamp); + await CallManager.handleCallTypeOffer(sender, callMessage, sentTimestamp); return; } @@ -54,7 +43,7 @@ export async function handleCallMessage( if (type === SignalService.CallMessage.Type.END_CALL) { await removeFromCache(envelope); - CallManager.handleEndCallMessage(sender); + CallManager.handleCallTypeEndCall(sender); return; } @@ -62,19 +51,19 @@ export async function handleCallMessage( if (type === SignalService.CallMessage.Type.ANSWER) { await removeFromCache(envelope); - await CallManager.handleCallAnsweredMessage(sender, callMessage); + await CallManager.handleCallTypeAnswer(sender, callMessage); return; } if (type === SignalService.CallMessage.Type.ICE_CANDIDATES) { await removeFromCache(envelope); - await CallManager.handleIceCandidatesMessage(sender, callMessage); + await CallManager.handleCallTypeIceCandidates(sender, callMessage); return; } await removeFromCache(envelope); // if this another type of call message, just add it to the manager - await CallManager.handleOtherCallMessage(sender, callMessage); + await CallManager.handleOtherCallTypes(sender, callMessage); } diff --git a/ts/session/crypto/BufferPadding.ts b/ts/session/crypto/BufferPadding.ts index 85c316c52..1793b82eb 100644 --- a/ts/session/crypto/BufferPadding.ts +++ b/ts/session/crypto/BufferPadding.ts @@ -32,7 +32,7 @@ export function removeMessagePadding(paddedData: ArrayBuffer): ArrayBuffer { * @param messageBuffer The buffer to add padding to. */ export function addMessagePadding(messageBuffer: Uint8Array): Uint8Array { - window?.log?.info('Adding message padding...'); + // window?.log?.info('Adding message padding...'); const plaintext = new Uint8Array(getPaddedMessageLength(messageBuffer.byteLength + 1) - 1); plaintext.set(new Uint8Array(messageBuffer)); diff --git a/ts/session/snode_api/SNodeAPI.ts b/ts/session/snode_api/SNodeAPI.ts index cad1cc906..1febb9d14 100644 --- a/ts/session/snode_api/SNodeAPI.ts +++ b/ts/session/snode_api/SNodeAPI.ts @@ -44,7 +44,7 @@ export function getLatestTimestampOffset() { window.log.warn('latestTimestampOffset is not set yet'); return 0; } - window.log.info('latestTimestampOffset is ', latestTimestampOffset); + // window.log.info('latestTimestampOffset is ', latestTimestampOffset); return latestTimestampOffset; } diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index f519e1e8f..13f3a4554 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -19,6 +19,9 @@ import { getMessageQueue } from '../sending'; import { PubKey } from '../types'; export type InputItem = { deviceId: string; label: string }; +// const VIDEO_WIDTH = 640; +// const VIDEO_RATIO = 16 / 9; + type CallManagerListener = | (( localStream: MediaStream | null, @@ -48,6 +51,7 @@ const callCache = new Map>(); let peerConnection: RTCPeerConnection | null; let remoteStream: MediaStream | null; let mediaDevices: MediaStream | null; +const INPUT_DISABLED_DEVICE_ID = 'off'; let makingOffer = false; let ignoreOffer = false; @@ -87,61 +91,101 @@ navigator.mediaDevices.addEventListener('devicechange', async () => { async function updateInputLists() { // Get the set of cameras connected const videoCameras = await getConnectedDevices('videoinput'); - camerasList = videoCameras.map(m => ({ - deviceId: m.deviceId, - label: m.label, - })); + + camerasList = [{ deviceId: INPUT_DISABLED_DEVICE_ID, label: 'Off' }].concat( + videoCameras.map(m => ({ + deviceId: m.deviceId, + label: m.label, + })) + ); + // Get the set of audio inputs connected const audiosInput = await getConnectedDevices('audioinput'); - audioInputsList = audiosInput.map(m => ({ - deviceId: m.deviceId, - label: m.label, - })); + audioInputsList = [{ deviceId: INPUT_DISABLED_DEVICE_ID, label: 'Off' }].concat( + audiosInput.map(m => ({ + deviceId: m.deviceId, + label: m.label, + })) + ); } export async function selectCameraByDeviceId(cameraDeviceId: string) { - console.warn('selecting cameraDeviceId ', cameraDeviceId); - if (camerasList.some(m => m.deviceId === cameraDeviceId)) { selectedCameraId = cameraDeviceId; - try { - mediaDevices = await openMediaDevices({ - audioInputId: selectedAudioInputId, - cameraId: selectedCameraId, + if (selectedCameraId === INPUT_DISABLED_DEVICE_ID) { + const sender = peerConnection?.getSenders().find(s => { + return s.track?.kind === 'video'; }); + if (sender?.track) { + sender.track.enabled = false; + } + return; + } + const devicesConfig = { + video: { + deviceId: selectedCameraId ? { exact: selectedCameraId } : undefined, + // width: VIDEO_WIDTH, + // height: Math.floor(VIDEO_WIDTH * VIDEO_RATIO), + }, + }; - mediaDevices.getTracks().map((track: MediaStreamTrack) => { - window.log.info('selectCameraByDeviceId adding track: ', track); - if (mediaDevices) { - peerConnection?.addTrack(track, mediaDevices); - } + try { + const newVideoStream = await navigator.mediaDevices.getUserMedia(devicesConfig); + const videoTrack = newVideoStream.getVideoTracks()[0]; + if (!peerConnection) { + throw new Error('cannot selectCameraByDeviceId without a peer connection'); + } + const sender = peerConnection.getSenders().find(s => { + return s.track?.kind === videoTrack.kind; }); - callVideoListener(); - } catch (err) { - console.warn('err', err); + if (sender) { + await sender.replaceTrack(videoTrack); + videoTrack.enabled = true; + mediaDevices?.getVideoTracks().forEach(t => { + t.stop(); + mediaDevices?.removeTrack(t); + }); + mediaDevices?.addTrack(videoTrack); + } else { + throw new Error('Failed to get sender for selectCameraByDeviceId '); + } + } catch (e) { + window.log.warn('selectCameraByDeviceId failed with', e.message); } } } export async function selectAudioInputByDeviceId(audioInputDeviceId: string) { - console.warn('selecting audioInputDeviceId', audioInputDeviceId); if (audioInputsList.some(m => m.deviceId === audioInputDeviceId)) { selectedAudioInputId = audioInputDeviceId; + const devicesConfig = { + audio: { + deviceId: selectedAudioInputId ? { exact: selectedAudioInputId } : undefined, + }, + }; + try { - mediaDevices = await openMediaDevices({ - audioInputId: selectedAudioInputId, - cameraId: selectedCameraId, + const newAudioStream = await navigator.mediaDevices.getUserMedia(devicesConfig); + const audioTrack = newAudioStream.getAudioTracks()[0]; + if (!peerConnection) { + throw new Error('cannot selectAudioInputByDeviceId without a peer connection'); + } + const sender = peerConnection.getSenders().find(s => { + return s.track?.kind === audioTrack.kind; }); - mediaDevices.getTracks().map((track: MediaStreamTrack) => { - window.log.info('selectAudioInputByDeviceId adding track: ', track); - if (mediaDevices) { - peerConnection?.addTrack(track, mediaDevices); - } - }); - callVideoListener(); - } catch (err) { - console.warn('err', err); + if (sender) { + await sender.replaceTrack(audioTrack); + mediaDevices?.getAudioTracks().forEach(t => { + t.stop(); + mediaDevices?.removeTrack(t); + }); + mediaDevices?.addTrack(audioTrack); + } else { + throw new Error('Failed to get sender for selectAudioInputByDeviceId '); + } + } catch (e) { + window.log.warn('selectAudioInputByDeviceId failed with', e.message); } } } @@ -151,14 +195,13 @@ async function handleNegotiationNeededEvent(event: Event, recipient: string) { try { makingOffer = true; const offer = await peerConnection?.createOffer(); - if (!offer) { throw new Error('Could not create offer in handleNegotiationNeededEvent'); } await peerConnection?.setLocalDescription(offer); if (offer && offer.sdp) { - const negotationOfferMessage = new CallMessage({ + const offerMessage = new CallMessage({ timestamp: Date.now(), type: SignalService.CallMessage.Type.OFFER, sdps: [offer.sdp], @@ -167,10 +210,10 @@ async function handleNegotiationNeededEvent(event: Event, recipient: string) { window.log.info('sending OFFER MESSAGE'); const negotationOfferSendResult = await getMessageQueue().sendToPubKeyNonDurably( PubKey.cast(recipient), - negotationOfferMessage + offerMessage ); if (typeof negotationOfferSendResult === 'number') { - window.log?.warn('setting last sent timestamp'); + // window.log?.warn('setting last sent timestamp'); lastOutgoingOfferTimestamp = negotationOfferSendResult; } } @@ -188,25 +231,47 @@ function handleIceCandidates(event: RTCPeerConnectionIceEvent, pubkey: string) { } } -// tslint:disable-next-line: function-name -export async function USER_callRecipient(recipient: string) { - await updateInputLists(); - window?.log?.info(`starting call with ${ed25519Str(recipient)}..`); - window.inboxStore?.dispatch(startingCallWith({ pubkey: recipient })); - if (peerConnection) { - window.log.info('closing existing peerconnection'); - peerConnection.close(); - peerConnection = null; - } - peerConnection = new RTCPeerConnection(configuration); - +async function openMediaDevicesAndAddTracks() { try { - mediaDevices = await openMediaDevices({}); + await updateInputLists(); + if (!camerasList.length) { + ToastUtils.pushNoCameraFound(); + return; + } + if (!audioInputsList.length) { + ToastUtils.pushNoAudioInputFound(); + return; + } - mediaDevices.getTracks().map((track: any) => { - window.log.info('USER_callRecipient adding track: ', track); + const firstAudio = audioInputsList[0].deviceId; + const firstVideo = camerasList[0].deviceId; + window.log.info(`openMediaDevices video:${firstVideo} audio:${firstAudio}`); + + const devicesConfig = { + audio: { + deviceId: firstAudio, + + echoCancellation: true, + }, + video: { + deviceId: firstVideo, + // width: VIDEO_WIDTH, + // height: Math.floor(VIDEO_WIDTH * VIDEO_RATIO), + }, + }; + + mediaDevices = await navigator.mediaDevices.getUserMedia(devicesConfig); + mediaDevices.getTracks().map(track => { + if (track.kind === 'video') { + track.enabled = false; + } if (mediaDevices) { + // FIXME audric why does this fails? + // track.onunmute = () => { + // if (mediaDevices) { peerConnection?.addTrack(track, mediaDevices); + // } + // }; } }); } catch (err) { @@ -215,27 +280,19 @@ export async function USER_callRecipient(recipient: string) { window.inboxStore?.dispatch(showSettingsSection(SessionSettingCategory.Privacy)); }); } - peerConnection.addEventListener('connectionstatechange', () => { - handleConnectionStateChanged(recipient); - }); - - peerConnection.addEventListener('icecandidate', event => { - handleIceCandidates(event, recipient); - }); - peerConnection.onnegotiationneeded = async (event: Event) => { - await handleNegotiationNeededEvent(event, recipient); - }; - - remoteStream = new MediaStream(); - callVideoListener(); +} - peerConnection.addEventListener('track', event => { - callVideoListener(); - if (remoteStream) { - remoteStream.addTrack(event.track); - } - }); +// tslint:disable-next-line: function-name +export async function USER_callRecipient(recipient: string) { + await updateInputLists(); + window?.log?.info(`starting call with ${ed25519Str(recipient)}..`); + window.inboxStore?.dispatch(startingCallWith({ pubkey: recipient })); + if (peerConnection) { + throw new Error('USER_callRecipient peerConnection is already initialized '); + } + peerConnection = createOrGetPeerConnection(recipient); + await openMediaDevicesAndAddTracks(); } const iceCandidates: Array = new Array(); @@ -272,32 +329,6 @@ const iceSenderDebouncer = _.debounce(async (recipient: string) => { await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(recipient), callIceCandicates); }, 2000); -const openMediaDevices = async ({ - audioInputId, - cameraId, -}: { - cameraId?: string; - audioInputId?: string; -}) => { - if (mediaDevices) { - window.log.info('stopping existing tracks in openMediaDevices'); - mediaDevices.getTracks().forEach(track => { - track.stop(); - }); - } - window.log.info('openMediaDevices ', { audioInputId, cameraId }); - - return navigator.mediaDevices.getUserMedia({ - audio: { - deviceId: audioInputId ? { exact: audioInputId } : undefined, - echoCancellation: true, - }, - video: { - deviceId: cameraId ? { exact: cameraId } : undefined, - }, - }); -}; - const findLastMessageTypeFromSender = (sender: string, msgType: SignalService.CallMessage.Type) => { const msgCacheFromSender = callCache.get(sender); if (!msgCacheFromSender) { @@ -328,6 +359,7 @@ function handleConnectionStateChanged(pubkey: string) { } function closeVideoCall() { + window.log.info('closingVideoCall ', peerConnection); if (peerConnection) { peerConnection.ontrack = null; peerConnection.onicecandidate = null; @@ -345,7 +377,7 @@ function closeVideoCall() { if (remoteStream) { remoteStream.getTracks().forEach(track => { - track.stop(); + remoteStream?.removeTrack(track); }); } @@ -353,11 +385,46 @@ function closeVideoCall() { peerConnection = null; } + mediaDevices = null; + remoteStream = null; if (videoEventsListener) { videoEventsListener(null, null, [], []); } } +function createOrGetPeerConnection(withPubkey: string) { + if (peerConnection) { + return peerConnection; + } + remoteStream = new MediaStream(); + peerConnection = new RTCPeerConnection(configuration); + + peerConnection.onnegotiationneeded = async (event: Event) => { + await handleNegotiationNeededEvent(event, withPubkey); + }; + peerConnection.onsignalingstatechange = handleSignalingStateChangeEvent; + + peerConnection.ontrack = event => { + event.track.onunmute = () => { + remoteStream?.addTrack(event.track); + callVideoListener(); + }; + event.track.onmute = () => { + remoteStream?.removeTrack(event.track); + callVideoListener(); + }; + }; + peerConnection.onconnectionstatechange = () => { + handleConnectionStateChanged(withPubkey); + }; + + peerConnection.onicecandidate = event => { + handleIceCandidates(event, withPubkey); + }; + + return peerConnection; +} + // tslint:disable-next-line: function-name export async function USER_acceptIncomingCallRequest(fromSender: string) { const msgCacheFromSender = callCache.get(fromSender); @@ -382,38 +449,12 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { window.inboxStore?.dispatch(answerCall({ pubkey: fromSender })); if (peerConnection) { - window.log.info('closing existing peerconnection'); - peerConnection.close(); - peerConnection = null; + throw new Error('USER_acceptIncomingCallRequest: peerConnection is already set.'); } - peerConnection = new RTCPeerConnection(configuration); - mediaDevices = await openMediaDevices({}); - mediaDevices.getTracks().map(track => { - // window.log.info('USER_acceptIncomingCallRequest adding track ', track); - if (mediaDevices) { - peerConnection?.addTrack(track, mediaDevices); - } - }); - remoteStream = new MediaStream(); - peerConnection.addEventListener('icecandidate', event => { - if (event.candidate) { - iceCandidates.push(event.candidate); - void iceSenderDebouncer(fromSender); - } - }); + peerConnection = createOrGetPeerConnection(fromSender); - peerConnection.addEventListener('signalingstatechange', handleSignalingStateChangeEvent); - - callVideoListener(); - - peerConnection.addEventListener('track', event => { - callVideoListener(); - remoteStream?.addTrack(event.track); - }); - peerConnection.addEventListener('connectionstatechange', () => { - handleConnectionStateChanged(fromSender); - }); + await openMediaDevicesAndAddTracks(); const { sdps } = lastOfferMessage; if (!sdps || sdps.length === 0) { @@ -430,22 +471,6 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { window.log?.error(`Error setting RTC Session Description ${e}`); } - const answer = await peerConnection.createAnswer({ - offerToReceiveAudio: true, - offerToReceiveVideo: true, - }); - if (!answer?.sdp || answer.sdp.length === 0) { - window.log.warn('failed to create answer'); - return; - } - await peerConnection.setLocalDescription(answer); - const answerSdp = answer.sdp; - const callAnswerMessage = new CallMessage({ - timestamp: Date.now(), - type: SignalService.CallMessage.Type.ANSWER, - sdps: [answerSdp], - }); - const lastCandidatesFromSender = findLastMessageTypeFromSender( fromSender, SignalService.CallMessage.Type.ICE_CANDIDATES @@ -461,9 +486,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { await peerConnection.addIceCandidate(candicate); } } - window.log.info('sending ANSWER MESSAGE'); - - await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), callAnswerMessage); + await buildAnswerAndSendIt(fromSender); } // tslint:disable-next-line: function-name @@ -478,32 +501,80 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) { window.log.info('sending END_CALL MESSAGE'); await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), endCallMessage); + + const convos = getConversationController().getConversations(); + const callingConvos = convos.filter(convo => convo.callState !== undefined); + if (callingConvos.length > 0) { + // we just got a new offer from someone we are already in a call with + if (callingConvos.length === 1 && callingConvos[0].id === fromSender) { + closeVideoCall(); + } + } } -export function handleEndCallMessage(sender: string) { +export function handleCallTypeEndCall(sender: string) { callCache.delete(sender); + window.log.info('handling callMessage END_CALL'); + if (videoEventsListener) { videoEventsListener(null, null, [], []); } - mediaDevices = null; - remoteStream = null; + closeVideoCall(); // // FIXME audric trigger UI cleanup window.inboxStore?.dispatch(endCall({ pubkey: sender })); } -export async function handleOfferCallMessage( +async function buildAnswerAndSendIt(sender: string) { + if (peerConnection) { + const answer = await peerConnection.createAnswer({ + offerToReceiveAudio: true, + offerToReceiveVideo: true, + }); + if (!answer?.sdp || answer.sdp.length === 0) { + window.log.warn('failed to create answer'); + return; + } + await peerConnection.setLocalDescription(answer); + const answerSdp = answer.sdp; + const callAnswerMessage = new CallMessage({ + timestamp: Date.now(), + type: SignalService.CallMessage.Type.ANSWER, + sdps: [answerSdp], + }); + + window.log.info('sending ANSWER MESSAGE'); + + await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(sender), callAnswerMessage); + } +} + +export async function handleCallTypeOffer( sender: string, callMessage: SignalService.CallMessage, incomingOfferTimestamp: number ) { try { + window.log.info('handling callMessage OFFER'); + const convos = getConversationController().getConversations(); const callingConvos = convos.filter(convo => convo.callState !== undefined); if (callingConvos.length > 0) { // we just got a new offer from someone we are already in a call with if (callingConvos.length === 1 && callingConvos[0].id === sender) { window.log.info('Got a new offer message from our ongoing call'); + const remoteDesc = new RTCSessionDescription({ + type: 'offer', + sdp: callMessage.sdps[0], + }); + + if (peerConnection) { + await peerConnection.setRemoteDescription(remoteDesc); + remoteStream?.getTracks().forEach(t => { + remoteStream?.removeTrack(t); + }); + await buildAnswerAndSendIt(sender); + } } else { await handleMissedCall(sender, incomingOfferTimestamp); return; @@ -549,14 +620,13 @@ async function handleMissedCall(sender: string, incomingOfferTimestamp: number) return; } -export async function handleCallAnsweredMessage( - sender: string, - callMessage: SignalService.CallMessage -) { +export async function handleCallTypeAnswer(sender: string, callMessage: SignalService.CallMessage) { if (!callMessage.sdps || callMessage.sdps.length === 0) { window.log.warn('cannot handle answered message without signal description protols'); return; } + window.log.info('handling callMessage ANSWER'); + if (!callCache.has(sender)) { callCache.set(sender, new Array()); } @@ -565,7 +635,7 @@ export async function handleCallAnsweredMessage( window.inboxStore?.dispatch(answerCall({ pubkey: sender })); const remoteDesc = new RTCSessionDescription({ type: 'answer', sdp: callMessage.sdps[0] }); if (peerConnection) { - window.log?.warn('Setting remote answer pending'); + // window.log?.info('Setting remote answer pending'); isSettingRemoteAnswerPending = true; await peerConnection.setRemoteDescription(remoteDesc); isSettingRemoteAnswerPending = false; @@ -574,7 +644,7 @@ export async function handleCallAnsweredMessage( } } -export async function handleIceCandidatesMessage( +export async function handleCallTypeIceCandidates( sender: string, callMessage: SignalService.CallMessage ) { @@ -582,12 +652,17 @@ export async function handleIceCandidatesMessage( window.log.warn('cannot handle iceCandicates message without candidates'); return; } + window.log.info('handling callMessage ICE_CANDIDATES'); + if (!callCache.has(sender)) { callCache.set(sender, new Array()); } callCache.get(sender)?.push(callMessage); - // window.inboxStore?.dispatch(incomingCall({ pubkey: sender })); + await addIceCandidateToExistingPeerConnection(callMessage); +} + +async function addIceCandidateToExistingPeerConnection(callMessage: SignalService.CallMessage) { if (peerConnection) { // tslint:disable-next-line: prefer-for-of for (let index = 0; index < callMessage.sdps.length; index++) { @@ -609,9 +684,6 @@ export async function handleIceCandidatesMessage( } // tslint:disable-next-line: no-async-without-await -export async function handleOtherCallMessage( - sender: string, - callMessage: SignalService.CallMessage -) { +export async function handleOtherCallTypes(sender: string, callMessage: SignalService.CallMessage) { callCache.get(sender)?.push(callMessage); }