From d5f6180ae64293211f46a003d7852005c038bc7d Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Mon, 22 Nov 2021 14:36:02 +1100 Subject: [PATCH] create offer and answer ourselves and do not use the negotiation needed event. this event is causing us to loop in negotiation needed when each side try to create one, gets the answer and so on... --- ts/session/utils/CallManager.ts | 167 ++++++++++++++++---------------- ts/state/ducks/call.tsx | 1 + 2 files changed, 87 insertions(+), 81 deletions(-) diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index 5d1171865..2a2f3c1c0 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -25,7 +25,10 @@ import { PnServer } from '../../pushnotification'; import { setIsRinging } from './RingingManager'; export type InputItem = { deviceId: string; label: string }; +// tslint:disable: function-name +const maxWidth = 1920; +const maxHeight = 1080; /** * This uuid is set only once we accepted a call or started one. */ @@ -166,6 +169,28 @@ if (typeof navigator !== 'undefined') { }); } +const silence = () => { + const ctx = new AudioContext(); + const oscillator = ctx.createOscillator(); + const dst = oscillator.connect(ctx.createMediaStreamDestination()); + oscillator.start(); + return Object.assign((dst as any).stream.getAudioTracks()[0], { enabled: false }); +}; + +const black = () => { + const canvas = Object.assign(document.createElement('canvas'), { + width: maxWidth, + height: maxHeight, + }); + canvas.getContext('2d')?.fillRect(0, 0, maxWidth, maxHeight); + const stream = (canvas as any).captureStream(); + return Object.assign(stream.getVideoTracks()[0], { enabled: false }); +}; + +const getBlackSilenceMediaStream = () => { + return new MediaStream([black(), silence()]); +}; + async function updateConnectedDevices() { // Get the set of cameras connected const videoCameras = await getConnectedDevices('videoinput'); @@ -219,6 +244,14 @@ export async function selectCameraByDeviceId(cameraDeviceId: string) { if (sender?.track) { sender.track.enabled = false; } + + // do the same changes locally + localStream?.getVideoTracks().forEach(t => { + t.stop(); + localStream?.removeTrack(t); + }); + localStream?.addTrack(getBlackSilenceMediaStream().getVideoTracks()[0]); + sendVideoStatusViaDataChannel(); callVideoListeners(); return; @@ -235,24 +268,23 @@ export async function selectCameraByDeviceId(cameraDeviceId: string) { try { const newVideoStream = await navigator.mediaDevices.getUserMedia(devicesConfig); const videoTrack = newVideoStream.getVideoTracks()[0]; + if (!peerConnection) { throw new Error('cannot selectCameraByDeviceId without a peer connection'); } - // video might be completely off on start. adding a track like this triggers a negotationneeded event - window.log.info('adding/replacing video track'); - const sender = peerConnection.getSenders().find(s => { - return s.track?.kind === videoTrack.kind; - }); + window.log.info('replacing video track'); + const videoSender = peerConnection + .getTransceivers() + .find(t => t.sender.track?.kind === 'video')?.sender; videoTrack.enabled = true; - if (sender) { - // this should not trigger a negotationneeded event - // and it is needed for when the video cam was never turn on - await sender.replaceTrack(videoTrack); + if (videoSender) { + await videoSender.replaceTrack(videoTrack); } else { - // this will trigger a negotiationeeded event - peerConnection.addTrack(videoTrack, newVideoStream); + throw new Error( + 'We should always have a videoSender as we are using a black video when no camera are in use' + ); } // do the same changes locally @@ -266,6 +298,7 @@ export async function selectCameraByDeviceId(cameraDeviceId: string) { callVideoListeners(); } catch (e) { window.log.warn('selectCameraByDeviceId failed with', e.message); + ToastUtils.pushToastError('selectCamera', e.message); callVideoListeners(); } } @@ -281,6 +314,12 @@ export async function selectAudioInputByDeviceId(audioInputDeviceId: string) { if (sender?.track) { sender.track.enabled = false; } + // do the same changes locally + localStream?.getAudioTracks().forEach(t => { + t.stop(); + localStream?.removeTrack(t); + }); + localStream?.addTrack(getBlackSilenceMediaStream().getAudioTracks()[0]); callVideoListeners(); return; } @@ -295,6 +334,7 @@ export async function selectAudioInputByDeviceId(audioInputDeviceId: string) { try { const newAudioStream = await navigator.mediaDevices.getUserMedia(devicesConfig); + const audioTrack = newAudioStream.getAudioTracks()[0]; if (!peerConnection) { throw new Error('cannot selectAudioInputByDeviceId without a peer connection'); @@ -331,18 +371,15 @@ export async function selectAudioOutputByDeviceId(audioOutputDeviceId: string) { } } -async function handleNegotiationNeededEvent(recipient: string) { +async function createOfferAndSendIt(recipient: string) { try { makingOffer = true; - window.log.info('got handleNegotiationNeeded event. creating offer'); - const offer = await peerConnection?.createOffer({ - offerToReceiveAudio: true, - offerToReceiveVideo: true, - }); + window.log.info('got createOfferAndSendIt event. creating offer'); + await (peerConnection as any)?.setLocalDescription(); + const offer = peerConnection?.localDescription; if (!offer) { throw new Error('Could not create an offer'); } - await peerConnection?.setLocalDescription(offer); if (!currentCallUUID) { window.log.warn('cannot send offer without a currentCallUUID'); @@ -357,18 +394,18 @@ async function handleNegotiationNeededEvent(recipient: string) { uuid: currentCallUUID, }); - window.log.info(`sending OFFER MESSAGE with callUUID: ${currentCallUUID}`); - const negotationOfferSendResult = await getMessageQueue().sendToPubKeyNonDurably( + window.log.info(`sending '${offer.type}'' with callUUID: ${currentCallUUID}`); + const negotiationOfferSendResult = await getMessageQueue().sendToPubKeyNonDurably( PubKey.cast(recipient), offerMessage ); - if (typeof negotationOfferSendResult === 'number') { + if (typeof negotiationOfferSendResult === 'number') { // window.log?.warn('setting last sent timestamp'); - lastOutgoingOfferTimestamp = negotationOfferSendResult; + lastOutgoingOfferTimestamp = negotiationOfferSendResult; } } } catch (err) { - window.log?.error(`Error on handling negotiation needed ${err}`); + window.log?.error(`Error createOfferAndSendIt ${err}`); } finally { makingOffer = false; } @@ -390,23 +427,13 @@ async function openMediaDevicesAndAddTracks() { return; } - selectedAudioInputId = audioInputsList[0].deviceId; + selectedAudioInputId = DEVICE_DISABLED_DEVICE_ID; //audioInputsList[0].deviceId; selectedCameraId = DEVICE_DISABLED_DEVICE_ID; window.log.info( `openMediaDevices videoDevice:${selectedCameraId} audioDevice:${selectedAudioInputId}` ); - const devicesConfig = { - audio: { - deviceId: { exact: selectedAudioInputId }, - - echoCancellation: true, - }, - // we don't need a video stream on start - video: false, - }; - - localStream = await navigator.mediaDevices.getUserMedia(devicesConfig); + localStream = getBlackSilenceMediaStream(); localStream.getTracks().map(track => { if (localStream) { peerConnection?.addTrack(track, localStream); @@ -420,7 +447,6 @@ async function openMediaDevicesAndAddTracks() { callVideoListeners(); } -// tslint:disable-next-line: function-name export async function USER_callRecipient(recipient: string) { if (!getCallMediaPermissionsSettings()) { ToastUtils.pushVideoCallPermissionNeeded(); @@ -457,6 +483,7 @@ export async function USER_callRecipient(recipient: string) { await openMediaDevicesAndAddTracks(); setIsRinging(true); + await createOfferAndSendIt(recipient); } const iceCandidates: Array = new Array(); @@ -579,7 +606,6 @@ function closeVideoCall() { window.inboxStore?.dispatch(endCall()); remoteVideoStreamIsMuted = true; - timestampAcceptedCall = undefined; makingOffer = false; ignoreOffer = false; @@ -626,7 +652,7 @@ function onDataChannelOnOpen() { sendVideoStatusViaDataChannel(); } -function createOrGetPeerConnection(withPubkey: string, isAcceptingCall = false) { +function createOrGetPeerConnection(withPubkey: string) { if (peerConnection) { return peerConnection; } @@ -640,21 +666,7 @@ function createOrGetPeerConnection(withPubkey: string, isAcceptingCall = false) dataChannel.onmessage = onDataChannelReceivedMessage; dataChannel.onopen = onDataChannelOnOpen; - - peerConnection.onnegotiationneeded = async () => { - const shouldTriggerAnotherNeg = - isAcceptingCall && timestampAcceptedCall && Date.now() - timestampAcceptedCall > 1000; - if (!isAcceptingCall || shouldTriggerAnotherNeg) { - await handleNegotiationNeededEvent(withPubkey); - } else { - window.log.info( - 'should negotaite again but we accepted the call recently, so swallowing this one' - ); - } - }; - peerConnection.onsignalingstatechange = handleSignalingStateChangeEvent; - peerConnection.ontrack = event => { event.track.onunmute = () => { remoteStream?.addTrack(event.track); @@ -694,9 +706,6 @@ function createOrGetPeerConnection(withPubkey: string, isAcceptingCall = false) return peerConnection; } -let timestampAcceptedCall: number | undefined; - -// tslint:disable-next-line: function-name export async function USER_acceptIncomingCallRequest(fromSender: string) { window.log.info('USER_acceptIncomingCallRequest'); setIsRinging(false); @@ -730,8 +739,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { } currentCallUUID = lastOfferMessage.uuid; - timestampAcceptedCall = Date.now(); - peerConnection = createOrGetPeerConnection(fromSender, true); + peerConnection = createOrGetPeerConnection(fromSender); await openMediaDevicesAndAddTracks(); @@ -783,7 +791,6 @@ export async function rejectCallAlreadyAnotherCall(fromSender: string, forcedUUI clearCallCacheFromPubkeyAndUUID(fromSender, forcedUUID); } -// tslint:disable-next-line: function-name export async function USER_rejectIncomingCallRequest(fromSender: string) { setIsRinging(false); // close the popup call @@ -822,7 +829,6 @@ async function sendCallMessageAndSync(callmessage: CallMessage, user: string) { ]); } -// tslint:disable-next-line: function-name export async function USER_hangup(fromSender: string) { window.log.info('USER_hangup'); @@ -889,16 +895,12 @@ async function buildAnswerAndSendIt(sender: string) { window.log.warn('cannot send answer without a currentCallUUID'); return; } - - const answer = await peerConnection.createAnswer({ - offerToReceiveAudio: true, - offerToReceiveVideo: true, - }); + await (peerConnection as any).setLocalDescription(); + const answer = peerConnection.localDescription; if (!answer?.sdp || answer.sdp.length === 0) { window.log.warn('failed to create answer'); return; } - await peerConnection.setLocalDescription(answer); const answerSdp = answer.sdp; const callAnswerMessage = new CallMessage({ timestamp: Date.now(), @@ -955,25 +957,26 @@ export async function handleCallTypeOffer( !makingOffer && (peerConnection?.signalingState === 'stable' || isSettingRemoteAnswerPending); const polite = lastOutgoingOfferTimestamp < incomingOfferTimestamp; const offerCollision = !readyForOffer; - ignoreOffer = !polite && offerCollision; + if (ignoreOffer) { window.log?.warn('Received offer when unready for offer; Ignoring offer.'); return; } - if (remoteCallUUID === currentCallUUID && currentCallUUID) { + if (peerConnection && remoteCallUUID === currentCallUUID && currentCallUUID) { window.log.info('Got a new offer message from our ongoing call'); - isSettingRemoteAnswerPending = false; - const remoteDesc = new RTCSessionDescription({ + + const remoteOfferDesc = new RTCSessionDescription({ type: 'offer', sdp: callMessage.sdps[0], }); isSettingRemoteAnswerPending = false; - if (peerConnection) { - await peerConnection.setRemoteDescription(remoteDesc); // SRD rolls back as needed - await buildAnswerAndSendIt(sender); - } + + await peerConnection.setRemoteDescription(remoteOfferDesc); // SRD rolls back as needed + isSettingRemoteAnswerPending = false; + + await buildAnswerAndSendIt(sender); } else { window.inboxStore?.dispatch(incomingCall({ pubkey: sender })); @@ -1103,19 +1106,21 @@ export async function handleCallTypeAnswer(sender: string, callMessage: SignalSe pubkey: sender, }) ); - const remoteDesc = new RTCSessionDescription({ - type: 'answer', - sdp: callMessage.sdps[0], - }); - // window.log?.info('Setting remote answer pending'); - isSettingRemoteAnswerPending = true; try { + isSettingRemoteAnswerPending = true; + + const remoteDesc = new RTCSessionDescription({ + type: 'answer', + sdp: callMessage.sdps[0], + }); + await peerConnection?.setRemoteDescription(remoteDesc); // SRD rolls back as needed } catch (e) { - window.log.warn('setRemoteDescription failed:', e); + window.log.warn('setRemoteDescriptio failed:', e); + } finally { + isSettingRemoteAnswerPending = false; } - isSettingRemoteAnswerPending = false; } export async function handleCallTypeIceCandidates( diff --git a/ts/state/ducks/call.tsx b/ts/state/ducks/call.tsx index 47e9e3bdb..7f0c55a27 100644 --- a/ts/state/ducks/call.tsx +++ b/ts/state/ducks/call.tsx @@ -92,6 +92,7 @@ const callSlice = createSlice({ // only set in full screen if we have an ongoing call if (state.ongoingWith && state.ongoingCallStatus === 'ongoing' && action.payload) { state.callIsInFullScreen = true; + return state; } state.callIsInFullScreen = false; return state;