From 6743201cc43c2b944ff8b7a41d57885761b4e5a4 Mon Sep 17 00:00:00 2001 From: Warrick Corfe-Tan Date: Wed, 29 Sep 2021 10:50:10 +1000 Subject: [PATCH 1/3] added perfect negotiation Adding toast for cam and audio permission when making a call. adding missed call message and toast when a call is received while mid-call. background call message work --- _locales/en/messages.json | 8 +- .../session/calling/CallContainer.tsx | 1 + ts/components/session/menu/Menu.tsx | 29 +++-- ts/models/messageType.ts | 1 + ts/receiver/callMessage.ts | 2 +- ts/receiver/closedGroups.ts | 2 +- ts/receiver/contentMessage.ts | 3 +- ts/session/sending/MessageQueue.ts | 4 +- ts/session/utils/CallManager.ts | 103 +++++++++++------- ts/session/utils/Toast.tsx | 21 ++++ 10 files changed, 112 insertions(+), 62 deletions(-) diff --git a/_locales/en/messages.json b/_locales/en/messages.json index 3af377b39..51aa6eb55 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -447,5 +447,11 @@ "incomingCall": "Incoming call", "accept": "Accept", "decline": "Decline", - "endCall": "End call" + "endCall": "End call", + "micAndCameraPermissionNeededTitle": "Camera and Microphone access required", + "micAndCameraPermissionNeeded": "You can enable microphone and camera access under: Settings (Gear icon) => Privacy", + "unableToCall": "Cancel your exiting call session.", + "unableToCallTitle": "Cannot start new call", + "callMissed": "Missed call from $name$", + "callMissedTitle": "Call missed" } diff --git a/ts/components/session/calling/CallContainer.tsx b/ts/components/session/calling/CallContainer.tsx index b947af116..aff8c2174 100644 --- a/ts/components/session/calling/CallContainer.tsx +++ b/ts/components/session/calling/CallContainer.tsx @@ -135,6 +135,7 @@ export const CallContainer = () => { Call with: {ongoingCallProps.name} +
{hasIncomingCall}
diff --git a/ts/components/session/menu/Menu.tsx b/ts/components/session/menu/Menu.tsx index 37bf13673..04cc7825c 100644 --- a/ts/components/session/menu/Menu.tsx +++ b/ts/components/session/menu/Menu.tsx @@ -1,6 +1,10 @@ import React from 'react'; -import { getNumberOfPinnedConversations } from '../../../state/selectors/conversations'; +import { + getHasIncomingCall, + getHasOngoingCall, + getNumberOfPinnedConversations, +} from '../../../state/selectors/conversations'; import { getFocusedSection } from '../../../state/selectors/section'; import { Item, Submenu } from 'react-contexify'; import { @@ -319,32 +323,27 @@ export function getMarkAllReadMenuItem(conversationId: string): JSX.Element | nu } export function getStartCallMenuItem(conversationId: string): JSX.Element | null { - // TODO: possibly conditionally show options? - // const callOptions = [ - // { - // name: 'Video call', - // value: 'video-call', - // }, - // // { - // // name: 'Audio call', - // // value: 'audio-call', - // // }, - // ]; - + const canCall = !(useSelector(getHasIncomingCall) || useSelector(getHasOngoingCall)); return ( { // TODO: either pass param to callRecipient or call different call methods based on item selected. + // TODO: one time redux-persisted permission modal? const convo = getConversationController().get(conversationId); + + if (!canCall) { + ToastUtils.pushUnableToCall(); + return; + } + if (convo) { convo.callState = 'connecting'; await convo.commit(); - await CallManager.USER_callRecipient(convo.id); } }} > - {'video call'} + {'Video Call'} ); } diff --git a/ts/models/messageType.ts b/ts/models/messageType.ts index 6e7509cdc..41df5c4a2 100644 --- a/ts/models/messageType.ts +++ b/ts/models/messageType.ts @@ -177,6 +177,7 @@ export interface MessageAttributesOptionals { direction?: any; messageHash?: string; isDeleted?: boolean; + isCall?: boolean; } /** diff --git a/ts/receiver/callMessage.ts b/ts/receiver/callMessage.ts index f2025d816..21334249a 100644 --- a/ts/receiver/callMessage.ts +++ b/ts/receiver/callMessage.ts @@ -46,7 +46,7 @@ export async function handleCallMessage( } await removeFromCache(envelope); - CallManager.handleOfferCallMessage(sender, callMessage); + CallManager.handleOfferCallMessage(sender, callMessage, sentTimestamp); return; } diff --git a/ts/receiver/closedGroups.ts b/ts/receiver/closedGroups.ts index 6725306fa..3306632a3 100644 --- a/ts/receiver/closedGroups.ts +++ b/ts/receiver/closedGroups.ts @@ -977,7 +977,7 @@ async function sendToGroupMembers( window?.log?.info(`Creating a new group and an encryptionKeyPair for group ${groupPublicKey}`); // evaluating if all invites sent, if failed give the option to retry failed invites via modal dialog const inviteResults = await Promise.all(promises); - const allInvitesSent = _.every(inviteResults, Boolean); + const allInvitesSent = _.every(inviteResults, inviteResult => inviteResult !== false); if (allInvitesSent) { // if (true) { diff --git a/ts/receiver/contentMessage.ts b/ts/receiver/contentMessage.ts index c3f900f33..83bd22ff6 100644 --- a/ts/receiver/contentMessage.ts +++ b/ts/receiver/contentMessage.ts @@ -401,7 +401,7 @@ export async function innerHandleContentMessage( await handleUnsendMessage(envelope, content.unsendMessage as SignalService.Unsend); } if (content.callMessage && window.lokiFeatureFlags?.useCallMessage) { - await handleCallMessage(envelope, content.callMessage as SignalService.CallMessage); + await handleCallMessage(envelope, content.callMessage as SignalService.CallMessage, messageHash); } } catch (e) { window?.log?.warn(e); @@ -569,6 +569,7 @@ export async function handleDataExtractionNotification( }, unread: 1, // 1 means unread expireTimer: 0, + isCall: true, }); convo.updateLastMessage(); } diff --git a/ts/session/sending/MessageQueue.ts b/ts/session/sending/MessageQueue.ts index 1a7cf9db4..d1a248f9a 100644 --- a/ts/session/sending/MessageQueue.ts +++ b/ts/session/sending/MessageQueue.ts @@ -131,7 +131,7 @@ export class MessageQueue { public async sendToPubKeyNonDurably( user: PubKey, message: ClosedGroupNewMessage | CallMessage - ): Promise { + ): Promise { let rawMessage; try { rawMessage = await MessageUtils.toRawMessage(user, message); @@ -141,7 +141,7 @@ export class MessageQueue { effectiveTimestamp, wrappedEnvelope ); - return !!wrappedEnvelope; + return effectiveTimestamp; } catch (error) { if (rawMessage) { await MessageSentHandler.handleMessageSentFailure(rawMessage, error); diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index 5e9a2ab24..cacbfd26c 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -1,4 +1,8 @@ import _ from 'lodash'; +import { ToastUtils } from '.'; +import { SessionSettingCategory } from '../../components/session/settings/SessionSettings'; +import { getConversationById } from '../../data/data'; +import { MessageModelType } from '../../models/messageType'; import { SignalService } from '../../protobuf'; import { answerCall, @@ -7,6 +11,8 @@ import { incomingCall, startingCallWith, } from '../../state/ducks/conversations'; +import { SectionType, showLeftPaneSection, showSettingsSection } from '../../state/ducks/section'; +import { getConversationController } from '../conversations'; import { CallMessage } from '../messages/outgoing/controlMessage/CallMessage'; import { ed25519Str } from '../onions/onionPath'; import { getMessageQueue } from '../sending'; @@ -33,6 +39,7 @@ const ENABLE_VIDEO = true; let makingOffer = false; let ignoreOffer = false; let isSettingRemoteAnswerPending = false; +let lastOutgoingOfferTimestamp = -Infinity; const configuration = { configuration: { @@ -67,8 +74,10 @@ export async function USER_callRecipient(recipient: string) { peerConnection?.addTrack(track, mediaDevices); }); } catch (err) { - console.error('Failed to open media devices. Check camera and mic app permissions'); - // TODO: implement toast popup + ToastUtils.pushMicAndCameraPermissionNeeded(() => { + window.inboxStore?.dispatch(showLeftPaneSection(SectionType.Settings)); + window.inboxStore?.dispatch(showSettingsSection(SessionSettingCategory.Privacy)); + }); } peerConnection.addEventListener('connectionstatechange', _event => { window.log.info('peerConnection?.connectionState caller :', peerConnection?.connectionState); @@ -92,21 +101,6 @@ export async function USER_callRecipient(recipient: string) { console.warn('negotiationneeded:', event); try { makingOffer = true; - // const offerDescription = await peerConnection?.createOffer({ - // offerToReceiveAudio: true, - // offerToReceiveVideo: true, - // }); - // if (!offerDescription) { - // console.error('Failed to create offer for negotiation'); - // return; - // } - // await peerConnection?.setLocalDescription(offerDescription); - // if (!offerDescription || !offerDescription.sdp || !offerDescription.sdp.length) { - // // window.log.warn(`failed to createOffer for recipient ${ed25519Str(recipient)}`); - // console.warn(`failed to createOffer for recipient ${ed25519Str(recipient)}`); - // return; - // } - // @ts-ignore await peerConnection?.setLocalDescription(); let offer = await peerConnection?.createOffer(); @@ -116,12 +110,20 @@ export async function USER_callRecipient(recipient: string) { const callOfferMessage = new CallMessage({ timestamp: Date.now(), type: SignalService.CallMessage.Type.OFFER, - // sdps: [offerDescription.sdp], sdps: [offer.sdp], }); window.log.info('sending OFFER MESSAGE'); - await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(recipient), callOfferMessage); + const sendResult = await getMessageQueue().sendToPubKeyNonDurably( + PubKey.cast(recipient), + callOfferMessage + ); + if (typeof sendResult === 'number') { + console.warn('setting last sent timestamp'); + lastOutgoingOfferTimestamp = sendResult; + } + + await new Promise(r => setTimeout(r, 10000)); } } catch (err) { console.error(err); @@ -161,7 +163,14 @@ export async function USER_callRecipient(recipient: string) { }); window.log.info('sending OFFER MESSAGE'); - await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(recipient), callOfferMessage); + let sendResult = await getMessageQueue().sendToPubKeyNonDurably( + PubKey.cast(recipient), + callOfferMessage + ); + if (typeof sendResult === 'number') { + console.warn('setting timestamp'); + lastOutgoingOfferTimestamp = sendResult; + } // FIXME audric dispatch UI update to show the calling UI } @@ -363,37 +372,30 @@ export function handleEndCallMessage(sender: string) { export async function handleOfferCallMessage( sender: string, - callMessage: SignalService.CallMessage + callMessage: SignalService.CallMessage, + incomingOfferTimestamp: number ) { try { console.warn({ callMessage }); + + const convos = getConversationController().getConversations(); + if (convos.some(convo => convo.callState !== undefined)) { + return await handleMissedCall(sender, incomingOfferTimestamp); + } + const readyForOffer = !makingOffer && (peerConnection?.signalingState == 'stable' || isSettingRemoteAnswerPending); - // TODO: How should politeness be decided between client / recipient? - ignoreOffer = !true && !readyForOffer; + // TODO: however sent offer last is the impolite user + const polite = lastOutgoingOfferTimestamp < incomingOfferTimestamp; + console.warn({ polite }); + ignoreOffer = !polite && !readyForOffer; + console.warn({ ignoreOffer }); if (ignoreOffer) { // window.log?.warn('Received offer when unready for offer; Ignoring offer.'); console.warn('Received offer when unready for offer; Ignoring offer.'); return; } - - // const description = await peerConnection?.createOffer({ - // const description = await peerConnection?.createOffer({ - // offerToReceiveVideo: true, - // offerToReceiveAudio: true, - // }) - - // @ts-ignore - await peerConnection?.setLocalDescription(); - console.warn(peerConnection?.localDescription); - - const message = new CallMessage({ - type: SignalService.CallMessage.Type.ANSWER, - timestamp: Date.now(), - }); - - await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(sender), message); - // TODO: send via our signalling with the sdp of our pc.localDescription + // don't need to do the sending here as we dispatch an answer in a } catch (err) { window.log?.error(`Error handling offer message ${err}`); } @@ -405,6 +407,25 @@ export async function handleOfferCallMessage( window.inboxStore?.dispatch(incomingCall({ pubkey: sender })); } +async function handleMissedCall(sender: string, incomingOfferTimestamp: number) { + const incomingCallConversation = await getConversationById(sender); + ToastUtils.pushedMissedCall(incomingCallConversation?.getNickname() || 'Unknown'); + + await incomingCallConversation?.addSingleMessage({ + conversationId: incomingCallConversation.id, + source: sender, + type: 'incoming' as MessageModelType, + sent_at: incomingOfferTimestamp, + received_at: Date.now(), + expireTimer: 0, + body: 'Missed call', + unread: 1, + isCall: false, + }); + incomingCallConversation?.updateLastMessage(); + return; +} + export async function handleCallAnsweredMessage( sender: string, callMessage: SignalService.CallMessage diff --git a/ts/session/utils/Toast.tsx b/ts/session/utils/Toast.tsx index cf3727337..1ce498724 100644 --- a/ts/session/utils/Toast.tsx +++ b/ts/session/utils/Toast.tsx @@ -134,6 +134,27 @@ export function pushMessageDeleteForbidden() { pushToastError('messageDeletionForbidden', window.i18n('messageDeletionForbidden')); } +export function pushUnableToCall() { + pushToastError('unableToCall', window.i18n('unableToCallTitle'), window.i18n('unableToCall')); +} + +export function pushedMissedCall(conversationName: string) { + pushToastInfo( + 'missedCall', + window.i18n('callMissedTitle'), + window.i18n('callMissedTitle', conversationName) + ); +} + +export function pushMicAndCameraPermissionNeeded(onClicked: () => void) { + pushToastInfo( + 'micAndCameraPermissionNeeded', + window.i18n('micAndCameraPermissionNeededTitle'), + window.i18n('micAndCameraPermissionNeeded'), + onClicked + ); +} + export function pushAudioPermissionNeeded(onClicked: () => void) { pushToastInfo( 'audioPermissionNeeded', From 0b8f3255b0393231bef7186ee650196fd86312df Mon Sep 17 00:00:00 2001 From: Warrick Corfe-Tan Date: Fri, 1 Oct 2021 14:06:04 +1000 Subject: [PATCH 2/3] corrections to pass yarn ready --- ts/receiver/callMessage.ts | 2 +- ts/receiver/contentMessage.ts | 2 +- ts/session/utils/CallManager.ts | 59 +++++++++++++++------------------ 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/ts/receiver/callMessage.ts b/ts/receiver/callMessage.ts index 21334249a..7fa2ee413 100644 --- a/ts/receiver/callMessage.ts +++ b/ts/receiver/callMessage.ts @@ -46,7 +46,7 @@ export async function handleCallMessage( } await removeFromCache(envelope); - CallManager.handleOfferCallMessage(sender, callMessage, sentTimestamp); + await CallManager.handleOfferCallMessage(sender, callMessage, sentTimestamp); return; } diff --git a/ts/receiver/contentMessage.ts b/ts/receiver/contentMessage.ts index 83bd22ff6..bf6477cb9 100644 --- a/ts/receiver/contentMessage.ts +++ b/ts/receiver/contentMessage.ts @@ -401,7 +401,7 @@ export async function innerHandleContentMessage( await handleUnsendMessage(envelope, content.unsendMessage as SignalService.Unsend); } if (content.callMessage && window.lokiFeatureFlags?.useCallMessage) { - await handleCallMessage(envelope, content.callMessage as SignalService.CallMessage, messageHash); + await handleCallMessage(envelope, content.callMessage as SignalService.CallMessage); } } catch (e) { window?.log?.warn(e); diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index cacbfd26c..3a661724f 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -68,8 +68,8 @@ export async function USER_callRecipient(recipient: string) { let mediaDevices: any; try { - const mediaDevices = await openMediaDevices(); - mediaDevices.getTracks().map(track => { + mediaDevices = await openMediaDevices(); + mediaDevices.getTracks().map((track: any) => { window.log.info('USER_callRecipient adding track: ', track); peerConnection?.addTrack(track, mediaDevices); }); @@ -86,7 +86,7 @@ export async function USER_callRecipient(recipient: string) { } }); peerConnection.addEventListener('ontrack', event => { - console.warn('ontrack:', event); + window.log?.warn('ontrack:', event); }); peerConnection.addEventListener('icecandidate', event => { // window.log.warn('event.candidate', event.candidate); @@ -98,35 +98,33 @@ export async function USER_callRecipient(recipient: string) { }); // peerConnection.addEventListener('negotiationneeded', async event => { peerConnection.onnegotiationneeded = async event => { - console.warn('negotiationneeded:', event); + window.log?.warn('negotiationneeded:', event); try { makingOffer = true; // @ts-ignore await peerConnection?.setLocalDescription(); - let offer = await peerConnection?.createOffer(); - console.warn(offer); + const offer = await peerConnection?.createOffer(); + window.log?.warn(offer); if (offer && offer.sdp) { - const callOfferMessage = new CallMessage({ + const negotationOfferMessage = new CallMessage({ timestamp: Date.now(), type: SignalService.CallMessage.Type.OFFER, sdps: [offer.sdp], }); window.log.info('sending OFFER MESSAGE'); - const sendResult = await getMessageQueue().sendToPubKeyNonDurably( + const negotationOfferSendResult = await getMessageQueue().sendToPubKeyNonDurably( PubKey.cast(recipient), - callOfferMessage + negotationOfferMessage ); - if (typeof sendResult === 'number') { - console.warn('setting last sent timestamp'); - lastOutgoingOfferTimestamp = sendResult; + if (typeof negotationOfferSendResult === 'number') { + window.log?.warn('setting last sent timestamp'); + lastOutgoingOfferTimestamp = negotationOfferSendResult; } - - await new Promise(r => setTimeout(r, 10000)); + // debug: await new Promise(r => setTimeout(r, 10000)); adding artificial wait for offer debugging } } catch (err) { - console.error(err); window.log?.error(`Error on handling negotiation needed ${err}`); } finally { makingOffer = false; @@ -156,20 +154,20 @@ export async function USER_callRecipient(recipient: string) { return; } await peerConnection.setLocalDescription(offerDescription); - const callOfferMessage = new CallMessage({ + const offerMessage = new CallMessage({ timestamp: Date.now(), type: SignalService.CallMessage.Type.OFFER, sdps: [offerDescription.sdp], }); window.log.info('sending OFFER MESSAGE'); - let sendResult = await getMessageQueue().sendToPubKeyNonDurably( + const offerSendResult = await getMessageQueue().sendToPubKeyNonDurably( PubKey.cast(recipient), - callOfferMessage + offerMessage ); - if (typeof sendResult === 'number') { - console.warn('setting timestamp'); - lastOutgoingOfferTimestamp = sendResult; + if (typeof offerSendResult === 'number') { + window.log?.warn('setting timestamp'); + lastOutgoingOfferTimestamp = offerSendResult; } // FIXME audric dispatch UI update to show the calling UI } @@ -264,13 +262,13 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { const remoteStream = new MediaStream(); peerConnection.addEventListener('icecandidate', event => { - console.warn('icecandidateerror:', event); + window.log?.warn('icecandidateerror:', event); // TODO: ICE stuff // signaler.send({candidate}); // probably event.candidate }); peerConnection.addEventListener('signalingstatechange', event => { - console.warn('signalingstatechange:', event); + window.log?.warn('signalingstatechange:', event); }); if (videoEventsListener) { @@ -376,23 +374,19 @@ export async function handleOfferCallMessage( incomingOfferTimestamp: number ) { try { - console.warn({ callMessage }); - const convos = getConversationController().getConversations(); if (convos.some(convo => convo.callState !== undefined)) { - return await handleMissedCall(sender, incomingOfferTimestamp); + await handleMissedCall(sender, incomingOfferTimestamp); + return; } const readyForOffer = - !makingOffer && (peerConnection?.signalingState == 'stable' || isSettingRemoteAnswerPending); - // TODO: however sent offer last is the impolite user + !makingOffer && (peerConnection?.signalingState === 'stable' || isSettingRemoteAnswerPending); const polite = lastOutgoingOfferTimestamp < incomingOfferTimestamp; - console.warn({ polite }); ignoreOffer = !polite && !readyForOffer; - console.warn({ ignoreOffer }); if (ignoreOffer) { // window.log?.warn('Received offer when unready for offer; Ignoring offer.'); - console.warn('Received offer when unready for offer; Ignoring offer.'); + window.log?.warn('Received offer when unready for offer; Ignoring offer.'); return; } // don't need to do the sending here as we dispatch an answer in a @@ -442,7 +436,7 @@ export async function handleCallAnsweredMessage( window.inboxStore?.dispatch(answerCall({ pubkey: sender })); const remoteDesc = new RTCSessionDescription({ type: 'answer', sdp: callMessage.sdps[0] }); if (peerConnection) { - console.warn('Setting remote answer pending'); + window.log?.warn('Setting remote answer pending'); isSettingRemoteAnswerPending = true; await peerConnection.setRemoteDescription(remoteDesc); isSettingRemoteAnswerPending = false; @@ -476,6 +470,7 @@ export async function handleIceCandidatesMessage( await peerConnection.addIceCandidate(candicate); } catch (err) { if (!ignoreOffer) { + window.log?.warn('Error handling ICE candidates message'); } } } From 1522e5102d7b71d3f79ca3c85763d839fa02c778 Mon Sep 17 00:00:00 2001 From: Warrick Corfe-Tan Date: Fri, 1 Oct 2021 15:13:02 +1000 Subject: [PATCH 3/3] removed isCall property from message. Fixed text --- _locales/en/messages.json | 2 +- ts/models/messageType.ts | 1 - ts/receiver/contentMessage.ts | 1 - ts/session/utils/CallManager.ts | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/_locales/en/messages.json b/_locales/en/messages.json index 51aa6eb55..ecb4b57b0 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -450,7 +450,7 @@ "endCall": "End call", "micAndCameraPermissionNeededTitle": "Camera and Microphone access required", "micAndCameraPermissionNeeded": "You can enable microphone and camera access under: Settings (Gear icon) => Privacy", - "unableToCall": "Cancel your exiting call session.", + "unableToCall": "cancel your ongoing call first", "unableToCallTitle": "Cannot start new call", "callMissed": "Missed call from $name$", "callMissedTitle": "Call missed" diff --git a/ts/models/messageType.ts b/ts/models/messageType.ts index 41df5c4a2..6e7509cdc 100644 --- a/ts/models/messageType.ts +++ b/ts/models/messageType.ts @@ -177,7 +177,6 @@ export interface MessageAttributesOptionals { direction?: any; messageHash?: string; isDeleted?: boolean; - isCall?: boolean; } /** diff --git a/ts/receiver/contentMessage.ts b/ts/receiver/contentMessage.ts index bf6477cb9..c3f900f33 100644 --- a/ts/receiver/contentMessage.ts +++ b/ts/receiver/contentMessage.ts @@ -569,7 +569,6 @@ export async function handleDataExtractionNotification( }, unread: 1, // 1 means unread expireTimer: 0, - isCall: true, }); convo.updateLastMessage(); } diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index 3a661724f..98491f8ae 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -414,7 +414,6 @@ async function handleMissedCall(sender: string, incomingOfferTimestamp: number) expireTimer: 0, body: 'Missed call', unread: 1, - isCall: false, }); incomingCallConversation?.updateLastMessage(); return;