From 8b611a286757018878c6e065338150c993aae65e Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 23 Sep 2021 13:37:38 +1000 Subject: [PATCH 1/3] make call UI react to incoming and ongoing calls --- .../session/calling/CallContainer.tsx | 146 +++++++----------- ts/components/session/menu/Menu.tsx | 57 +++---- ts/models/conversation.ts | 9 +- ts/receiver/callMessage.ts | 2 +- ts/session/sending/MessageSender.ts | 5 +- ts/session/utils/CallManager.ts | 60 +++---- ts/state/ducks/conversations.ts | 82 ++++++++++ ts/state/selectors/conversations.ts | 41 +++++ 8 files changed, 242 insertions(+), 160 deletions(-) diff --git a/ts/components/session/calling/CallContainer.tsx b/ts/components/session/calling/CallContainer.tsx index 4c0f2f3de..cf4bd433e 100644 --- a/ts/components/session/calling/CallContainer.tsx +++ b/ts/components/session/calling/CallContainer.tsx @@ -1,8 +1,14 @@ -import React, { useState } from 'react'; +import React from 'react'; +import { useSelector } from 'react-redux'; import styled from 'styled-components'; import _ from 'underscore'; -import { getConversationController } from '../../../session/conversations/ConversationController'; import { CallManager } from '../../../session/utils'; +import { + getHasIncomingCall, + getHasIncomingCallFrom, + getHasOngoingCall, + getHasOngoingCallWith, +} from '../../../state/selectors/conversations'; import { SessionButton, SessionButtonColor } from '../SessionButton'; import { SessionWrapperModal } from '../SessionWrapperModal'; @@ -59,108 +65,74 @@ const CallWindowControls = styled.div` transform: translateY(-100%); `; -// type WindowPositionType = { -// top: string; -// left: string; -// } | null; - -type CallStateType = 'connecting' | 'ongoing' | 'incoming' | null; - -const fakeCaller = '054774a456f15c7aca42fe8d245983549000311aaebcf58ce246250c41fe227676'; - +// TODO: +/** + * Add mute input, deafen, end call, possibly add person to call + * duration - look at how duration calculated for recording. + */ export const CallContainer = () => { - const conversations = getConversationController().getConversations(); - - // TODO: - /** - * Add mute input, deafen, end call, possibly add person to call - * duration - look at how duration calculated for recording. - */ - - const [connectionState, setConnectionState] = useState('incoming'); - // const [callWindowPosition, setCallWindowPosition] = useState(null) - - // picking a conversation at random to test with - const foundConvo = conversations.find(convo => convo.id === fakeCaller); - - if (!foundConvo) { - throw new Error('fakeconvo not found'); - } - foundConvo.callState = 'incoming'; - console.warn('foundConvo: ', foundConvo); + const hasIncomingCall = useSelector(getHasIncomingCall); + const incomingCallProps = useSelector(getHasIncomingCallFrom); + const ongoingCallProps = useSelector(getHasOngoingCallWith); + const hasOngoingCall = useSelector(getHasOngoingCall); - const firstCallingConvo = _.first(conversations.filter(convo => convo.callState !== undefined)); + const ongoingOrIncomingPubkey = ongoingCallProps?.id || incomingCallProps?.id; //#region input handlers const handleAcceptIncomingCall = async () => { - console.warn('accept call'); - - if (firstCallingConvo) { - setConnectionState('connecting'); - firstCallingConvo.callState = 'connecting'; - await CallManager.USER_acceptIncomingCallRequest(fakeCaller); - // some delay - setConnectionState('ongoing'); - firstCallingConvo.callState = 'ongoing'; + if (incomingCallProps?.id) { + await CallManager.USER_acceptIncomingCallRequest(incomingCallProps.id); } - // set conversationState = setting up }; const handleDeclineIncomingCall = async () => { - // set conversation.callState = null or undefined // close the modal - if (firstCallingConvo) { - firstCallingConvo.callState = undefined; + if (incomingCallProps?.id) { + await CallManager.USER_rejectIncomingCallRequest(incomingCallProps.id); } - console.warn('declined call'); - await CallManager.USER_rejectIncomingCallRequest(fakeCaller); }; const handleEndCall = async () => { // call method to end call connection - console.warn('ending the call'); - await CallManager.USER_rejectIncomingCallRequest(fakeCaller); + if (ongoingOrIncomingPubkey) { + await CallManager.USER_rejectIncomingCallRequest(ongoingOrIncomingPubkey); + } }; - const handleMouseDown = () => { - // reposition call window - }; //#endregion - return ( - <> - {connectionState === 'connecting' ? 'connecting...' : null} - {connectionState === 'ongoing' ? ( - - - - {firstCallingConvo ? firstCallingConvo.getName() : 'Group name not found'} - - - - - - - - - ) : null} - - {!connectionState ? ( - 'none' - ) : null} - - {connectionState === 'incoming' ? ( - -
- - -
- - ) : null} - - ); + if (!hasOngoingCall && !hasIncomingCall) { + return null; + } + + if (hasOngoingCall && ongoingCallProps) { + return ( + + + {ongoingCallProps.name} + + + + + + + ); + } + + if (hasIncomingCall) { + return ( + +
+ + +
+ + ); + } + // display spinner while connecting + return null; }; diff --git a/ts/components/session/menu/Menu.tsx b/ts/components/session/menu/Menu.tsx index 2491dc454..37bf13673 100644 --- a/ts/components/session/menu/Menu.tsx +++ b/ts/components/session/menu/Menu.tsx @@ -28,8 +28,7 @@ import { } from '../../../interactions/conversationInteractions'; import { SessionButtonColor } from '../SessionButton'; import { getTimerOptions } from '../../../state/selectors/timerOptions'; -import { ToastUtils } from '../../../session/utils'; -import { getConversationById } from '../../../data/data'; +import { CallManager, ToastUtils } from '../../../session/utils'; const maxNumberOfPinnedConversations = 5; @@ -321,38 +320,32 @@ export function getMarkAllReadMenuItem(conversationId: string): JSX.Element | nu export function getStartCallMenuItem(conversationId: string): JSX.Element | null { // TODO: possibly conditionally show options? - const callOptions = [ - { - name: 'Video call', - value: 'video-call', - }, - { - name: 'Audio call', - value: 'audio-call', - }, - ]; + // const callOptions = [ + // { + // name: 'Video call', + // value: 'video-call', + // }, + // // { + // // name: 'Audio call', + // // value: 'audio-call', + // // }, + // ]; return ( - - {callOptions.map(item => ( - { - // TODO: either pass param to callRecipient or call different call methods based on item selected. - const convo = await getConversationById(conversationId); - if (convo) { - // window?.libsession?.Utils.CallManager.USER_callRecipient( - // '054774a456f15c7aca42fe8d245983549000311aaebcf58ce246250c41fe227676' - // ); - window?.libsession?.Utils.CallManager.USER_callRecipient(convo.id); - convo.callState = 'connecting'; - } - }} - > - {item.name} - - ))} - + { + // TODO: either pass param to callRecipient or call different call methods based on item selected. + const convo = getConversationController().get(conversationId); + if (convo) { + convo.callState = 'connecting'; + await convo.commit(); + + await CallManager.USER_callRecipient(convo.id); + } + }} + > + {'video call'} + ); } diff --git a/ts/models/conversation.ts b/ts/models/conversation.ts index 1fa9609a6..dd4a416cd 100644 --- a/ts/models/conversation.ts +++ b/ts/models/conversation.ts @@ -177,6 +177,8 @@ export const fillConvoAttributesWithDefaults = ( }); }; +export type CallState = 'offering' | 'incoming' | 'connecting' | 'ongoing' | 'none' | undefined; + export class ConversationModel extends Backbone.Model { public updateLastMessage: () => any; public throttledBumpTyping: any; @@ -184,7 +186,7 @@ export class ConversationModel extends Backbone.Model { public markRead: (newestUnreadDate: number, providedOptions?: any) => Promise; public initialPromise: any; - public callState: 'offering' | 'incoming' | 'connecting' | 'ongoing' | 'none' | undefined; + public callState: CallState; private typingRefreshTimer?: NodeJS.Timeout | null; private typingPauseTimer?: NodeJS.Timeout | null; @@ -440,6 +442,7 @@ export class ConversationModel extends Backbone.Model { const left = !!this.get('left'); const expireTimer = this.get('expireTimer'); const currentNotificationSetting = this.get('triggerNotificationsFor'); + const callState = this.callState; // to reduce the redux store size, only set fields which cannot be undefined // for instance, a boolean can usually be not set if false, etc @@ -544,6 +547,10 @@ export class ConversationModel extends Backbone.Model { text: lastMessageText, }; } + + if (callState) { + toRet.callState = callState; + } return toRet; } diff --git a/ts/receiver/callMessage.ts b/ts/receiver/callMessage.ts index 4e610dddf..f2025d816 100644 --- a/ts/receiver/callMessage.ts +++ b/ts/receiver/callMessage.ts @@ -46,7 +46,7 @@ export async function handleCallMessage( } await removeFromCache(envelope); - await CallManager.handleOfferCallMessage(sender, callMessage); + CallManager.handleOfferCallMessage(sender, callMessage); return; } diff --git a/ts/session/sending/MessageSender.ts b/ts/session/sending/MessageSender.ts index a5ba3d557..8946425e0 100644 --- a/ts/session/sending/MessageSender.ts +++ b/ts/session/sending/MessageSender.ts @@ -17,8 +17,9 @@ import { storeOnNode } from '../snode_api/SNodeAPI'; import { getSwarmFor } from '../snode_api/snodePool'; import { firstTrue } from '../utils/Promise'; import { MessageSender } from '.'; -import { getConversationById, getMessageById } from '../../../ts/data/data'; +import { getMessageById } from '../../../ts/data/data'; import { SNodeAPI } from '../snode_api'; +import { getConversationController } from '../conversations'; const DEFAULT_CONNECTIONS = 3; @@ -173,7 +174,7 @@ export async function TEST_sendMessageToSnode( throw new window.textsecure.EmptySwarmError(pubKey, 'Ran out of swarm nodes to query'); } - const conversation = await getConversationById(pubKey); + const conversation = getConversationController().get(pubKey); const isClosedGroup = conversation?.isClosedGroup(); // If message also has a sync message, save that hash. Otherwise save the hash from the regular message send i.e. only closed groups in this case. diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index abb958e79..25e8e4f19 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -1,26 +1,11 @@ import _ from 'lodash'; import { SignalService } from '../../protobuf'; +import { answerCall, callConnected, endCall, incomingCall } from '../../state/ducks/conversations'; import { CallMessage } from '../messages/outgoing/controlMessage/CallMessage'; import { ed25519Str } from '../onions/onionPath'; import { getMessageQueue } from '../sending'; import { PubKey } from '../types'; -const incomingCall = ({ sender }: { sender: string }) => { - return { type: 'incomingCall', payload: sender }; -}; -const endCall = ({ sender }: { sender: string }) => { - return { type: 'endCall', payload: sender }; -}; -const answerCall = ({ sender, sdps }: { sender: string; sdps: Array }) => { - return { - type: 'answerCall', - payload: { - sender, - sdps, - }, - }; -}; - /** * This field stores all the details received by a sender about a call in separate messages. */ @@ -28,7 +13,7 @@ const callCache = new Map>(); let peerConnection: RTCPeerConnection | null; -const ENABLE_VIDEO = true; +const ENABLE_VIDEO = false; const configuration = { configuration: { @@ -57,13 +42,12 @@ export async function USER_callRecipient(recipient: string) { const mediaDevices = await openMediaDevices(); mediaDevices.getTracks().map(track => { - window.log.info('USER_callRecipient adding track: ', track); peerConnection?.addTrack(track, mediaDevices); }); peerConnection.addEventListener('connectionstatechange', _event => { - window.log.info('peerConnection?.connectionState:', peerConnection?.connectionState); + window.log.info('peerConnection?.connectionState caller :', peerConnection?.connectionState); if (peerConnection?.connectionState === 'connected') { - // Peers connected! + window.inboxStore?.dispatch(callConnected({ pubkey: recipient })); } }); @@ -180,27 +164,32 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { peerConnection = new RTCPeerConnection(configuration); const mediaDevices = await openMediaDevices(); mediaDevices.getTracks().map(track => { - window.log.info('USER_acceptIncomingCallRequest adding track ', track); + // window.log.info('USER_acceptIncomingCallRequest adding track ', track); peerConnection?.addTrack(track, mediaDevices); }); peerConnection.addEventListener('icecandidateerror', event => { console.warn('icecandidateerror:', event); }); - peerConnection.addEventListener('negotiationneeded', async event => { + peerConnection.addEventListener('negotiationneeded', event => { console.warn('negotiationneeded:', event); }); - peerConnection.addEventListener('signalingstatechange', event => { - console.warn('signalingstatechange:', event); + peerConnection.addEventListener('signalingstatechange', _event => { + // console.warn('signalingstatechange:', event); }); peerConnection.addEventListener('ontrack', event => { console.warn('ontrack:', event); }); peerConnection.addEventListener('connectionstatechange', _event => { - window.log.info('peerConnection?.connectionState:', peerConnection?.connectionState, _event); + window.log.info( + 'peerConnection?.connectionState recipient:', + peerConnection?.connectionState, + 'with: ', + fromSender + ); if (peerConnection?.connectionState === 'connected') { - // Peers connected! + window.inboxStore?.dispatch(callConnected({ pubkey: fromSender })); } }); @@ -237,7 +226,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { ); if (lastCandidatesFromSender) { - console.warn('found sender ice candicate message already sent. Using it'); + window.log.info('found sender ice candicate message already sent. Using it'); for (let index = 0; index < lastCandidatesFromSender.sdps.length; index++) { const sdp = lastCandidatesFromSender.sdps[index]; const sdpMLineIndex = lastCandidatesFromSender.sdpMLineIndexes[index]; @@ -250,7 +239,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), callAnswerMessage); - window.inboxStore?.dispatch(answerCall({ sender: fromSender, sdps })); + window.inboxStore?.dispatch(answerCall({ pubkey: fromSender })); } // tslint:disable-next-line: function-name @@ -261,7 +250,7 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) { }); callCache.delete(fromSender); - window.inboxStore?.dispatch(endCall({ sender: fromSender })); + window.inboxStore?.dispatch(endCall({ pubkey: fromSender })); window.log.info('sending END_CALL MESSAGE'); await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), endCallMessage); @@ -271,18 +260,15 @@ export function handleEndCallMessage(sender: string) { callCache.delete(sender); // // FIXME audric trigger UI cleanup - window.inboxStore?.dispatch(endCall({ sender })); + window.inboxStore?.dispatch(endCall({ pubkey: sender })); } -export async function handleOfferCallMessage( - sender: string, - callMessage: SignalService.CallMessage -) { +export function handleOfferCallMessage(sender: string, callMessage: SignalService.CallMessage) { if (!callCache.has(sender)) { callCache.set(sender, new Array()); } callCache.get(sender)?.push(callMessage); - window.inboxStore?.dispatch(incomingCall({ sender })); + window.inboxStore?.dispatch(incomingCall({ pubkey: sender })); } export async function handleCallAnsweredMessage( @@ -298,7 +284,7 @@ export async function handleCallAnsweredMessage( } callCache.get(sender)?.push(callMessage); - window.inboxStore?.dispatch(incomingCall({ sender })); + window.inboxStore?.dispatch(answerCall({ pubkey: sender })); const remoteDesc = new RTCSessionDescription({ type: 'answer', sdp: callMessage.sdps[0] }); if (peerConnection) { await peerConnection.setRemoteDescription(remoteDesc); @@ -320,7 +306,7 @@ export async function handleIceCandidatesMessage( } callCache.get(sender)?.push(callMessage); - window.inboxStore?.dispatch(incomingCall({ sender })); + // window.inboxStore?.dispatch(incomingCall({ pubkey: sender })); if (peerConnection) { // tslint:disable-next-line: prefer-for-of for (let index = 0; index < callMessage.sdps.length; index++) { diff --git a/ts/state/ducks/conversations.ts b/ts/state/ducks/conversations.ts index 789d5974e..1319c044a 100644 --- a/ts/state/ducks/conversations.ts +++ b/ts/state/ducks/conversations.ts @@ -3,6 +3,7 @@ import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit'; import { getConversationController } from '../../session/conversations'; import { getFirstUnreadMessageIdInConversation, getMessagesByConversation } from '../../data/data'; import { + CallState, ConversationNotificationSettingType, ConversationTypeEnum, } from '../../models/conversation'; @@ -243,6 +244,7 @@ export interface ReduxConversationType { currentNotificationSetting?: ConversationNotificationSettingType; isPinned?: boolean; + callState?: CallState; } export interface NotificationForConvoOption { @@ -747,6 +749,81 @@ const conversationsSlice = createSlice({ state.mentionMembers = action.payload; return state; }, + incomingCall(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) { + const callerPubkey = action.payload.pubkey; + const existingCallState = state.conversationLookup[callerPubkey].callState; + if (existingCallState !== undefined && existingCallState !== 'none') { + return state; + } + const foundConvo = getConversationController().get(callerPubkey); + if (!foundConvo) { + return state; + } + + // we have to update the model itself. + // not the db (as we dont want to store that field in it) + // and not the redux store directly as it gets overriden by the commit() of the conversationModel + foundConvo.callState = 'incoming'; + + void foundConvo.commit(); + return state; + }, + endCall(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) { + const callerPubkey = action.payload.pubkey; + const existingCallState = state.conversationLookup[callerPubkey].callState; + if (!existingCallState || existingCallState === 'none') { + return state; + } + + const foundConvo = getConversationController().get(callerPubkey); + if (!foundConvo) { + return state; + } + + // we have to update the model itself. + // not the db (as we dont want to store that field in it) + // and not the redux store directly as it gets overriden by the commit() of the conversationModel + foundConvo.callState = 'none'; + + void foundConvo.commit(); + return state; + }, + answerCall(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) { + const callerPubkey = action.payload.pubkey; + const existingCallState = state.conversationLookup[callerPubkey].callState; + if (!existingCallState || existingCallState !== 'incoming') { + return state; + } + const foundConvo = getConversationController().get(callerPubkey); + if (!foundConvo) { + return state; + } + + // we have to update the model itself. + // not the db (as we dont want to store that field in it) + // and not the redux store directly as it gets overriden by the commit() of the conversationModel + + foundConvo.callState = 'connecting'; + void foundConvo.commit(); + return state; + }, + callConnected(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) { + const callerPubkey = action.payload.pubkey; + const existingCallState = state.conversationLookup[callerPubkey].callState; + if (!existingCallState || existingCallState === 'ongoing') { + return state; + } + const foundConvo = getConversationController().get(callerPubkey); + if (!foundConvo) { + return state; + } + // we have to update the model itself. + // not the db (as we dont want to store that field in it) + // and not the redux store directly as it gets overriden by the commit() of the conversationModel + foundConvo.callState = 'ongoing'; + void foundConvo.commit(); + return state; + }, }, extraReducers: (builder: any) => { // Add reducers for additional action types here, and handle loading state as needed @@ -806,6 +883,11 @@ export const { quotedMessageToAnimate, setNextMessageToPlayId, updateMentionsMembers, + // calls + incomingCall, + endCall, + answerCall, + callConnected, } = actions; export async function openConversationWithMessages(args: { diff --git a/ts/state/selectors/conversations.ts b/ts/state/selectors/conversations.ts index aecbdf0ec..2a01c8a38 100644 --- a/ts/state/selectors/conversations.ts +++ b/ts/state/selectors/conversations.ts @@ -66,6 +66,47 @@ export const getSelectedConversation = createSelector( } ); +export const getHasIncomingCallFrom = createSelector( + getConversations, + (state: ConversationsStateType): ReduxConversationType | undefined => { + const foundEntry = Object.entries(state.conversationLookup).find( + ([_convoKey, convo]) => convo.callState === 'incoming' + ); + + if (!foundEntry) { + return undefined; + } + return foundEntry[1]; + } +); + +export const getHasOngoingCallWith = createSelector( + getConversations, + (state: ConversationsStateType): ReduxConversationType | undefined => { + const foundEntry = Object.entries(state.conversationLookup).find( + ([_convoKey, convo]) => + convo.callState === 'connecting' || + convo.callState === 'offering' || + convo.callState === 'ongoing' + ); + + if (!foundEntry) { + return undefined; + } + return foundEntry[1]; + } +); + +export const getHasIncomingCall = createSelector( + getHasIncomingCallFrom, + (withConvo: ReduxConversationType | undefined): boolean => !!withConvo +); + +export const getHasOngoingCall = createSelector( + getHasOngoingCallWith, + (withConvo: ReduxConversationType | undefined): boolean => !!withConvo +); + /** * Returns true if the current conversation selected is a group conversation. * Returns false if the current conversation selected is not a group conversation, or none are selected From 94bc3da2c705366f676913687cd483235d449c05 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 23 Sep 2021 14:50:24 +1000 Subject: [PATCH 2/3] working video calls accept with real streaming with android --- .../session/calling/CallContainer.tsx | 3 +- ts/session/utils/CallManager.ts | 64 ++++++++++++++----- ts/state/ducks/conversations.ts | 18 ++++++ 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/ts/components/session/calling/CallContainer.tsx b/ts/components/session/calling/CallContainer.tsx index cf4bd433e..ef1a51781 100644 --- a/ts/components/session/calling/CallContainer.tsx +++ b/ts/components/session/calling/CallContainer.tsx @@ -110,7 +110,8 @@ export const CallContainer = () => { {ongoingCallProps.name} - + + diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index 25e8e4f19..28b737947 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -1,6 +1,12 @@ import _ from 'lodash'; import { SignalService } from '../../protobuf'; -import { answerCall, callConnected, endCall, incomingCall } from '../../state/ducks/conversations'; +import { + answerCall, + callConnected, + endCall, + incomingCall, + startingCallWith, +} from '../../state/ducks/conversations'; import { CallMessage } from '../messages/outgoing/controlMessage/CallMessage'; import { ed25519Str } from '../onions/onionPath'; import { getMessageQueue } from '../sending'; @@ -13,7 +19,7 @@ const callCache = new Map>(); let peerConnection: RTCPeerConnection | null; -const ENABLE_VIDEO = false; +const ENABLE_VIDEO = true; const configuration = { configuration: { @@ -32,7 +38,7 @@ const configuration = { // tslint:disable-next-line: function-name export async function USER_callRecipient(recipient: string) { window?.log?.info(`starting call with ${ed25519Str(recipient)}..`); - + window.inboxStore?.dispatch(startingCallWith({ pubkey: recipient })); if (peerConnection) { window.log.info('closing existing peerconnection'); peerConnection.close(); @@ -50,7 +56,9 @@ export async function USER_callRecipient(recipient: string) { window.inboxStore?.dispatch(callConnected({ pubkey: recipient })); } }); - + peerConnection.addEventListener('ontrack', event => { + console.warn('ontrack:', event); + }); peerConnection.addEventListener('icecandidate', event => { // window.log.warn('event.candidate', event.candidate); @@ -60,6 +68,20 @@ export async function USER_callRecipient(recipient: string) { } }); + const localVideo = document.querySelector('#video-local') as any; + if (localVideo) { + localVideo.srcObject = mediaDevices; + } + const remoteStream = new MediaStream(); + + peerConnection.addEventListener('track', event => { + const remoteVideo = document.querySelector('#video-remote') as any; + if (remoteVideo) { + remoteVideo.srcObject = remoteStream; + } + remoteStream.addTrack(event.track); + }); + const offerDescription = await peerConnection.createOffer({ offerToReceiveAudio: true, offerToReceiveVideo: ENABLE_VIDEO, @@ -155,6 +177,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { ); return; } + window.inboxStore?.dispatch(answerCall({ pubkey: fromSender })); if (peerConnection) { window.log.info('closing existing peerconnection'); @@ -167,19 +190,20 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { // window.log.info('USER_acceptIncomingCallRequest adding track ', track); peerConnection?.addTrack(track, mediaDevices); }); - peerConnection.addEventListener('icecandidateerror', event => { - console.warn('icecandidateerror:', event); - }); - peerConnection.addEventListener('negotiationneeded', event => { - console.warn('negotiationneeded:', event); - }); - peerConnection.addEventListener('signalingstatechange', _event => { - // console.warn('signalingstatechange:', event); - }); + const localVideo = document.querySelector('#video-local') as any; + if (localVideo) { + localVideo.srcObject = mediaDevices; + } - peerConnection.addEventListener('ontrack', event => { - console.warn('ontrack:', event); + const remoteStream = new MediaStream(); + + peerConnection.addEventListener('track', event => { + const remoteVideo = document.querySelector('#video-remote') as any; + if (remoteVideo) { + remoteVideo.srcObject = remoteStream; + } + remoteStream.addTrack(event.track); }); peerConnection.addEventListener('connectionstatechange', _event => { window.log.info( @@ -238,8 +262,6 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { window.log.info('sending ANSWER MESSAGE'); await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), callAnswerMessage); - - window.inboxStore?.dispatch(answerCall({ pubkey: fromSender })); } // tslint:disable-next-line: function-name @@ -258,6 +280,14 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) { export function handleEndCallMessage(sender: string) { callCache.delete(sender); + const remoteVideo = document.querySelector('#video-remote') as any; + if (remoteVideo) { + remoteVideo.srcObject = null; + } + const localVideo = document.querySelector('#video-local') as any; + if (localVideo) { + localVideo.srcObject = null; + } // // FIXME audric trigger UI cleanup window.inboxStore?.dispatch(endCall({ pubkey: sender })); diff --git a/ts/state/ducks/conversations.ts b/ts/state/ducks/conversations.ts index 1319c044a..f31be1a8a 100644 --- a/ts/state/ducks/conversations.ts +++ b/ts/state/ducks/conversations.ts @@ -824,6 +824,23 @@ const conversationsSlice = createSlice({ void foundConvo.commit(); return state; }, + startingCallWith(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) { + const callerPubkey = action.payload.pubkey; + const existingCallState = state.conversationLookup[callerPubkey].callState; + if (existingCallState && existingCallState !== 'none') { + return state; + } + const foundConvo = getConversationController().get(callerPubkey); + if (!foundConvo) { + return state; + } + // we have to update the model itself. + // not the db (as we dont want to store that field in it) + // and not the redux store directly as it gets overriden by the commit() of the conversationModel + foundConvo.callState = 'offering'; + void foundConvo.commit(); + return state; + }, }, extraReducers: (builder: any) => { // Add reducers for additional action types here, and handle loading state as needed @@ -888,6 +905,7 @@ export const { endCall, answerCall, callConnected, + startingCallWith, } = actions; export async function openConversationWithMessages(args: { From c54f63ab4547f41c60eaa28f7d942dbe0b0173ef Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Mon, 27 Sep 2021 13:57:31 +1000 Subject: [PATCH 3/3] add listener for video calls events --- _locales/en/messages.json | 6 +- package.json | 1 + .../session/calling/CallContainer.tsx | 88 ++++++++++++------- ts/session/utils/CallManager.ts | 43 +++++---- yarn.lock | 8 ++ 5 files changed, 93 insertions(+), 53 deletions(-) diff --git a/_locales/en/messages.json b/_locales/en/messages.json index 8b649412b..3af377b39 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -443,5 +443,9 @@ "messageDeletedPlaceholder": "This message has been deleted", "messageDeleted": "Message deleted", "surveyTitle": "Take our Session Survey", - "goToOurSurvey": "Go to our survey" + "goToOurSurvey": "Go to our survey", + "incomingCall": "Incoming call", + "accept": "Accept", + "decline": "Decline", + "endCall": "End call" } diff --git a/package.json b/package.json index 7eac174a1..2a7359247 100644 --- a/package.json +++ b/package.json @@ -102,6 +102,7 @@ "react": "^17.0.2", "react-contexify": "5.0.0", "react-dom": "^17.0.2", + "react-draggable": "^4.4.4", "react-emoji": "^0.5.0", "react-emoji-render": "^1.2.4", "react-h5-audio-player": "^3.2.0", diff --git a/ts/components/session/calling/CallContainer.tsx b/ts/components/session/calling/CallContainer.tsx index ef1a51781..b947af116 100644 --- a/ts/components/session/calling/CallContainer.tsx +++ b/ts/components/session/calling/CallContainer.tsx @@ -1,5 +1,9 @@ -import React from 'react'; +import React, { useEffect, useRef } from 'react'; import { useSelector } from 'react-redux'; +import Draggable from 'react-draggable'; + +// tslint:disable-next-line: no-submodule-imports +import useMountedState from 'react-use/lib/useMountedState'; import styled from 'styled-components'; import _ from 'underscore'; import { CallManager } from '../../../session/utils'; @@ -15,12 +19,14 @@ import { SessionWrapperModal } from '../SessionWrapperModal'; export const CallWindow = styled.div` position: absolute; z-index: 9; - padding: 2rem; + padding: 1rem; top: 50vh; left: 50vw; transform: translate(-50%, -50%); display: flex; flex-direction: column; + background-color: var(--color-modal-background); + border: var(--session-border); `; // similar styling to modal header @@ -28,7 +34,7 @@ const CallWindowHeader = styled.div` display: flex; flex-direction: row; justify-content: space-between; - align-items: center; + align-self: center; padding: $session-margin-lg; @@ -39,30 +45,29 @@ const CallWindowHeader = styled.div` font-weight: 700; `; -// TODO: Add proper styling for this -const VideoContainer = styled.video` - width: 200px; - height: 200px; +const VideoContainer = styled.div` + position: relative; + max-height: 60vh; +`; + +const VideoContainerRemote = styled.video` + max-height: inherit; +`; +const VideoContainerLocal = styled.video` + max-height: 45%; + max-width: 45%; + position: absolute; + bottom: 0; + right: 0; `; const CallWindowInner = styled.div` - position: relative; - background-color: pink; - border: 1px solid #d3d3d3; text-align: center; - padding: 2rem; - display: flex; - flex-direction: column; + padding: 1rem; `; const CallWindowControls = styled.div` - position: absolute; - top: 100%; - left: 0; - width: 100%; - /* background: green; */ padding: 5px; - transform: translateY(-100%); `; // TODO: @@ -77,6 +82,24 @@ export const CallContainer = () => { const hasOngoingCall = useSelector(getHasOngoingCall); const ongoingOrIncomingPubkey = ongoingCallProps?.id || incomingCallProps?.id; + const videoRefRemote = useRef(); + const videoRefLocal = useRef(); + const mountedState = useMountedState(); + + useEffect(() => { + CallManager.setVideoEventsListener( + (localStream: MediaStream | null, remoteStream: MediaStream | null) => { + if (mountedState() && videoRefRemote?.current && videoRefLocal?.current) { + videoRefLocal.current.srcObject = localStream; + videoRefRemote.current.srcObject = remoteStream; + } + } + ); + + return () => { + CallManager.setVideoEventsListener(null); + }; + }, []); //#region input handlers const handleAcceptIncomingCall = async () => { @@ -107,26 +130,31 @@ export const CallContainer = () => { if (hasOngoingCall && ongoingCallProps) { return ( - - - {ongoingCallProps.name} - - + + + Call with: {ongoingCallProps.name} + + + + + + + - + - - + + ); } if (hasIncomingCall) { return ( - +
- + diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index 28b737947..cc6621db3 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -12,6 +12,15 @@ import { ed25519Str } from '../onions/onionPath'; import { getMessageQueue } from '../sending'; import { PubKey } from '../types'; +type CallManagerListener = + | ((localStream: MediaStream | null, remoteStream: MediaStream | null) => void) + | null; +let videoEventsListener: CallManagerListener; + +export function setVideoEventsListener(listener: CallManagerListener) { + videoEventsListener = listener; +} + /** * This field stores all the details received by a sender about a call in separate messages. */ @@ -67,17 +76,15 @@ export async function USER_callRecipient(recipient: string) { void iceSenderDebouncer(recipient); } }); + const remoteStream = new MediaStream(); - const localVideo = document.querySelector('#video-local') as any; - if (localVideo) { - localVideo.srcObject = mediaDevices; + if (videoEventsListener) { + videoEventsListener(mediaDevices, remoteStream); } - const remoteStream = new MediaStream(); peerConnection.addEventListener('track', event => { - const remoteVideo = document.querySelector('#video-remote') as any; - if (remoteVideo) { - remoteVideo.srcObject = remoteStream; + if (videoEventsListener) { + videoEventsListener(mediaDevices, remoteStream); } remoteStream.addTrack(event.track); }); @@ -190,18 +197,15 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { // window.log.info('USER_acceptIncomingCallRequest adding track ', track); peerConnection?.addTrack(track, mediaDevices); }); + const remoteStream = new MediaStream(); - const localVideo = document.querySelector('#video-local') as any; - if (localVideo) { - localVideo.srcObject = mediaDevices; + if (videoEventsListener) { + videoEventsListener(mediaDevices, remoteStream); } - const remoteStream = new MediaStream(); - peerConnection.addEventListener('track', event => { - const remoteVideo = document.querySelector('#video-remote') as any; - if (remoteVideo) { - remoteVideo.srcObject = remoteStream; + if (videoEventsListener) { + videoEventsListener(mediaDevices, remoteStream); } remoteStream.addTrack(event.track); }); @@ -280,13 +284,8 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) { export function handleEndCallMessage(sender: string) { callCache.delete(sender); - const remoteVideo = document.querySelector('#video-remote') as any; - if (remoteVideo) { - remoteVideo.srcObject = null; - } - const localVideo = document.querySelector('#video-local') as any; - if (localVideo) { - localVideo.srcObject = null; + if (videoEventsListener) { + videoEventsListener(null, null); } // // FIXME audric trigger UI cleanup diff --git a/yarn.lock b/yarn.lock index 95d56a27e..9b47919d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7237,6 +7237,14 @@ react-dom@^17.0.2: object-assign "^4.1.1" scheduler "^0.20.2" +react-draggable@^4.4.4: + version "4.4.4" + resolved "https://registry.yarnpkg.com/react-draggable/-/react-draggable-4.4.4.tgz#5b26d9996be63d32d285a426f41055de87e59b2f" + integrity sha512-6e0WdcNLwpBx/YIDpoyd2Xb04PB0elrDrulKUgdrIlwuYvxh5Ok9M+F8cljm8kPXXs43PmMzek9RrB1b7mLMqA== + dependencies: + clsx "^1.1.1" + prop-types "^15.6.0" + react-emoji-render@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/react-emoji-render/-/react-emoji-render-1.2.4.tgz#fa3542a692e1eed3236f0f12b8e3a61b2818e2c2"