diff --git a/_locales/en/messages.json b/_locales/en/messages.json index cf1836256..3df2bc21d 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -454,5 +454,7 @@ "unableToCallTitle": "Cannot start new call", "callMissed": "Missed call from $name$", "callMissedTitle": "Call missed", - "startVideoCall": "Start Video Call" + "startVideoCall": "Start Video Call", + "noCameraFound": "No camera found", + "noAudioInputFound": "No audio input found" } diff --git a/ts/components/Lightbox.tsx b/ts/components/Lightbox.tsx index 339953747..e777d43ff 100644 --- a/ts/components/Lightbox.tsx +++ b/ts/components/Lightbox.tsx @@ -133,7 +133,7 @@ interface IconButtonProps { } const IconButton = ({ onClick, type }: IconButtonProps) => { - const clickHandler = (_event: React.MouseEvent): void => { + const clickHandler = (): void => { if (!onClick) { return; } diff --git a/ts/components/session/calling/CallContainer.tsx b/ts/components/session/calling/CallContainer.tsx index 8b746a8ba..625cf11f1 100644 --- a/ts/components/session/calling/CallContainer.tsx +++ b/ts/components/session/calling/CallContainer.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useSelector } from 'react-redux'; import Draggable, { DraggableData, DraggableEvent } from 'react-draggable'; @@ -6,20 +6,23 @@ import Draggable, { DraggableData, DraggableEvent } from 'react-draggable'; import useMountedState from 'react-use/lib/useMountedState'; import styled from 'styled-components'; import _ from 'underscore'; -import { CallManager } from '../../../session/utils'; +import { CallManager, ToastUtils } from '../../../session/utils'; import { getHasOngoingCall, getHasOngoingCallWith, getSelectedConversationKey, } from '../../../state/selectors/conversations'; -import { SessionButton } from '../SessionButton'; +import { openConversationWithMessages } from '../../../state/ducks/conversations'; +import { SessionIconButton } from '../icon'; +import { animation, contextMenu, Item, Menu } from 'react-contexify'; +import { InputItem } from '../../../session/utils/CallManager'; export const DraggableCallWindow = styled.div` position: absolute; z-index: 9; box-shadow: var(--color-session-shadow); max-height: 300px; - width: 300px; + width: 12vw; display: flex; flex-direction: column; background-color: var(--color-modal-background); @@ -36,18 +39,59 @@ const StyledDraggableVideoElement = styled(StyledVideoElement)` padding: 0 0; `; -const CallWindowControls = styled.div` - padding: 5px; - flex-shrink: 0; +const DraggableCallWindowInner = styled.div` + cursor: pointer; `; -const DraggableCallWindowInner = styled.div``; - const VideoContainer = styled.div` height: 100%; width: 50%; `; +export const InConvoCallWindow = styled.div` + padding: 1rem; + display: flex; + height: 50%; + + background: radial-gradient(black, #505050); + + flex-shrink: 0; + min-height: 200px; + align-items: center; +`; + +const InConvoCallWindowControls = styled.div` + position: absolute; + + bottom: 0px; + width: fit-content; + padding: 10px; + border-radius: 10px; + height: 45px; + margin-left: auto; + margin-right: auto; + left: 0; + right: 0; + transition: all 0.25s ease-in-out; + + display: flex; + background-color: white; + + align-items: center; + justify-content: center; + opacity: 0.3; + &:hover { + opacity: 1; + } +`; + +const RelativeCallWindow = styled.div` + position: relative; + height: 100%; + display: flex; + flex-grow: 1; +`; + // TODO: /** * Add mute input, deafen, end call, possibly add person to call @@ -58,8 +102,10 @@ export const DraggableCallContainer = () => { const selectedConversationKey = useSelector(getSelectedConversationKey); const hasOngoingCall = useSelector(getHasOngoingCall); - const [positionX, setPositionX] = useState(0); - const [positionY, setPositionY] = useState(0); + const [positionX, setPositionX] = useState(window.innerWidth / 2); + const [positionY, setPositionY] = useState(window.innerHeight / 2); + const [lastPositionX, setLastPositionX] = useState(0); + const [lastPositionY, setLastPositionY] = useState(0); const ongoingCallPubkey = ongoingCallProps?.id; const videoRefRemote = useRef(undefined); @@ -96,25 +142,30 @@ export const DraggableCallContainer = () => { }; }, [ongoingCallPubkey, selectedConversationKey]); - const handleEndCall = async () => { - // call method to end call connection - if (ongoingCallPubkey) { - await CallManager.USER_rejectIncomingCallRequest(ongoingCallPubkey); + const openCallingConversation = useCallback(() => { + if (ongoingCallPubkey && ongoingCallPubkey !== selectedConversationKey) { + void openConversationWithMessages({ conversationKey: ongoingCallPubkey }); } - }; + }, [ongoingCallPubkey, selectedConversationKey]); if (!hasOngoingCall || !ongoingCallProps || ongoingCallPubkey === selectedConversationKey) { return null; } - console.warn('rendering with pos', positionX, positionY); - return ( { - console.warn('setting position ', { x: data.x, y: data.y }); + onStart={(_e: DraggableEvent, data: DraggableData) => { + setLastPositionX(data.x); + setLastPositionY(data.y); + }} + onStop={(e: DraggableEvent, data: DraggableData) => { + e.stopPropagation(); + if (data.x === lastPositionX && data.y === lastPositionY) { + // drag did not change anything. Consider this to be a click + openCallingConversation(); + } setPositionX(data.x); setPositionY(data.y); }} @@ -123,45 +174,93 @@ export const DraggableCallContainer = () => { - - - ); }; -export const InConvoCallWindow = styled.div` - padding: 1rem; - display: flex; - height: 50%; - - /* background-color: var(--color-background-primary); */ - - background: radial-gradient(black, #505050); +const VideoInputMenu = ({ + triggerId, + camerasList, +}: { + triggerId: string; + camerasList: Array; +}) => { + return ( + + {camerasList.map(m => { + return ( + { + void CallManager.selectCameraByDeviceId(m.deviceId); + }} + > + {m.label.substr(0, 40)} + + ); + })} + + ); +}; - flex-shrink: 0; - min-height: 200px; - align-items: center; -`; +const AudioInputMenu = ({ + triggerId, + audioInputsList, +}: { + triggerId: string; + audioInputsList: Array; +}) => { + return ( + + {audioInputsList.map(m => { + return ( + { + void CallManager.selectAudioInputByDeviceId(m.deviceId); + }} + > + {m.label.substr(0, 40)} + + ); + })} + + ); +}; export const InConversationCallContainer = () => { const ongoingCallProps = useSelector(getHasOngoingCallWith); const selectedConversationKey = useSelector(getSelectedConversationKey); const hasOngoingCall = useSelector(getHasOngoingCall); + const [currentConnectedCameras, setCurrentConnectedCameras] = useState>([]); + const [currentConnectedAudioInputs, setCurrentConnectedAudioInputs] = useState>( + [] + ); const ongoingCallPubkey = ongoingCallProps?.id; const videoRefRemote = useRef(); const videoRefLocal = useRef(); const mountedState = useMountedState(); + const videoTriggerId = 'video-menu-trigger-id'; + const audioTriggerId = 'audio-menu-trigger-id'; + useEffect(() => { if (ongoingCallPubkey === selectedConversationKey) { CallManager.setVideoEventsListener( - (localStream: MediaStream | null, remoteStream: MediaStream | null) => { + ( + localStream: MediaStream | null, + remoteStream: MediaStream | null, + camerasList: Array, + audioInputList: Array + ) => { if (mountedState() && videoRefRemote?.current && videoRefLocal?.current) { videoRefLocal.current.srcObject = localStream; videoRefRemote.current.srcObject = remoteStream; + setCurrentConnectedCameras(camerasList); + + setCurrentConnectedAudioInputs(audioInputList); } } ); @@ -169,6 +268,8 @@ export const InConversationCallContainer = () => { return () => { CallManager.setVideoEventsListener(null); + setCurrentConnectedCameras([]); + setCurrentConnectedAudioInputs([]); }; }, [ongoingCallPubkey, selectedConversationKey]); @@ -185,12 +286,58 @@ export const InConversationCallContainer = () => { return ( - - - - - - + + + + + + + + + + + ) => { + if (currentConnectedCameras.length === 0) { + ToastUtils.pushNoCameraFound(); + return; + } + contextMenu.show({ + id: videoTriggerId, + event: e, + }); + }} + iconColor="black" + /> + ) => { + if (currentConnectedAudioInputs.length === 0) { + ToastUtils.pushNoAudioInputFound(); + return; + } + contextMenu.show({ + id: audioTriggerId, + event: e, + }); + }} + /> + + + + ); }; diff --git a/ts/components/session/icon/Icons.tsx b/ts/components/session/icon/Icons.tsx index de70a859c..8ed5c0b01 100644 --- a/ts/components/session/icon/Icons.tsx +++ b/ts/components/session/icon/Icons.tsx @@ -22,10 +22,12 @@ export type SessionIconType = | 'file' | 'gear' | 'globe' + | 'hangup' | 'info' | 'link' | 'lock' | 'microphone' + | 'microphoneFull' | 'moon' | 'mute' | 'oxen' @@ -60,7 +62,8 @@ export type SessionIconType = | 'timer45' | 'timer50' | 'timer55' - | 'timer60'; + | 'timer60' + | 'videoCamera'; export type SessionIconSize = 'tiny' | 'small' | 'medium' | 'large' | 'huge' | 'huge2' | 'max'; @@ -205,6 +208,12 @@ export const icons = { viewBox: '0.5 0 30 30', ratio: 1, }, + hangup: { + path: + 'M983.7,530.6c7.7,53.1,12.6,125.7-11.1,153.6c-39.4,46-288.8,46-288.8-46c0-46.3,41-76.7,1.7-122.7c-38.7-45.2-108.2-46-185.4-46s-146.7,0.7-185.4,46c-39.4,46,1.7,76.3,1.7,122.7c0,92-249.4,92-288.8,46C3.7,656.4,8.7,583.7,16.3,530.6c5.9-35.5,20.8-73.7,68.5-122.5l0,0c71.5-66.8,179.8-121.3,411.4-122.5v0c1.3,0,2.5,0,3.8,0s2.5,0,3.8,0v0c231.6,1.2,339.8,55.7,411.4,122.5l0,0C962.9,456.9,977.8,495.2,983.7,530.6z', + viewBox: '0 0 1000 1000', + ratio: 1, + }, info: { path: 'M17.5,2.4c-1.82-1.5-4.21-2.1-6.57-1.64c-3.09,0.6-5.57,3.09-6.15,6.19c-0.4,2.1,0.04,4.21,1.22,5.95 C7.23,14.7,8,16.41,8.36,18.12c0.17,0.81,0.89,1.41,1.72,1.41h4.85c0.83,0,1.55-0.59,1.72-1.42c0.37-1.82,1.13-3.55,2.19-4.99 c1-1.36,1.53-2.96,1.53-4.65C20.37,6.11,19.32,3.9,17.5,2.4z M17.47,12.11c-1.21,1.64-2.07,3.6-2.55,5.72l-4.91-0.05 c-0.4-1.93-1.25-3.84-2.62-5.84c-0.93-1.36-1.27-3.02-0.95-4.67c0.46-2.42,2.39-4.37,4.81-4.83c0.41-0.08,0.82-0.12,1.23-0.12 c1.44,0,2.82,0.49,3.94,1.4c1.43,1.18,2.25,2.91,2.25,4.76C18.67,9.79,18.25,11.04,17.47,12.11z M15.94,20.27H9.61c-0.47,0-0.85,0.38-0.85,0.85s0.38,0.85,0.85,0.85h6.33c0.47,0,0.85-0.38,0.85-0.85 S16.41,20.27,15.94,20.27z M15.94,22.7H9.61c-0.47,0-0.85,0.38-0.85,0.85s0.38,0.85,0.85,0.85h6.33c0.47,0,0.85-0.38,0.85-0.85 S16.41,22.7,15.94,22.7z M12.5,3.28c-2.89,0-5.23,2.35-5.23,5.23c0,0.47,0.38,0.85,0.85,0.85s0.85-0.38,0.85-0.85 c0-1.95,1.59-3.53,3.54-3.53c0.47,0,0.85-0.38,0.85-0.85S12.97,3.28,12.5,3.28z', @@ -229,6 +238,12 @@ export const icons = { viewBox: '28 0 30 30', ratio: 1, }, + microphoneFull: { + path: + 'M44,28c-0.552,0-1,0.447-1,1v6c0,7.72-6.28,14-14,14s-14-6.28-14-14v-6c0-0.553-0.448-1-1-1s-1,0.447-1,1v6c0,8.485,6.644,15.429,15,15.949V56h-5c-0.552,0-1,0.447-1,1s0.448,1,1,1h12c0.552,0,1-0.447,1-1s-0.448-1-1-1h-5v-5.051c8.356-0.52,15-7.465,15-15.949v-6C45,28.447,44.552,28,44,28zM29,46c6.065,0,11-4.935,11-11V11c0-6.065-4.935-11-11-11S18,4.935,18,11v24C18,41.065,22.935,46,29,46z', + viewBox: '0 0 58 58', + ratio: 1, + }, moon: { path: 'M11.1441877,12.8180303 C8.90278993,10.5766325 8.24397847,7.29260898 9.27752593,4.437982 C6.09633644,5.5873034 3.89540402,8.67837285 4.00385273,12.2078365 C4.13368986,16.4333868 7.52883112,19.8285281 11.7543814,19.9583652 C15.2838451,20.0668139 18.3749145,17.8658815 19.5242359,14.684692 C16.669609,15.7182395 13.3855854,15.059428 11.1441877,12.8180303 Z M21.9576498,12.8823459 C21.4713729,18.1443552 16.9748949,22.1197182 11.692957,21.9574217 C6.41101918,21.7951253 2.16709261,17.5511988 2.00479619,12.2692609 C1.84249977,6.98732307 5.81786273,2.49084501 11.0798721,2.00456809 C11.9400195,1.92507947 12.4895134,2.90008536 11.9760569,3.59473245 C10.2106529,5.98311963 10.4582768,9.30369233 12.5584012,11.4038167 C14.6585256,13.5039411 17.9790983,13.7515651 20.3674855,11.986161 C21.0621326,11.4727046 22.0371385,12.0221984 21.9576498,12.8823459', @@ -441,4 +456,10 @@ export const icons = { viewBox: '0 0 12 12', ratio: 1, }, + videoCamera: { + path: + 'M488.3,142.5v203.1c0,15.7-17,25.5-30.6,17.7l-84.6-48.8v13.9c0,41.8-33.9,75.7-75.7,75.7H75.7C33.9,404.1,0,370.2,0,328.4 V159.9c0-41.8,33.9-75.7,75.7-75.7h221.8c41.8,0,75.7,33.9,75.7,75.7v13.9l84.6-48.8C471.3,117,488.3,126.9,488.3,142.5z', + viewBox: '0 0 488.3 488.3', + ratio: 1, + }, }; diff --git a/ts/components/session/icon/SessionIconButton.tsx b/ts/components/session/icon/SessionIconButton.tsx index 89f5ac0a3..e56c92415 100644 --- a/ts/components/session/icon/SessionIconButton.tsx +++ b/ts/components/session/icon/SessionIconButton.tsx @@ -5,7 +5,7 @@ import { SessionNotificationCount } from '../SessionNotificationCount'; import _ from 'lodash'; interface SProps extends SessionIconProps { - onClick?: any; + onClick?: (e: React.MouseEvent) => void; notificationCount?: number; isSelected?: boolean; isHidden?: boolean; @@ -27,10 +27,10 @@ const SessionIconButtonInner = (props: SProps) => { borderRadius, iconPadding, } = props; - const clickHandler = (e: any) => { + const clickHandler = (e: React.MouseEvent) => { if (props.onClick) { e.stopPropagation(); - props.onClick(); + props.onClick(e); } }; diff --git a/ts/session/utils/CallManager.ts b/ts/session/utils/CallManager.ts index f23fea93f..f519e1e8f 100644 --- a/ts/session/utils/CallManager.ts +++ b/ts/session/utils/CallManager.ts @@ -17,19 +17,29 @@ import { CallMessage } from '../messages/outgoing/controlMessage/CallMessage'; import { ed25519Str } from '../onions/onionPath'; import { getMessageQueue } from '../sending'; import { PubKey } from '../types'; +export type InputItem = { deviceId: string; label: string }; type CallManagerListener = - | ((localStream: MediaStream | null, remoteStream: MediaStream | null) => void) + | (( + localStream: MediaStream | null, + remoteStream: MediaStream | null, + camerasList: Array, + audioInputsList: Array + ) => void) | null; let videoEventsListener: CallManagerListener; -export function setVideoEventsListener(listener: CallManagerListener) { - videoEventsListener = listener; +function callVideoListener() { if (videoEventsListener) { - videoEventsListener(mediaDevices, remoteStream); + videoEventsListener(mediaDevices, remoteStream, camerasList, audioInputsList); } } +export function setVideoEventsListener(listener: CallManagerListener) { + videoEventsListener = listener; + callVideoListener(); +} + /** * This field stores all the details received by a sender about a call in separate messages. */ @@ -39,8 +49,6 @@ let peerConnection: RTCPeerConnection | null; let remoteStream: MediaStream | null; let mediaDevices: MediaStream | null; -const ENABLE_VIDEO = true; - let makingOffer = false; let ignoreOffer = false; let isSettingRemoteAnswerPending = false; @@ -49,7 +57,7 @@ let lastOutgoingOfferTimestamp = -Infinity; const configuration = { configuration: { offerToReceiveAudio: true, - offerToReceiveVideo: ENABLE_VIDEO, + offerToReceiveVideo: true, }, iceServers: [ { urls: 'stun:stun.l.google.com:19302' }, @@ -60,8 +68,129 @@ const configuration = { ], }; +let selectedCameraId: string | undefined; +let selectedAudioInputId: string | undefined; +let camerasList: Array = []; +let audioInputsList: Array = []; + +async function getConnectedDevices(type: 'videoinput' | 'audioinput') { + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices.filter(device => device.kind === type); +} + +// Listen for changes to media devices and update the list accordingly +navigator.mediaDevices.addEventListener('devicechange', async () => { + await updateInputLists(); + callVideoListener(); +}); + +async function updateInputLists() { + // Get the set of cameras connected + const videoCameras = await getConnectedDevices('videoinput'); + camerasList = videoCameras.map(m => ({ + deviceId: m.deviceId, + label: m.label, + })); + // Get the set of audio inputs connected + const audiosInput = await getConnectedDevices('audioinput'); + audioInputsList = audiosInput.map(m => ({ + deviceId: m.deviceId, + label: m.label, + })); +} + +export async function selectCameraByDeviceId(cameraDeviceId: string) { + console.warn('selecting cameraDeviceId ', cameraDeviceId); + + if (camerasList.some(m => m.deviceId === cameraDeviceId)) { + selectedCameraId = cameraDeviceId; + + try { + mediaDevices = await openMediaDevices({ + audioInputId: selectedAudioInputId, + cameraId: selectedCameraId, + }); + + mediaDevices.getTracks().map((track: MediaStreamTrack) => { + window.log.info('selectCameraByDeviceId adding track: ', track); + if (mediaDevices) { + peerConnection?.addTrack(track, mediaDevices); + } + }); + callVideoListener(); + } catch (err) { + console.warn('err', err); + } + } +} +export async function selectAudioInputByDeviceId(audioInputDeviceId: string) { + console.warn('selecting audioInputDeviceId', audioInputDeviceId); + if (audioInputsList.some(m => m.deviceId === audioInputDeviceId)) { + selectedAudioInputId = audioInputDeviceId; + try { + mediaDevices = await openMediaDevices({ + audioInputId: selectedAudioInputId, + cameraId: selectedCameraId, + }); + + mediaDevices.getTracks().map((track: MediaStreamTrack) => { + window.log.info('selectAudioInputByDeviceId adding track: ', track); + if (mediaDevices) { + peerConnection?.addTrack(track, mediaDevices); + } + }); + callVideoListener(); + } catch (err) { + console.warn('err', err); + } + } +} + +async function handleNegotiationNeededEvent(event: Event, recipient: string) { + window.log?.warn('negotiationneeded:', event); + try { + makingOffer = true; + const offer = await peerConnection?.createOffer(); + + if (!offer) { + throw new Error('Could not create offer in handleNegotiationNeededEvent'); + } + await peerConnection?.setLocalDescription(offer); + + if (offer && offer.sdp) { + const negotationOfferMessage = new CallMessage({ + timestamp: Date.now(), + type: SignalService.CallMessage.Type.OFFER, + sdps: [offer.sdp], + }); + + window.log.info('sending OFFER MESSAGE'); + const negotationOfferSendResult = await getMessageQueue().sendToPubKeyNonDurably( + PubKey.cast(recipient), + negotationOfferMessage + ); + if (typeof negotationOfferSendResult === 'number') { + window.log?.warn('setting last sent timestamp'); + lastOutgoingOfferTimestamp = negotationOfferSendResult; + } + } + } catch (err) { + window.log?.error(`Error on handling negotiation needed ${err}`); + } finally { + makingOffer = false; + } +} + +function handleIceCandidates(event: RTCPeerConnectionIceEvent, pubkey: string) { + if (event.candidate) { + iceCandidates.push(event.candidate); + void iceSenderDebouncer(pubkey); + } +} + // tslint:disable-next-line: function-name export async function USER_callRecipient(recipient: string) { + await updateInputLists(); window?.log?.info(`starting call with ${ed25519Str(recipient)}..`); window.inboxStore?.dispatch(startingCallWith({ pubkey: recipient })); if (peerConnection) { @@ -72,7 +201,8 @@ export async function USER_callRecipient(recipient: string) { peerConnection = new RTCPeerConnection(configuration); try { - mediaDevices = await openMediaDevices(); + mediaDevices = await openMediaDevices({}); + mediaDevices.getTracks().map((track: any) => { window.log.info('USER_callRecipient adding track: ', track); if (mediaDevices) { @@ -85,99 +215,27 @@ export async function USER_callRecipient(recipient: string) { window.inboxStore?.dispatch(showSettingsSection(SessionSettingCategory.Privacy)); }); } - peerConnection.addEventListener('connectionstatechange', _event => { - window.log.info('peerConnection?.connectionState caller :', peerConnection?.connectionState); - if (peerConnection?.connectionState === 'connected') { - window.inboxStore?.dispatch(callConnected({ pubkey: recipient })); - } + peerConnection.addEventListener('connectionstatechange', () => { + handleConnectionStateChanged(recipient); }); - peerConnection.addEventListener('ontrack', event => { - window.log?.warn('ontrack:', event); - }); - peerConnection.addEventListener('icecandidate', event => { - // window.log.warn('event.candidate', event.candidate); - if (event.candidate) { - iceCandidates.push(event.candidate); - void iceSenderDebouncer(recipient); - } + peerConnection.addEventListener('icecandidate', event => { + handleIceCandidates(event, recipient); }); - // peerConnection.addEventListener('negotiationneeded', async event => { - peerConnection.onnegotiationneeded = async event => { - window.log?.warn('negotiationneeded:', event); - try { - makingOffer = true; - // @ts-ignore - await peerConnection?.setLocalDescription(); - const offer = await peerConnection?.createOffer(); - window.log?.warn(offer); - - if (offer && offer.sdp) { - const negotationOfferMessage = new CallMessage({ - timestamp: Date.now(), - type: SignalService.CallMessage.Type.OFFER, - sdps: [offer.sdp], - }); - - window.log.info('sending OFFER MESSAGE'); - const negotationOfferSendResult = await getMessageQueue().sendToPubKeyNonDurably( - PubKey.cast(recipient), - negotationOfferMessage - ); - if (typeof negotationOfferSendResult === 'number') { - window.log?.warn('setting last sent timestamp'); - lastOutgoingOfferTimestamp = negotationOfferSendResult; - } - // debug: await new Promise(r => setTimeout(r, 10000)); adding artificial wait for offer debugging - } - } catch (err) { - window.log?.error(`Error on handling negotiation needed ${err}`); - } finally { - makingOffer = false; - } + peerConnection.onnegotiationneeded = async (event: Event) => { + await handleNegotiationNeededEvent(event, recipient); }; remoteStream = new MediaStream(); - if (videoEventsListener) { - videoEventsListener(mediaDevices, remoteStream); - } + callVideoListener(); peerConnection.addEventListener('track', event => { - if (videoEventsListener) { - videoEventsListener(mediaDevices, remoteStream); - } + callVideoListener(); if (remoteStream) { remoteStream.addTrack(event.track); } }); - - const offerDescription = await peerConnection.createOffer({ - offerToReceiveAudio: true, - offerToReceiveVideo: ENABLE_VIDEO, - }); - - if (!offerDescription || !offerDescription.sdp || !offerDescription.sdp.length) { - window.log.warn(`failed to createOffer for recipient ${ed25519Str(recipient)}`); - return; - } - await peerConnection.setLocalDescription(offerDescription); - const offerMessage = new CallMessage({ - timestamp: Date.now(), - type: SignalService.CallMessage.Type.OFFER, - sdps: [offerDescription.sdp], - }); - - window.log.info('sending OFFER MESSAGE'); - const offerSendResult = await getMessageQueue().sendToPubKeyNonDurably( - PubKey.cast(recipient), - offerMessage - ); - if (typeof offerSendResult === 'number') { - window.log?.warn('setting timestamp'); - lastOutgoingOfferTimestamp = offerSendResult; - } - // FIXME audric dispatch UI update to show the calling UI } const iceCandidates: Array = new Array(); @@ -214,10 +272,29 @@ const iceSenderDebouncer = _.debounce(async (recipient: string) => { await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(recipient), callIceCandicates); }, 2000); -const openMediaDevices = async () => { +const openMediaDevices = async ({ + audioInputId, + cameraId, +}: { + cameraId?: string; + audioInputId?: string; +}) => { + if (mediaDevices) { + window.log.info('stopping existing tracks in openMediaDevices'); + mediaDevices.getTracks().forEach(track => { + track.stop(); + }); + } + window.log.info('openMediaDevices ', { audioInputId, cameraId }); + return navigator.mediaDevices.getUserMedia({ - video: ENABLE_VIDEO, - audio: true, + audio: { + deviceId: audioInputId ? { exact: audioInputId } : undefined, + echoCancellation: true, + }, + video: { + deviceId: cameraId ? { exact: cameraId } : undefined, + }, }); }; @@ -234,9 +311,57 @@ const findLastMessageTypeFromSender = (sender: string, msgType: SignalService.Ca return lastOfferMessage; }; +function handleSignalingStateChangeEvent() { + if (peerConnection?.signalingState === 'closed') { + closeVideoCall(); + } +} + +function handleConnectionStateChanged(pubkey: string) { + window.log.info('handleConnectionStateChanged :', peerConnection?.connectionState); + + if (peerConnection?.signalingState === 'closed') { + closeVideoCall(); + } else if (peerConnection?.connectionState === 'connected') { + window.inboxStore?.dispatch(callConnected({ pubkey })); + } +} + +function closeVideoCall() { + if (peerConnection) { + peerConnection.ontrack = null; + peerConnection.onicecandidate = null; + peerConnection.oniceconnectionstatechange = null; + peerConnection.onconnectionstatechange = null; + peerConnection.onsignalingstatechange = null; + peerConnection.onicegatheringstatechange = null; + peerConnection.onnegotiationneeded = null; + + if (mediaDevices) { + mediaDevices.getTracks().forEach(track => { + track.stop(); + }); + } + + if (remoteStream) { + remoteStream.getTracks().forEach(track => { + track.stop(); + }); + } + + peerConnection.close(); + peerConnection = null; + } + + if (videoEventsListener) { + videoEventsListener(null, null, [], []); + } +} + // tslint:disable-next-line: function-name export async function USER_acceptIncomingCallRequest(fromSender: string) { const msgCacheFromSender = callCache.get(fromSender); + await updateInputLists(); if (!msgCacheFromSender) { window?.log?.info( 'incoming call request cannot be accepted as the corresponding message is not found' @@ -262,7 +387,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { peerConnection = null; } peerConnection = new RTCPeerConnection(configuration); - mediaDevices = await openMediaDevices(); + mediaDevices = await openMediaDevices({}); mediaDevices.getTracks().map(track => { // window.log.info('USER_acceptIncomingCallRequest adding track ', track); if (mediaDevices) { @@ -272,35 +397,22 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { remoteStream = new MediaStream(); peerConnection.addEventListener('icecandidate', event => { - window.log?.warn('icecandidateerror:', event); - // TODO: ICE stuff - // signaler.send({candidate}); // probably event.candidate + if (event.candidate) { + iceCandidates.push(event.candidate); + void iceSenderDebouncer(fromSender); + } }); - peerConnection.addEventListener('signalingstatechange', event => { - window.log?.warn('signalingstatechange:', event); - }); + peerConnection.addEventListener('signalingstatechange', handleSignalingStateChangeEvent); - if (videoEventsListener) { - videoEventsListener(mediaDevices, remoteStream); - } + callVideoListener(); peerConnection.addEventListener('track', event => { - if (videoEventsListener) { - videoEventsListener(mediaDevices, remoteStream); - } + callVideoListener(); remoteStream?.addTrack(event.track); }); - peerConnection.addEventListener('connectionstatechange', _event => { - window.log.info( - 'peerConnection?.connectionState recipient:', - peerConnection?.connectionState, - 'with: ', - fromSender - ); - if (peerConnection?.connectionState === 'connected') { - window.inboxStore?.dispatch(callConnected({ pubkey: fromSender })); - } + peerConnection.addEventListener('connectionstatechange', () => { + handleConnectionStateChanged(fromSender); }); const { sdps } = lastOfferMessage; @@ -320,7 +432,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) { const answer = await peerConnection.createAnswer({ offerToReceiveAudio: true, - offerToReceiveVideo: ENABLE_VIDEO, + offerToReceiveVideo: true, }); if (!answer?.sdp || answer.sdp.length === 0) { window.log.warn('failed to create answer'); @@ -371,7 +483,7 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) { export function handleEndCallMessage(sender: string) { callCache.delete(sender); if (videoEventsListener) { - videoEventsListener(null, null); + videoEventsListener(null, null, [], []); } mediaDevices = null; remoteStream = null; @@ -387,9 +499,15 @@ export async function handleOfferCallMessage( ) { try { const convos = getConversationController().getConversations(); - if (convos.some(convo => convo.callState !== undefined)) { - await handleMissedCall(sender, incomingOfferTimestamp); - return; + const callingConvos = convos.filter(convo => convo.callState !== undefined); + if (callingConvos.length > 0) { + // we just got a new offer from someone we are already in a call with + if (callingConvos.length === 1 && callingConvos[0].id === sender) { + window.log.info('Got a new offer message from our ongoing call'); + } else { + await handleMissedCall(sender, incomingOfferTimestamp); + return; + } } const readyForOffer = @@ -481,7 +599,7 @@ export async function handleIceCandidatesMessage( await peerConnection.addIceCandidate(candicate); } catch (err) { if (!ignoreOffer) { - window.log?.warn('Error handling ICE candidates message'); + window.log?.warn('Error handling ICE candidates message', err); } } } diff --git a/ts/session/utils/Toast.tsx b/ts/session/utils/Toast.tsx index 1ce498724..e1271a331 100644 --- a/ts/session/utils/Toast.tsx +++ b/ts/session/utils/Toast.tsx @@ -235,3 +235,11 @@ export function pushUserRemovedFromModerators() { export function pushInvalidPubKey() { pushToastSuccess('invalidPubKey', window.i18n('invalidPubkeyFormat')); } + +export function pushNoCameraFound() { + pushToastWarning('noCameraFound', window.i18n('noCameraFound')); +} + +export function pushNoAudioInputFound() { + pushToastWarning('noAudioInputFound', window.i18n('noAudioInputFound')); +}