keejef
/
session-desktop
mirror of https://github.com/oxen-io/session-desktop
2
0
Fork 0
Code Issues 8 Projects Releases Wiki Activity

fix toggling input source and off in video calls

Browse Source
pull/1969/head
Audric Ackermann 4 years ago
parent fbd51c2974
commit a629985a4f
No known key found for this signature in database
GPG Key ID: 999F434D76324AD4
4 changed files with 249 additions and 188 deletions
Show Stats Download Patch File Download Diff File

35
ts/receiver/callMessage.ts
Unescape Escape View File

@ -18,25 +18,14 @@ export async function handleCallMessage(
const sentTimestamp = _.toNumber(envelope.timestamp);
const { type } = callMessage;
switch (type) {
case SignalService.CallMessage.Type.END_CALL:
window.log.info('handling callMessage END_CALL');
break;
case SignalService.CallMessage.Type.ANSWER:
window.log.info('handling callMessage ANSWER');
break;
case SignalService.CallMessage.Type.ICE_CANDIDATES:
window.log.info('handling callMessage ICE_CANDIDATES');
break;
case SignalService.CallMessage.Type.OFFER:
window.log.info('handling callMessage OFFER');
break;
case SignalService.CallMessage.Type.PROVISIONAL_ANSWER:
window.log.info('handling callMessage PROVISIONAL_ANSWER');
break;
default:
window.log.info('handling callMessage unknown');
if (type === SignalService.CallMessage.Type.PROVISIONAL_ANSWER) {
await removeFromCache(envelope);
window.log.info('Skipping callMessage PROVISIONAL_ANSWER');
return;
}
if (type === SignalService.CallMessage.Type.OFFER) {
if (Math.max(sentTimestamp - (Date.now() - currentOffset)) > TTL_DEFAULT.CALL_MESSAGE) {
window?.log?.info('Dropping incoming OFFER callMessage sent a while ago: ', sentTimestamp);
@ -46,7 +35,7 @@ export async function handleCallMessage(
}
await removeFromCache(envelope);
await CallManager.handleOfferCallMessage(sender, callMessage, sentTimestamp);
await CallManager.handleCallTypeOffer(sender, callMessage, sentTimestamp);
return;
}
@ -54,7 +43,7 @@ export async function handleCallMessage(
if (type === SignalService.CallMessage.Type.END_CALL) {
await removeFromCache(envelope);
CallManager.handleEndCallMessage(sender);
CallManager.handleCallTypeEndCall(sender);
return;
}
@ -62,19 +51,19 @@ export async function handleCallMessage(
if (type === SignalService.CallMessage.Type.ANSWER) {
await removeFromCache(envelope);
await CallManager.handleCallAnsweredMessage(sender, callMessage);
await CallManager.handleCallTypeAnswer(sender, callMessage);
return;
}
if (type === SignalService.CallMessage.Type.ICE_CANDIDATES) {
await removeFromCache(envelope);
await CallManager.handleIceCandidatesMessage(sender, callMessage);
await CallManager.handleCallTypeIceCandidates(sender, callMessage);
return;
}
await removeFromCache(envelope);
// if this another type of call message, just add it to the manager
await CallManager.handleOtherCallMessage(sender, callMessage);
await CallManager.handleOtherCallTypes(sender, callMessage);
}

2
ts/session/crypto/BufferPadding.ts
Unescape Escape View File

@ -32,7 +32,7 @@ export function removeMessagePadding(paddedData: ArrayBuffer): ArrayBuffer {
* @param messageBuffer The buffer to add padding to.
*/
export function addMessagePadding(messageBuffer: Uint8Array): Uint8Array {
window?.log?.info('Adding message padding...');
// window?.log?.info('Adding message padding...');
const plaintext = new Uint8Array(getPaddedMessageLength(messageBuffer.byteLength + 1) - 1);
plaintext.set(new Uint8Array(messageBuffer));

2
ts/session/snode_api/SNodeAPI.ts
Unescape Escape View File

@ -44,7 +44,7 @@ export function getLatestTimestampOffset() {
window.log.warn('latestTimestampOffset is not set yet');
return 0;
}
window.log.info('latestTimestampOffset is ', latestTimestampOffset);
// window.log.info('latestTimestampOffset is ', latestTimestampOffset);
return latestTimestampOffset;
}

380
ts/session/utils/CallManager.ts
Unescape Escape View File

@ -19,6 +19,9 @@ import { getMessageQueue } from '../sending';
import { PubKey } from '../types';
export type InputItem = { deviceId: string; label: string };
// const VIDEO_WIDTH = 640;
// const VIDEO_RATIO = 16 / 9;
type CallManagerListener =
| ((
localStream: MediaStream | null,
@ -48,6 +51,7 @@ const callCache = new Map<string, Array<SignalService.CallMessage>>();
let peerConnection: RTCPeerConnection | null;
let remoteStream: MediaStream | null;
let mediaDevices: MediaStream | null;
const INPUT_DISABLED_DEVICE_ID = 'off';
let makingOffer = false;
let ignoreOffer = false;
@ -87,61 +91,101 @@ navigator.mediaDevices.addEventListener('devicechange', async () => {
async function updateInputLists() {
// Get the set of cameras connected
const videoCameras = await getConnectedDevices('videoinput');
camerasList = videoCameras.map(m => ({
camerasList = [{ deviceId: INPUT_DISABLED_DEVICE_ID, label: 'Off' }].concat(
videoCameras.map(m => ({
deviceId: m.deviceId,
label: m.label,
}));
}))
);
// Get the set of audio inputs connected
const audiosInput = await getConnectedDevices('audioinput');
audioInputsList = audiosInput.map(m => ({
audioInputsList = [{ deviceId: INPUT_DISABLED_DEVICE_ID, label: 'Off' }].concat(
audiosInput.map(m => ({
deviceId: m.deviceId,
label: m.label,
}));
}))
);
}
export async function selectCameraByDeviceId(cameraDeviceId: string) {
console.warn('selecting cameraDeviceId ', cameraDeviceId);
if (camerasList.some(m => m.deviceId === cameraDeviceId)) {
selectedCameraId = cameraDeviceId;
try {
mediaDevices = await openMediaDevices({
audioInputId: selectedAudioInputId,
cameraId: selectedCameraId,
if (selectedCameraId === INPUT_DISABLED_DEVICE_ID) {
const sender = peerConnection?.getSenders().find(s => {
return s.track?.kind === 'video';
});
if (sender?.track) {
sender.track.enabled = false;
}
return;
}
const devicesConfig = {
video: {
deviceId: selectedCameraId ? { exact: selectedCameraId } : undefined,
// width: VIDEO_WIDTH,
// height: Math.floor(VIDEO_WIDTH * VIDEO_RATIO),
},
};
mediaDevices.getTracks().map((track: MediaStreamTrack) => {
window.log.info('selectCameraByDeviceId adding track: ', track);
if (mediaDevices) {
peerConnection?.addTrack(track, mediaDevices);
try {
const newVideoStream = await navigator.mediaDevices.getUserMedia(devicesConfig);
const videoTrack = newVideoStream.getVideoTracks()[0];
if (!peerConnection) {
throw new Error('cannot selectCameraByDeviceId without a peer connection');
}
const sender = peerConnection.getSenders().find(s => {
return s.track?.kind === videoTrack.kind;
});
callVideoListener();
} catch (err) {
console.warn('err', err);
if (sender) {
await sender.replaceTrack(videoTrack);
videoTrack.enabled = true;
mediaDevices?.getVideoTracks().forEach(t => {
t.stop();
mediaDevices?.removeTrack(t);
});
mediaDevices?.addTrack(videoTrack);
} else {
throw new Error('Failed to get sender for selectCameraByDeviceId ');
}
} catch (e) {
window.log.warn('selectCameraByDeviceId failed with', e.message);
}
}
}
export async function selectAudioInputByDeviceId(audioInputDeviceId: string) {
console.warn('selecting audioInputDeviceId', audioInputDeviceId);
if (audioInputsList.some(m => m.deviceId === audioInputDeviceId)) {
selectedAudioInputId = audioInputDeviceId;
const devicesConfig = {
audio: {
deviceId: selectedAudioInputId ? { exact: selectedAudioInputId } : undefined,
},
};
try {
mediaDevices = await openMediaDevices({
audioInputId: selectedAudioInputId,
cameraId: selectedCameraId,
const newAudioStream = await navigator.mediaDevices.getUserMedia(devicesConfig);
const audioTrack = newAudioStream.getAudioTracks()[0];
if (!peerConnection) {
throw new Error('cannot selectAudioInputByDeviceId without a peer connection');
}
const sender = peerConnection.getSenders().find(s => {
return s.track?.kind === audioTrack.kind;
});
mediaDevices.getTracks().map((track: MediaStreamTrack) => {
window.log.info('selectAudioInputByDeviceId adding track: ', track);
if (mediaDevices) {
peerConnection?.addTrack(track, mediaDevices);
}
if (sender) {
await sender.replaceTrack(audioTrack);
mediaDevices?.getAudioTracks().forEach(t => {
t.stop();
mediaDevices?.removeTrack(t);
});
callVideoListener();
} catch (err) {
console.warn('err', err);
mediaDevices?.addTrack(audioTrack);
} else {
throw new Error('Failed to get sender for selectAudioInputByDeviceId ');
}
} catch (e) {
window.log.warn('selectAudioInputByDeviceId failed with', e.message);
}
}
}
@ -151,14 +195,13 @@ async function handleNegotiationNeededEvent(event: Event, recipient: string) {
try {
makingOffer = true;
const offer = await peerConnection?.createOffer();
if (!offer) {
throw new Error('Could not create offer in handleNegotiationNeededEvent');
}
await peerConnection?.setLocalDescription(offer);
if (offer && offer.sdp) {
const negotationOfferMessage = new CallMessage({
const offerMessage = new CallMessage({
timestamp: Date.now(),
type: SignalService.CallMessage.Type.OFFER,
sdps: [offer.sdp],
@ -167,10 +210,10 @@ async function handleNegotiationNeededEvent(event: Event, recipient: string) {
window.log.info('sending OFFER MESSAGE');
const negotationOfferSendResult = await getMessageQueue().sendToPubKeyNonDurably(
PubKey.cast(recipient),
negotationOfferMessage
offerMessage
);
if (typeof negotationOfferSendResult === 'number') {
window.log?.warn('setting last sent timestamp');
// window.log?.warn('setting last sent timestamp');
lastOutgoingOfferTimestamp = negotationOfferSendResult;
}
}
@ -188,25 +231,47 @@ function handleIceCandidates(event: RTCPeerConnectionIceEvent, pubkey: string) {
}
}
// tslint:disable-next-line: function-name
export async function USER_callRecipient(recipient: string) {
async function openMediaDevicesAndAddTracks() {
try {
await updateInputLists();
window?.log?.info(`starting call with ${ed25519Str(recipient)}..`);
window.inboxStore?.dispatch(startingCallWith({ pubkey: recipient }));
if (peerConnection) {
window.log.info('closing existing peerconnection');
peerConnection.close();
peerConnection = null;
if (!camerasList.length) {
ToastUtils.pushNoCameraFound();
return;
}
if (!audioInputsList.length) {
ToastUtils.pushNoAudioInputFound();
return;
}
peerConnection = new RTCPeerConnection(configuration);
try {
mediaDevices = await openMediaDevices({});
const firstAudio = audioInputsList[0].deviceId;
const firstVideo = camerasList[0].deviceId;
window.log.info(`openMediaDevices video:${firstVideo} audio:${firstAudio}`);
const devicesConfig = {
audio: {
deviceId: firstAudio,
mediaDevices.getTracks().map((track: any) => {
window.log.info('USER_callRecipient adding track: ', track);
echoCancellation: true,
},
video: {
deviceId: firstVideo,
// width: VIDEO_WIDTH,
// height: Math.floor(VIDEO_WIDTH * VIDEO_RATIO),
},
};
mediaDevices = await navigator.mediaDevices.getUserMedia(devicesConfig);
mediaDevices.getTracks().map(track => {
if (track.kind === 'video') {
track.enabled = false;
}
if (mediaDevices) {
// FIXME audric why does this fails?
// track.onunmute = () => {
// if (mediaDevices) {
peerConnection?.addTrack(track, mediaDevices);
// }
// };
}
});
} catch (err) {
@ -215,27 +280,19 @@ export async function USER_callRecipient(recipient: string) {
window.inboxStore?.dispatch(showSettingsSection(SessionSettingCategory.Privacy));
});
}
peerConnection.addEventListener('connectionstatechange', () => {
handleConnectionStateChanged(recipient);
});
peerConnection.addEventListener('icecandidate', event => {
handleIceCandidates(event, recipient);
});
peerConnection.onnegotiationneeded = async (event: Event) => {
await handleNegotiationNeededEvent(event, recipient);
};
remoteStream = new MediaStream();
callVideoListener();
}
peerConnection.addEventListener('track', event => {
callVideoListener();
if (remoteStream) {
remoteStream.addTrack(event.track);
// tslint:disable-next-line: function-name
export async function USER_callRecipient(recipient: string) {
await updateInputLists();
window?.log?.info(`starting call with ${ed25519Str(recipient)}..`);
window.inboxStore?.dispatch(startingCallWith({ pubkey: recipient }));
if (peerConnection) {
throw new Error('USER_callRecipient peerConnection is already initialized ');
}
});
peerConnection = createOrGetPeerConnection(recipient);
await openMediaDevicesAndAddTracks();
}
const iceCandidates: Array<RTCIceCandidate> = new Array();
@ -272,32 +329,6 @@ const iceSenderDebouncer = _.debounce(async (recipient: string) => {
await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(recipient), callIceCandicates);
}, 2000);
const openMediaDevices = async ({
audioInputId,
cameraId,
}: {
cameraId?: string;
audioInputId?: string;
}) => {
if (mediaDevices) {
window.log.info('stopping existing tracks in openMediaDevices');
mediaDevices.getTracks().forEach(track => {
track.stop();
});
}
window.log.info('openMediaDevices ', { audioInputId, cameraId });
return navigator.mediaDevices.getUserMedia({
audio: {
deviceId: audioInputId ? { exact: audioInputId } : undefined,
echoCancellation: true,
},
video: {
deviceId: cameraId ? { exact: cameraId } : undefined,
},
});
};
const findLastMessageTypeFromSender = (sender: string, msgType: SignalService.CallMessage.Type) => {
const msgCacheFromSender = callCache.get(sender);
if (!msgCacheFromSender) {
@ -328,6 +359,7 @@ function handleConnectionStateChanged(pubkey: string) {
}
function closeVideoCall() {
window.log.info('closingVideoCall ', peerConnection);
if (peerConnection) {
peerConnection.ontrack = null;
peerConnection.onicecandidate = null;
@ -345,7 +377,7 @@ function closeVideoCall() {
if (remoteStream) {
remoteStream.getTracks().forEach(track => {
track.stop();
remoteStream?.removeTrack(track);
});
}
@ -353,11 +385,46 @@ function closeVideoCall() {
peerConnection = null;
}
mediaDevices = null;
remoteStream = null;
if (videoEventsListener) {
videoEventsListener(null, null, [], []);
}
}
function createOrGetPeerConnection(withPubkey: string) {
if (peerConnection) {
return peerConnection;
}
remoteStream = new MediaStream();
peerConnection = new RTCPeerConnection(configuration);
peerConnection.onnegotiationneeded = async (event: Event) => {
await handleNegotiationNeededEvent(event, withPubkey);
};
peerConnection.onsignalingstatechange = handleSignalingStateChangeEvent;
peerConnection.ontrack = event => {
event.track.onunmute = () => {
remoteStream?.addTrack(event.track);
callVideoListener();
};
event.track.onmute = () => {
remoteStream?.removeTrack(event.track);
callVideoListener();
};
};
peerConnection.onconnectionstatechange = () => {
handleConnectionStateChanged(withPubkey);
};
peerConnection.onicecandidate = event => {
handleIceCandidates(event, withPubkey);
};
return peerConnection;
}
// tslint:disable-next-line: function-name
export async function USER_acceptIncomingCallRequest(fromSender: string) {
const msgCacheFromSender = callCache.get(fromSender);
@ -382,38 +449,12 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) {
window.inboxStore?.dispatch(answerCall({ pubkey: fromSender }));
if (peerConnection) {
window.log.info('closing existing peerconnection');
peerConnection.close();
peerConnection = null;
throw new Error('USER_acceptIncomingCallRequest: peerConnection is already set.');
}
peerConnection = new RTCPeerConnection(configuration);
mediaDevices = await openMediaDevices({});
mediaDevices.getTracks().map(track => {
// window.log.info('USER_acceptIncomingCallRequest adding track ', track);
if (mediaDevices) {
peerConnection?.addTrack(track, mediaDevices);
}
});
remoteStream = new MediaStream();
peerConnection.addEventListener('icecandidate', event => {
if (event.candidate) {
iceCandidates.push(event.candidate);
void iceSenderDebouncer(fromSender);
}
});
peerConnection = createOrGetPeerConnection(fromSender);
peerConnection.addEventListener('signalingstatechange', handleSignalingStateChangeEvent);
callVideoListener();
peerConnection.addEventListener('track', event => {
callVideoListener();
remoteStream?.addTrack(event.track);
});
peerConnection.addEventListener('connectionstatechange', () => {
handleConnectionStateChanged(fromSender);
});
await openMediaDevicesAndAddTracks();
const { sdps } = lastOfferMessage;
if (!sdps || sdps.length === 0) {
@ -430,22 +471,6 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) {
window.log?.error(`Error setting RTC Session Description ${e}`);
}
const answer = await peerConnection.createAnswer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
if (!answer?.sdp || answer.sdp.length === 0) {
window.log.warn('failed to create answer');
return;
}
await peerConnection.setLocalDescription(answer);
const answerSdp = answer.sdp;
const callAnswerMessage = new CallMessage({
timestamp: Date.now(),
type: SignalService.CallMessage.Type.ANSWER,
sdps: [answerSdp],
});
const lastCandidatesFromSender = findLastMessageTypeFromSender(
fromSender,
SignalService.CallMessage.Type.ICE_CANDIDATES
@ -461,9 +486,7 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) {
await peerConnection.addIceCandidate(candicate);
}
}
window.log.info('sending ANSWER MESSAGE');
await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), callAnswerMessage);
await buildAnswerAndSendIt(fromSender);
}
// tslint:disable-next-line: function-name
@ -478,32 +501,80 @@ export async function USER_rejectIncomingCallRequest(fromSender: string) {
window.log.info('sending END_CALL MESSAGE');
await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), endCallMessage);
const convos = getConversationController().getConversations();
const callingConvos = convos.filter(convo => convo.callState !== undefined);
if (callingConvos.length > 0) {
// we just got a new offer from someone we are already in a call with
if (callingConvos.length === 1 && callingConvos[0].id === fromSender) {
closeVideoCall();
}
}
}
export function handleEndCallMessage(sender: string) {
export function handleCallTypeEndCall(sender: string) {
callCache.delete(sender);
window.log.info('handling callMessage END_CALL');
if (videoEventsListener) {
videoEventsListener(null, null, [], []);
}
mediaDevices = null;
remoteStream = null;
closeVideoCall();
//
// FIXME audric trigger UI cleanup
window.inboxStore?.dispatch(endCall({ pubkey: sender }));
}
export async function handleOfferCallMessage(
async function buildAnswerAndSendIt(sender: string) {
if (peerConnection) {
const answer = await peerConnection.createAnswer({
offerToReceiveAudio: true,
offerToReceiveVideo: true,
});
if (!answer?.sdp || answer.sdp.length === 0) {
window.log.warn('failed to create answer');
return;
}
await peerConnection.setLocalDescription(answer);
const answerSdp = answer.sdp;
const callAnswerMessage = new CallMessage({
timestamp: Date.now(),
type: SignalService.CallMessage.Type.ANSWER,
sdps: [answerSdp],
});
window.log.info('sending ANSWER MESSAGE');
await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(sender), callAnswerMessage);
}
}
export async function handleCallTypeOffer(
sender: string,
callMessage: SignalService.CallMessage,
incomingOfferTimestamp: number
) {
try {
window.log.info('handling callMessage OFFER');
const convos = getConversationController().getConversations();
const callingConvos = convos.filter(convo => convo.callState !== undefined);
if (callingConvos.length > 0) {
// we just got a new offer from someone we are already in a call with
if (callingConvos.length === 1 && callingConvos[0].id === sender) {
window.log.info('Got a new offer message from our ongoing call');
const remoteDesc = new RTCSessionDescription({
type: 'offer',
sdp: callMessage.sdps[0],
});
if (peerConnection) {
await peerConnection.setRemoteDescription(remoteDesc);
remoteStream?.getTracks().forEach(t => {
remoteStream?.removeTrack(t);
});
await buildAnswerAndSendIt(sender);
}
} else {
await handleMissedCall(sender, incomingOfferTimestamp);
return;
@ -549,14 +620,13 @@ async function handleMissedCall(sender: string, incomingOfferTimestamp: number)
return;
}
export async function handleCallAnsweredMessage(
sender: string,
callMessage: SignalService.CallMessage
) {
export async function handleCallTypeAnswer(sender: string, callMessage: SignalService.CallMessage) {
if (!callMessage.sdps || callMessage.sdps.length === 0) {
window.log.warn('cannot handle answered message without signal description protols');
return;
}
window.log.info('handling callMessage ANSWER');
if (!callCache.has(sender)) {
callCache.set(sender, new Array());
}
@ -565,7 +635,7 @@ export async function handleCallAnsweredMessage(
window.inboxStore?.dispatch(answerCall({ pubkey: sender }));
const remoteDesc = new RTCSessionDescription({ type: 'answer', sdp: callMessage.sdps[0] });
if (peerConnection) {
window.log?.warn('Setting remote answer pending');
// window.log?.info('Setting remote answer pending');
isSettingRemoteAnswerPending = true;
await peerConnection.setRemoteDescription(remoteDesc);
isSettingRemoteAnswerPending = false;
@ -574,7 +644,7 @@ export async function handleCallAnsweredMessage(
}
}
export async function handleIceCandidatesMessage(
export async function handleCallTypeIceCandidates(
sender: string,
callMessage: SignalService.CallMessage
) {
@ -582,12 +652,17 @@ export async function handleIceCandidatesMessage(
window.log.warn('cannot handle iceCandicates message without candidates');
return;
}
window.log.info('handling callMessage ICE_CANDIDATES');
if (!callCache.has(sender)) {
callCache.set(sender, new Array());
}
callCache.get(sender)?.push(callMessage);
// window.inboxStore?.dispatch(incomingCall({ pubkey: sender }));
await addIceCandidateToExistingPeerConnection(callMessage);
}
async function addIceCandidateToExistingPeerConnection(callMessage: SignalService.CallMessage) {
if (peerConnection) {
// tslint:disable-next-line: prefer-for-of
for (let index = 0; index < callMessage.sdps.length; index++) {
@ -609,9 +684,6 @@ export async function handleIceCandidatesMessage(
}
// tslint:disable-next-line: no-async-without-await
export async function handleOtherCallMessage(
sender: string,
callMessage: SignalService.CallMessage
) {
export async function handleOtherCallTypes(sender: string, callMessage: SignalService.CallMessage) {
callCache.get(sender)?.push(callMessage);
}

Write Preview
Loading…
Cancel
Save