Twilio-video.js: Mute / Disable audio or video doesn't work for multiple participants

[bonnopc]

Hello, I have been following the Live Demo to integrate the Twilio's Programmable Video JavaScript SDK. I have integrating the SDK for two participants mainly. Main features of this SDK works fine, but when a second participant enters into the Room the first participant can't actually mute/disable his audio/video.

• [x] I have verified that the issue occurs with the latest twilio-video.js release and is not marked as a known issue in the CHANGELOG.md.
• [x] I reviewed the Common Issues and open GitHub issues and verified that this report represents a potentially new issue.
• [x] I verified that the Quickstart application works in my environment.
• [x] I am not sharing any Personally Identifiable Information (PII)
  or sensitive account information (API keys, credentials, etc.) when reporting this issue.

Code to reproduce the issue:

export default function ControlMediaButton ({ room, mediaType, participants }) {
    const [ isEnabled, setIsEnabled ] = useState(true);

    useEffect(() => {
        updateTracks('ENABLE')
    }, [participants])

    const updateTracks = nextState => {
        const publications = mediaType === 'audio' ? room.localParticipant.audioTracks : room.localParticipant.videoTracks;

        publications.forEach(pub => {
            console.log("updateTracks", nextState)
            if(nextState === "DISABLE"){
                pub.track.disable()
            } else{
                pub.track.enable()
            }
        });
    }

    const handleMedia = useCallback((nextState) => {
        updateTracks(nextState)
        setIsEnabled(val => !val)
    }, [room])

    return (
        <Tooltip placement='top' title={`${mediaType === 'audio' ? `${isEnabled ? 'Mute' : 'Unmute'} Audio` : `${isEnabled ? 'Disable' : 'Enable'} Video`}`}>
            <Fab
                size="small"
                onClick={() => handleMedia(isEnabled ? 'DISABLE' : 'ENABLE')}
                className={`${isEnabled ? '' : 'active'}`}
                name="hello"
            >
                {
                    mediaType === 'audio' ?
                    <MicOffIcon fontSize="small" /> :
                    <VideocamOffIcon fontSize="small" />
                }
            </Fab>
        </Tooltip>
    )
}

Component For Participant

import React, { useState, useEffect, useRef } from 'react';
// Other imports

const useStyles = makeStyles(theme => ({
    // JSS codes
}))

const Participant = ({ 
    participant, rightActions, endBtn, leftActions, 
    secondaryParticipant, handleFullScreen, name, avatar, localParticipant
}) => {
    const [videoTracks, setVideoTracks] = useState([]);
    const [audioTracks, setAudioTracks] = useState([]);

    const videoRef = useRef();
    const audioRef = useRef();

    const classes = useStyles();

    if(!participant) return null;

    const FullScreenButtonRef = useRef(null)

    const trackpubsToTracks = (trackMap) => trackMap ? Array.from(trackMap.values()).map((publication) => publication.track).filter((track) => track !== null) : [];

    useEffect(() => {
        setVideoTracks(trackpubsToTracks(participant?.videoTracks));
        setAudioTracks(trackpubsToTracks(participant?.audioTracks));

        const trackSubscribed = (track) => {
            if (track.kind === "video") {
                setVideoTracks((videoTracks) => [...videoTracks, track]);
            } else if (track.kind === "audio") {
                setAudioTracks((audioTracks) => [...audioTracks, track]);
            }
        };

        const trackUnsubscribed = (track) => {
            if (track.kind === "video") {
                setVideoTracks((videoTracks) => videoTracks.filter((v) => v !== track));
            } else if (track.kind === "audio") {
                setAudioTracks((audioTracks) => audioTracks.filter((a) => a !== track));
            }
        };

        participant.on("trackSubscribed", trackSubscribed);
        participant.on("trackUnsubscribed", trackUnsubscribed);

        return () => {
            setVideoTracks([]);
            setAudioTracks([]);
            participant.removeAllListeners();
        };
    }, [participant]);

    useEffect(() => {
        const videoTrack = videoTracks[0];

        if (videoTrack) {
            videoTrack.attach(videoRef.current);

            return () => {
                videoTrack.detach();
            };
        }
    }, [videoTracks]);

    useEffect(() => {
        const audioTrack = audioTracks[0];

        if (audioTrack) {
            audioTrack.attach(audioRef.current);

            return () => {
                audioTrack.detach();
            };
        }
    }, [audioTracks]);

    return (
         // returns component
    )
};

export default Participant;

Component for Room (acts as Parent Component)

import React, { Component, Fragment } from "react"
import { makeStyles } from "@material-ui/core/styles"
// other imports

const useStyles = makeStyles(theme => ({
    // JSS codes
}));

function StyledVideo({ children }){
    const classes = useStyles();
    return children(classes)
}

export default class VideoCallRoom extends Component {
    state = {
        localTracks: null
    }

    componentDidMount(){
        this.props.actionToggleConnectingState(true)
        if(this.props.videoCall?.isRunning && this.props.item?.uid) this.getToken()
    }

    componentWillUnmount(){
        if(this.props.videoCall.twilioVideoRoom){
            this.props.actionDisconnectTwilioCall(this.props.videoCall.twilioVideoRoom,this.state.localTracks)
        }
    }

    async getToken(){
        const roomName = await this.props.actionCreateTwilioRoom(this.props.item.uid)
        const token = await this.props.actionGetTwilioVideoToken(this.props.item.uid)

        if(token && roomName){
            this.connectUser(token,roomName)
        } else {
            this.props.actionToggleConnectingState(false)
        }
    }

    async reconnectCall(){
        await this.props.actionDisconnectTwilioCall(this.props.videoCall.twilioVideoRoom,this.state.localTracks);

        await this.connectUser(this.props.videoCall.twilioVideoToken,this.props.videoCall.twilioRoomName);
    }

    async connectUser(token,roomName){
        const status = await this.props.actionConnectUser({
            token,
            roomName,
            participantConnected: this.handleParticipantConnected.bind(this),
            participantDisconnected: this.handleParticipantDisconnected.bind(this),
            disconnect: this.endCall.bind(this)
        })

        if(status && status.localTracks) this.setState({ localTracks: status.localTracks })
    }

    handleParticipantConnected(participant){
        console.log("participant Connected ==> ", participant)
        this.props.actionAddOrRemoveParticipants(participant,this.props.videoCall.participants,"ADD")
    }

    handleParticipantDisconnected(participant){
        console.log("participant Disconnected ==>", participant)
        this.props.actionAddOrRemoveParticipants(participant,this.props.videoCall.participants,"REMOVE")
    }

    endCall(){
        if(this.props.videoCall.twilioVideoRoom){
            this.props.actionDisconnectTwilioCall(this.props.videoCall.twilioVideoRoom,this.state.localTracks)
        }

        this.props.actionToggleVideoCall(false)

        // remove all particiapnts
        if(this.props.videoCall.participants && this.props.videoCall.participants.length){
            this.props.actionAddOrRemoveParticipants()
        }
    }

    async handleEndButtonClick(){
        this.endCall()
        await this.props.actionCompleteTwilioCall(this.props.item?.uid)
    }

    render(){
        const { videoCall, profile } = this.props

        return (
            <StyledVideo>
                {classes => (
                    <div className={classes.videoContainer}>
                        {
                            videoCall.isConnecting ?
                            <div className={classes.loadingContainer}>
                                Connecting...
                            </div> :
                            <FullScreenVideoCall>
                                { handleFullScreen => (
                                    <Fragment>
                                        {
                                            videoCall.twilioVideoRoom ?
                                            <VideoParticipant 
                                                key={ videoCall.participants?.length ? videoCall.participants[0]?.sid : videoCall.twilioVideoRoom.localParticipant.sid } 
                                                participant={ videoCall.participants?.length ? videoCall.participants[0] : videoCall.twilioVideoRoom.localParticipant } 
                                                endBtn={(
                                                    <Fab
                                                        // color="secondary"
                                                        size="medium"
                                                        onClick={this.handleEndButtonClick.bind(this)}
                                                    >
                                                        <CloseIcon/>
                                                    </Fab>
                                                )}
                                                rightActions={(
                                                    <Fragment>

                                                        <ControlMediaButton mediaType="audio" room={videoCall.twilioVideoRoom} participants={videoCall.participants} />
                                                    </Fragment>
                                                )}

                                                secondaryParticipant={videoCall.participants?.length ? (
                                                    <VideoParticipant 
                                                        key={videoCall.twilioVideoRoom.localParticipant.sid} 
                                                        participant={videoCall.twilioVideoRoom.localParticipant}
                                                        name={profile?.doctorProfile?.meta?.title?.en ? `${profile.doctorProfile.meta.title.en} ${profile?.doctorProfile?.meta?.firstName?.en} ${profile?.doctorProfile?.meta?.lastName?.en}` : null}
                                                        avatar={profile?.doctorProfile?.meta?.avatar?.url ? profile.doctorProfile.meta.avatar.url : null}
                                                    />
                                                ) : null}
                                                localParticipant={videoCall.twilioVideoRoom.localParticipant}
                                                handleFullScreen={handleFullScreen}
                                                name={videoCall.participants?.length ? null : profile?.doctorProfile?.meta?.title?.en ? `${profile.doctorProfile.meta.title.en} ${profile?.doctorProfile?.meta?.firstName?.en} ${profile?.doctorProfile?.meta?.lastName?.en}` : null}
                                                avatar={videoCall.participants?.length ? null : profile?.doctorProfile?.meta?.avatar?.url ? profile.doctorProfile.meta.avatar.url : null}
                                            /> : ""
                                        }
                                    </Fragment>
                                )}
                            </FullScreenVideoCall>
                        }
                    </div>
                )}
            </StyledVideo>
        )
    }
}

Expected behavior:
This should always mute/unmute audio or disable/enable video.

Actual behavior:
This is not working when another participant joins (or for first participant) but works fine for single participant (or for second participant).

Software versions:

• [x] Browser(s): Version 84.0.4147.125 (Official Build) (64-bit)
• [x] Operating System: Ubuntu 20.04, Android 9
• [x] twilio-video.js: v2.7.1
• [x] Third-party libraries: React.js, Redux.js

[makarandp0]

Hello @bonnopc, Thank you for writing about this issue. Do you see similar issue in the our react demo app ? Since your app is based on the react demo app, I would start with looking at the differences between the mute / unmute handling with your app and the demo app.

Can you share room-sid participant-sid and logs from the browser console (from the 1st participant) when this issue happens, we can perhaps find something from those.

Thanks,
Makarand

[romshiri]

I have the exact same problem. I checked the demo app but I did nothing different.

[polygonwood]

I also encounter the same issue, just by using the example code of the documentation :

room.localParticipant.audioTracks.forEach(publication => {
publication.track.disable();
});

inspection of the values show that status is indeed changed to 'enabled=false' down to mediaStream level, but still sound is sent.

I have not attached the localParticipant audio to any htmlElement (it's actually a LocalAudioTrack class element created from a MediaStream track.

Example data : Room RMf89ec3e5f959b17e53f8695524b46ad3

you can see status 'enabled' false why audio was playing

Room {localParticipant: LocalParticipant, name: "GXuWuyqCCBfQdbtBH", participants: Map(1), …}
dominantSpeaker: (...)
isRecording: (...)
localParticipant: LocalParticipant
audioTracks: Map(1)
[[Entries]]
0: {"MTeb3730f4ba3f4fc9975df631e2abdacb" => LocalAudioTrackPublication}
key: "MTeb3730f4ba3f4fc9975df631e2abdacb"
value: LocalAudioTrackPublication
isTrackEnabled: false
kind: "audio"
priority: (...)
track: LocalAudioTrack
id: "2934c9be-4895-461f-9cec-2430427bd162"
isEnabled: (...)
isStarted: (...)
isStopped: (...)
kind: "audio"
mediaStreamTrack: MediaStreamTrack
contentHint: ""
enabled: false
id: "2934c9be-4895-461f-9cec-2430427bd162"
kind: "audio"
label: "MediaStreamAudioDestinationNode"
muted: false
onended: null
onmute: null
onunmute: null
readyState: "live"
__proto__: MediaStreamTrack
name: "2934c9be-4895-461f-9cec-2430427bd162"
_MediaStream: ƒ MediaStream()
arguments: null
caller: null
length: 0

[kishorevarma]

Yeah I am also facing same issue, as you mentioned @polygonwood
in MediaStreamTrack muted value always false, even though track is disabled.

[polygonwood]

Disabling has only 'local' effect I learned, the remote side is unaffected (not what one expects from reading the explanations on disable, but not applicable to webrtc connections ...). Hence you have to use the twilio eventing (or your own) to eventually silence the remote side ...

[kishorevarma]

you mean even if we disable local audio track, it will stream the local audio to remote participants.
remote particiapants code should listen to event and see which participant has muted their audio, and silent(mute) that participant their side. did I understand correct ? does this worked for you ?

[polygonwood]

correct

On Wed, Feb 3, 2021 at 9:20 PM Kishore varma [e-mail]
wrote:

you mean even if we disable local audio track, it will stream the local
audio to remote participants.
remote particiapants code should listen to event and see which participant
has muted their audio, and silent(mute) that participant their side. did I
understand correct ? does this worked for you ?

—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
https://github.com/twilio/twilio-video.js/issues/1143#issuecomment-772796017,
or unsubscribe
https://github.com/notifications/unsubscribe-auth/AMTJC2ZWJS4SCM2CMD5KIVLS5GV2NANCNFSM4QBZPSSA
.

[makarandp0]

Hello @kishorevarma, sorry for the late response. Calling disable on LocalAudioTrack mutes it for the remote participants. Remote Participants does not need to silent it on their side. They should however still listen to disabled event in order to update the UI to show the track as muted.

Please check out this quick start app that demonstrates such controls.

Thanks,
Makarand

[makarandp0]

I am closing this issue now, If you have further questions please feel free to open another issue