Dan Burdetsky
Posted on July 23, 2023
I am currently experiencing an issue with my Android STB device when I attempt to play a WebRTC stream with stereo audio (two audio channels). The main issue is that the device always downmixes the Left/Right channels to mono, regardless of the different tests and actions I've taken.
The WebRTC stream is received from Red5 Stream Manager servers, and I'm using the org.webrtc:google-webrtc:1.0.32006 library.
Here's a summary of the actions I've undertaken and the corresponding findings:
I tested the source audio on both the PC Chrome browser and Android Chrome browser on the STB. The audio played correctly in stereo on both, confirming that the source audio is functioning properly.
To confirm that the audio stream arrives in stereo, I implemented additional logging. The analysis confirmed that the audio stream does indeed arrive with two channels:
Stats ID: RTCCodec_audio_Outbound_111
Stats Type: codec
payloadType: 111
mimeType: audio/opus
clockRate: 48000
channels: 2
sdpFmtpLine:
maxaveragebitrate=128000;maxplaybackrate=48000;minptime=10;sprop-stereo=1;stereo=1;useinbandfec=1
--------------------------------------------------
I tried various audio-related implementations, but the audio output continued to play in mono, so decided to stick with javaAudioDeviceModule
I also attempted SDP munging manipulation and performed component updates, but neither action resolved the issue.
I made adjustments to audio parameters within the app, but the desired stereo output remained elusive:
JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(context) .setSamplesReadyCallback(null) .setUseHardwareAcousticEchoCanceler(false) .setUseHardwareNoiseSuppressor(false) .setAudioRecordErrorCallback(null) .setAudioTrackErrorCallback(null) .setUseStereoInput(true) .setUseStereoOutput(true) .createAudioDeviceModule();
- The issue persists across various devices, including the Amino Amigo, Amino H-200, and an XIAOMI Android cell phone.
Below is the relevant code class that I'm working with:
`package com.twizted.videoflowplayer.webrtc;
import android.content.Context;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import org.webrtc.AudioSource;
import org.webrtc.AudioTrack;
import org.webrtc.CandidatePairChangeEvent;
import org.webrtc.DataChannel;
import org.webrtc.DefaultVideoDecoderFactory;
import org.webrtc.DefaultVideoEncoderFactory;
import org.webrtc.EglBase;
import org.webrtc.IceCandidate;
import org.webrtc.MediaConstraints;
import org.webrtc.MediaStream;
import org.webrtc.PeerConnection;
import org.webrtc.PeerConnectionFactory;
import org.webrtc.RTCStats;
import org.webrtc.RTCStatsCollectorCallback;
import org.webrtc.RTCStatsReport;
import org.webrtc.RendererCommon;
import org.webrtc.RtpReceiver;
import org.webrtc.RtpTransceiver;
import org.webrtc.SessionDescription;
import org.webrtc.SurfaceViewRenderer;
import org.webrtc.VideoTrack;
import org.webrtc.audio.JavaAudioDeviceModule;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
public class WebRTCNativeClient implements PeerConnection.Observer, Red5MediaSignallingEvents {
private static final String TAG = "WebRTCNativeClient";
private Handler handler = new Handler(Looper.getMainLooper());
private Context context;
private final IWebRTCListener webRTCListener;
private String stunServerUri = "stun:stun.l.google.com:19302";
private List<PeerConnection.IceServer> peerIceServers = new ArrayList<>();
private PeerConnection peerConnection;
private PeerConnectionFactory peerConnectionFactory;
private EglBase rootEglBase;
private SurfaceViewRenderer renderer;
private MediaConstraints sdpMediaConstraints;
private boolean iceConnected;
private WebSocketHandler wsHandler;
private Timer statsTimer;
private String streamId;
private boolean debug;
private String url;
private AudioTrack localAudioTrack;
public WebRTCNativeClient(IWebRTCListener webRTCListener, Context context) {
this.webRTCListener = webRTCListener;
this.context = context;
}
public void setRenderer(SurfaceViewRenderer renderer) {
this.renderer = renderer;
}
public void init(String url, String streamId, boolean debug) {
if (peerConnection != null) {
Log.w(TAG, "There is already an active peerconnection client ");
return;
}
if (url == null) {
Log.e(TAG, "Didn't get any URL!");
return;
}
this.url = url;
this.streamId = streamId;
this.debug = debug;
iceConnected = false;
sdpMediaConstraints = new MediaConstraints();
sdpMediaConstraints.mandatory.add(
new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"));
sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair(
"OfferToReceiveVideo", "true"));
PeerConnection.IceServer peerIceServer = PeerConnection.IceServer.builder(stunServerUri).createIceServer();
peerIceServers.add(peerIceServer);
rootEglBase = EglBase.create();
renderer.init(rootEglBase.getEglBaseContext(), null);
renderer.setZOrderMediaOverlay(true);
renderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT);
// Initialize PeerConnectionFactory globals.
PeerConnectionFactory.InitializationOptions initializationOptions =
PeerConnectionFactory.InitializationOptions.builder(context)
.createInitializationOptions();
PeerConnectionFactory.initialize(initializationOptions);
// Create a new PeerConnectionFactory instance - using Hardware encoder and decoder.
PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
DefaultVideoEncoderFactory defaultVideoEncoderFactory = new DefaultVideoEncoderFactory(
rootEglBase.getEglBaseContext(), true, true);
DefaultVideoDecoderFactory defaultVideoDecoderFactory = new DefaultVideoDecoderFactory(rootEglBase.getEglBaseContext());
JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(context)
.setSamplesReadyCallback(null)
.setUseHardwareAcousticEchoCanceler(false)
.setUseHardwareNoiseSuppressor(false)
.setAudioRecordErrorCallback(null)
.setAudioTrackErrorCallback(null)
.setUseStereoInput(true)
.setUseStereoOutput(true)
.createAudioDeviceModule();
peerConnectionFactory = PeerConnectionFactory.builder()
.setOptions(options)
.setAudioDeviceModule(javaAudioDeviceModule)
.setVideoEncoderFactory(defaultVideoEncoderFactory)
.setVideoDecoderFactory(defaultVideoDecoderFactory)
.createPeerConnectionFactory();
createPeerConnection();
}
private void createPeerConnection() {
PeerConnection.RTCConfiguration rtcConfig =
new PeerConnection.RTCConfiguration(peerIceServers);
rtcConfig.iceTransportsType = PeerConnection.IceTransportsType.ALL;
rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED;
rtcConfig.bundlePolicy = PeerConnection.BundlePolicy.MAXBUNDLE;
rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.REQUIRE;
rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
rtcConfig.keyType = PeerConnection.KeyType.ECDSA;
peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this);
// Set up audio track
// final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
// localAudioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
//
// if (peerConnection != null) {
// peerConnection.addTrack(localAudioTrack);
// }
}
public void startStream(String url, String streamId, boolean debug) {
init(url, streamId, debug);
if (wsHandler == null) {
wsHandler = new WebSocketHandler(this, handler, this.streamId);
wsHandler.connect(url);
} else if (!wsHandler.isConnected()) {
wsHandler.disconnect(true);
wsHandler = new WebSocketHandler(this, handler, this.streamId);
wsHandler.connect(url);
}
wsHandler.startPlay();
}
public void stopStream() {
disconnect();
}
public void disconnect() {
release();
}
private void release() {
iceConnected = false;
cancelTimer();
if (wsHandler != null && wsHandler.getSignallingListener().equals(this)) {
wsHandler.disconnect(true);
wsHandler = null;
}
if (renderer != null) {
renderer.release();
}
if (peerConnection != null) {
peerConnection.close();
peerConnection = null;
}
}
private void cancelTimer() {
if (statsTimer != null) {
statsTimer.cancel();
statsTimer = null;
}
}
public boolean isStreaming() {
return iceConnected;
}
private void gotRemoteStream(MediaStream stream) {
final VideoTrack videoTrack = stream.videoTracks.get(0);
handler.post(() -> {
try {
videoTrack.addSink(renderer);
} catch (Exception e) {
e.printStackTrace();
}
});
}
@Override
public void onSignalingChange(PeerConnection.SignalingState signalingState) {
Log.d(TAG, "onSignalingChange() called with: signalingState = [" + signalingState + "]");
}
@Override
public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
Log.d(TAG, "onIceConnectionChange() called with: iceConnectionState = [" + iceConnectionState + "]");
handler.post(() -> {
if (iceConnectionState == PeerConnection.IceConnectionState.CONNECTED) {
iceConnected = true;
enableStatsEvents(debug, 1000);
if (webRTCListener != null) {
webRTCListener.onIceConnected();
}
} else if (iceConnectionState == PeerConnection.IceConnectionState.DISCONNECTED ||
iceConnectionState == PeerConnection.IceConnectionState.CLOSED) {
iceConnected = false;
disconnect();
if (webRTCListener != null) {
webRTCListener.onIceDisconnected();
}
} else if (iceConnectionState == PeerConnection.IceConnectionState.FAILED) {
iceConnected = false;
disconnect();
if (webRTCListener != null) {
webRTCListener.onError("ICE connection failed.");
}
}
});
}
public void enableStatsEvents(boolean enable, int periodMs) {
Log.d(TAG, "enableStatsEvents() called with: enabled = [" + enable + "] --- [" + handler + "] --- [" + peerConnection + "] --- [" + webRTCListener + "]");
if (enable) {
try {
if (statsTimer == null) statsTimer = new Timer();
statsTimer.schedule(new TimerTask() {
@Override
public void run() {
handler.post(() -> {
if (peerConnection == null) return;
peerConnection.getStats(new RTCStatsCollectorCallback() {
@Override
public void onStatsDelivered(RTCStatsReport rtcStatsReport) {
if (webRTCListener != null)
webRTCListener.onReport(rtcStatsReport);
printAudioStats(rtcStatsReport);
}
});
});
}
}, 0, periodMs);
} catch (Exception e) {
Log.e(TAG, "Can not schedule statistics timer", e);
}
} else {
cancelTimer();
}
}
private void printAudioStats(RTCStatsReport rtcStatsReport) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("Audio Statistics:\n");
for (RTCStats stats : rtcStatsReport.getStatsMap().values()) {
stringBuilder.append("Stats ID: ").append(stats.getId()).append("\n");
stringBuilder.append("Stats Type: ").append(stats.getType()).append("\n");
Map<String, Object> members = stats.getMembers();
for (String statKey : members.keySet()) {
stringBuilder.append(statKey).append(": ").append(members.get(statKey)).append("\n");
}
stringBuilder.append("--------------------------------------------------\n");
}
Log.d(TAG, stringBuilder.toString());
}
@Override
public void onStandardizedIceConnectionChange(PeerConnection.IceConnectionState newState) {
Log.d(TAG, "onStandardizedIceConnectionChange() called with: newState = [" + newState + "]");
}
@Override
public void onConnectionChange(PeerConnection.PeerConnectionState newState) {
Log.d(TAG, "onConnectionChange() called with: newState = [" + newState + "]");
}
@Override
public void onIceConnectionReceivingChange(boolean b) {
Log.d(TAG, "onIceConnectionReceivingChange() called with: b = [" + b + "]");
}
@Override
public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
Log.d(TAG, "onIceGatheringChange() called with: iceGatheringState = [" + iceGatheringState + "]");
}
@Override
public void onIceCandidate(IceCandidate iceCandidate) {
Log.d(TAG, "onIceCandidate() called with: iceCandidate = [" + iceCandidate + "]");
handler.post(() -> {
if (wsHandler != null) wsHandler.sendLocalIceCandidate(iceCandidate);
});
}
@Override
public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
Log.d(TAG, "onIceCandidatesRemoved() called with: iceCandidates = [" + iceCandidates + "]");
}
@Override
public void onSelectedCandidatePairChanged(CandidatePairChangeEvent event) {
Log.d(TAG, "onSelectedCandidatePairChanged() called with: event = [" + event + "]");
}
@Override
public void onAddStream(MediaStream mediaStream) {
Log.d(TAG, "onAddStream() called with: mediaStream = [" + mediaStream + "]");
gotRemoteStream(mediaStream);
}
@Override
public void onRemoveStream(MediaStream mediaStream) {
Log.d(TAG, "onRemoveStream() called with: mediaStream = [" + mediaStream + "]");
}
@Override
public void onDataChannel(DataChannel dataChannel) {
Log.d(TAG, "onDataChannel() called with: dataChannel = [" + dataChannel + "]");
}
@Override
public void onRenegotiationNeeded() {
Log.d(TAG, "onRenegotiationNeeded() called");
}
@Override
public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
Log.d(TAG, "onAddTrack() called with: rtpReceiver = [" + rtpReceiver + "] -- mediaStreams = [" + mediaStreams + "]");
}
@Override
public void onTrack(RtpTransceiver transceiver) {
Log.d(TAG, "onTrack() called with: transceiver = [" + transceiver + "]");
}
@Override
public void onRemoteIceCandidate(String streamId, IceCandidate candidate) {
handler.post(() -> {
if (peerConnection == null) {
Log.e(TAG, "Received ICE candidate for a non-initialized peer connection.");
return;
}
peerConnection.addIceCandidate(candidate);
});
}
@Override
public void onTakeConfiguration(String streamId, SessionDescription sdp) {
handler.post(() -> {
if (sdp.type == SessionDescription.Type.OFFER) {
peerConnection.setRemoteDescription(new CustomSdpObserver("remoteDesc"), sdp);
peerConnection.createAnswer(new CustomSdpObserver("createAnswer") {
@Override
public void onCreateSuccess(SessionDescription sessionDescription) {
super.onCreateSuccess(sessionDescription);
peerConnection.setLocalDescription(new CustomSdpObserver("setLocalDescription"), sessionDescription);
handler.post(() -> wsHandler.sendConfiguration(sessionDescription, WebSocketConstants.ANSWER));
}
}, sdpMediaConstraints);
}
});
}
@Override
public void onPlayStarted(String streamId) {
handler.post(() -> {
if (webRTCListener != null) {
webRTCListener.onPlayStarted();
}
});
}
@Override
public void onPlayFinished(String streamId) {
handler.post(() -> {
if (webRTCListener != null) {
webRTCListener.onPlayFinished();
}
disconnect();
});
}
@Override
public void noStreamExistsToPlay(String streamId) {
handler.post(() -> {
if (webRTCListener != null) {
webRTCListener.noStreamExistsToPlay();
}
});
}
@Override
public void onStreamLeaved(String streamId) {
}
@Override
public void onBitrateMeasurement(String streamId, int targetBitrate, int videoBitrate, int audioBitrate) {
}
}
`
Can anyone help me understand why the device continues to downmix the stereo audio to mono, and how I might fix this issue? Any guidance or suggestions would be greatly appreciated. Thanks in advance!
💖 💪 🙅 🚩
Dan Burdetsky
Posted on July 23, 2023
Join Our Newsletter. No Spam, Only the good stuff.
Sign up to receive the latest update from our blog.