Add a way to intercept the audio samples before processing

sdk/android/BUILD.gn

@@ -427,6 +427,7 @@ if (is_android) {
  "src/java/org/webrtc/audio/WebRtcAudioRecord.java",
  "src/java/org/webrtc/audio/WebRtcAudioTrack.java",
  "src/java/org/webrtc/audio/WebRtcAudioUtils.java",
+ "src/java/org/webrtc/audio/AudioRecordDataCallback.java",
  ]
 
  deps = [

sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java

@@ -51,6 +51,7 @@ public static class Builder {
  private AudioAttributes audioAttributes;
  private boolean useLowLatency;
  private boolean enableVolumeLogger;
+ private AudioRecordDataCallback audioRecordDataCallback;
 
  private Builder(Context context) {
  this.context = context;
@@ -221,6 +222,16 @@ public Builder setEnableVolumeLogger(boolean enableVolumeLogger) {
  return this;
  }
 
+ /**
+ * Can be used to gain access to the raw ByteBuffer from the recording device before it's
+ * fed into WebRTC. You can use this to manipulate the ByteBuffer (e.g. audio filters).
+ * Make sure that the operation is fast.
+ */
+ public Builder setAudioRecordDataCallback(AudioRecordDataCallback audioRecordDataCallback) {
+ this.audioRecordDataCallback = audioRecordDataCallback;
+ return this;
+ }
+
  /**
  * Construct an AudioDeviceModule based on the supplied arguments. The caller takes ownership
  * and is responsible for calling release().
@@ -255,7 +266,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() {
  }
  final WebRtcAudioRecord audioInput = new WebRtcAudioRecord(context, executor, audioManager,
  audioSource, audioFormat, audioRecordErrorCallback, audioRecordStateCallback,
- samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor);
+ samplesReadyCallback, audioRecordDataCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor);
  final WebRtcAudioTrack audioOutput =
  new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback,
  audioTrackStateCallback, useLowLatency, enableVolumeLogger);

sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java

@@ -0,0 +1,16 @@
+package org.webrtc.audio;
+
+import androidx.annotation.NonNull;
+
+import java.nio.ByteBuffer;
+
+public interface AudioRecordDataCallback {
+ /**
+ * Invoked after an audio sample is recorded. Can be used to manipulate
+ * the ByteBuffer before it's fed into WebRTC. Currently the audio in the
+ * ByteBuffer is always PCM 16bit and the buffer sample size is ~10ms.
+ *
+ * @param audioFormat format in android.media.AudioFormat
+ */
+ void onAudioDataRecorded(int audioFormat, int channelCount, int sampleRate, @NonNull ByteBuffer audioBuffer);
+}

sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java

@@ -104,6 +104,7 @@ class WebRtcAudioRecord {
 
  private final @Nullable AudioRecordErrorCallback errorCallback;
  private final @Nullable AudioRecordStateCallback stateCallback;
+ private final @Nullable AudioRecordDataCallback audioRecordDataCallback;
  private final @Nullable SamplesReadyCallback audioSamplesReadyCallback;
  private final boolean isAcousticEchoCancelerSupported;
  private final boolean isNoiseSuppressorSupported;
@@ -153,6 +154,13 @@ public void run() {
  captureTimeNs = audioTimestamp.nanoTime;
  }
  }
+
+ // Allow the client to intercept the ByteBuffer (to modify it)
+ if (audioRecordDataCallback != null) {
+ audioRecordDataCallback.onAudioDataRecorded(audioRecord.getAudioFormat(),
+ audioRecord.getChannelCount(), audioRecord.getSampleRate(), byteBuffer);
+ }
+
  nativeDataIsRecorded(nativeAudioRecord, bytesRead, captureTimeNs);
  }
  if (audioSamplesReadyCallback != null) {
@@ -196,7 +204,8 @@ public void stopThread() {
  WebRtcAudioRecord(Context context, AudioManager audioManager) {
  this(context, newDefaultScheduler() /* scheduler */, audioManager, DEFAULT_AUDIO_SOURCE,
  DEFAULT_AUDIO_FORMAT, null /* errorCallback */, null /* stateCallback */,
- null /* audioSamplesReadyCallback */, WebRtcAudioEffects.isAcousticEchoCancelerSupported(),
+ null /* audioSamplesReadyCallback */, null /* audioRecordCallback */,
+ WebRtcAudioEffects.isAcousticEchoCancelerSupported(),
  WebRtcAudioEffects.isNoiseSuppressorSupported());
  }
 
@@ -205,6 +214,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler,
  @Nullable AudioRecordErrorCallback errorCallback,
  @Nullable AudioRecordStateCallback stateCallback,
  @Nullable SamplesReadyCallback audioSamplesReadyCallback,
+ @Nullable AudioRecordDataCallback audioRecordDataCallback,
  boolean isAcousticEchoCancelerSupported, boolean isNoiseSuppressorSupported) {
  if (isAcousticEchoCancelerSupported && !WebRtcAudioEffects.isAcousticEchoCancelerSupported()) {
  throw new IllegalArgumentException("HW AEC not supported");
@@ -220,6 +230,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler,
  this.errorCallback = errorCallback;
  this.stateCallback = stateCallback;
  this.audioSamplesReadyCallback = audioSamplesReadyCallback;
+ this.audioRecordDataCallback = audioRecordDataCallback;
  this.isAcousticEchoCancelerSupported = isAcousticEchoCancelerSupported;
  this.isNoiseSuppressorSupported = isNoiseSuppressorSupported;
  Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo());