Merge pull request #5989 from matrix-org/travis/voicemessages/safari

Support voice messages on Safari

Author: turt2live

__mocks__/empty.js

@@ -0,0 +1,2 @@
+// Yes, this is empty.
+module.exports = {};

package.json

@@ -186,7 +186,10 @@
  ],
  "moduleNameMapper": {
  "\\.(gif|png|svg|ttf|woff2)$": "<rootDir>/__mocks__/imageMock.js",
- "\\$webapp/i18n/languages.json": "<rootDir>/__mocks__/languages.json"
+ "\\$webapp/i18n/languages.json": "<rootDir>/__mocks__/languages.json",
+ "decoderWorker\\.min\\.js": "<rootDir>/__mocks__/empty.js",
+ "decoderWorker\\.min\\.wasm": "<rootDir>/__mocks__/empty.js",
+ "waveWorker\\.min\\.js": "<rootDir>/__mocks__/empty.js"
  },
  "transformIgnorePatterns": [
  "/node_modules/(?!matrix-js-sdk).+$"

src/@types/global.d.ts

@@ -52,6 +52,9 @@ declare global {
  init: () => Promise<void>;
  };
 
+ // Needed for Safari, unknown to TypeScript
+ webkitAudioContext: typeof AudioContext;
+
  mxContentMessages: ContentMessages;
  mxToastStore: ToastStore;
  mxDeviceListener: DeviceListener;

src/rageshake/rageshake.js

@@ -73,7 +73,9 @@ class ConsoleLogger {
 
  // Convert objects and errors to helpful things
  args = args.map((arg) => {
- if (arg instanceof Error) {
+ if (arg instanceof DOMException) {
+ return arg.message + ` (${arg.name} | ${arg.code}) ` + (arg.stack ? `\n${arg.stack}` : '');
+ } else if (arg instanceof Error) {
  return arg.message + (arg.stack ? `\n${arg.stack}` : '');
  } else if (typeof (arg) === 'object') {
  try {

src/voice/Playback.ts

@@ -21,6 +21,7 @@ import {SimpleObservable} from "matrix-widget-api";
 import {IDestroyable} from "../utils/IDestroyable";
 import {PlaybackClock} from "./PlaybackClock";
 import {clamp} from "../utils/numbers";
+import {createAudioContext, decodeOgg} from "./compat";
 
 export enum PlaybackState {
  Decoding = "decoding",
@@ -49,7 +50,7 @@ export class Playback extends EventEmitter implements IDestroyable {
  */
  constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
  super();
- this.context = new AudioContext();
+ this.context = createAudioContext();
  this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES);
  this.waveformObservable.update(this.resampledWaveform);
  this.clock = new PlaybackClock(this.context);
@@ -91,7 +92,23 @@ export class Playback extends EventEmitter implements IDestroyable {
  }
 
  public async prepare() {
- this.audioBuf = await this.context.decodeAudioData(this.buf);
+ // Safari compat: promise API not supported on this function
+ this.audioBuf = await new Promise((resolve, reject) => {
+ this.context.decodeAudioData(this.buf, b => resolve(b), async e => {
+ // This error handler is largely for Safari as well, which doesn't support Opus/Ogg
+ // very well.
+ console.error("Error decoding recording: ", e);
+ console.warn("Trying to re-encode to WAV instead...");
+
+ const wav = await decodeOgg(this.buf);
+
+ // noinspection ES6MissingAwait - not needed when using callbacks
+ this.context.decodeAudioData(wav, b => resolve(b), e => {
+ console.error("Still failed to decode recording: ", e);
+ reject(e);
+ });
+ });
+ });
 
  // Update the waveform to the real waveform once we have channel data to use. We don't
  // exactly trust the user-provided waveform to be accurate...

src/voice/VoiceRecording.ts

@@ -19,16 +19,17 @@ import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
 import {MatrixClient} from "matrix-js-sdk/src/client";
 import CallMediaHandler from "../CallMediaHandler";
 import {SimpleObservable} from "matrix-widget-api";
-import {clamp} from "../utils/numbers";
+import {clamp, percentageOf, percentageWithin} from "../utils/numbers";
 import EventEmitter from "events";
 import {IDestroyable} from "../utils/IDestroyable";
 import {Singleflight} from "../utils/Singleflight";
 import {PayloadEvent, WORKLET_NAME} from "./consts";
 import {UPDATE_EVENT} from "../stores/AsyncStore";
 import {Playback} from "./Playback";
+import {createAudioContext} from "./compat";
 
 const CHANNELS = 1; // stereo isn't important
-const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
+export const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
 const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus.
 const TARGET_MAX_LENGTH = 120; // 2 minutes in seconds. Somewhat arbitrary, though longer == larger files.
 const TARGET_WARN_TIME_LEFT = 10; // 10 seconds, also somewhat arbitrary.
@@ -55,6 +56,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
  private recorderStream: MediaStream;
  private recorderFFT: AnalyserNode;
  private recorderWorklet: AudioWorkletNode;
+ private recorderProcessor: ScriptProcessorNode;
  private buffer = new Uint8Array(0); // use this.audioBuffer to access
  private mxc: string;
  private recording = false;
@@ -90,78 +92,107 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
  }
 
  private async makeRecorder() {
- this.recorderStream = await navigator.mediaDevices.getUserMedia({
- audio: {
- channelCount: CHANNELS,
- noiseSuppression: true, // browsers ignore constraints they can't honour
- deviceId: CallMediaHandler.getAudioInput(),
- },
- });
- this.recorderContext = new AudioContext({
- // latencyHint: "interactive", // we don't want a latency hint (this causes data smoothing)
- });
- this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream);
- this.recorderFFT = this.recorderContext.createAnalyser();
-
- // Bring the FFT time domain down a bit. The default is 2048, and this must be a power
- // of two. We use 64 points because we happen to know down the line we need less than
- // that, but 32 would be too few. Large numbers are not helpful here and do not add
- // precision: they introduce higher precision outputs of the FFT (frequency data), but
- // it makes the time domain less than helpful.
- this.recorderFFT.fftSize = 64;
-
- // Set up our worklet. We use this for timing information and waveform analysis: the
- // web audio API prefers this be done async to avoid holding the main thread with math.
- const mxRecorderWorkletPath = document.body.dataset.vectorRecorderWorkletScript;
- if (!mxRecorderWorkletPath) {
- throw new Error("Unable to create recorder: no worklet script registered");
- }
- await this.recorderContext.audioWorklet.addModule(mxRecorderWorkletPath);
- this.recorderWorklet = new AudioWorkletNode(this.recorderContext, WORKLET_NAME);
-
- // Connect our inputs and outputs
- this.recorderSource.connect(this.recorderFFT);
- this.recorderSource.connect(this.recorderWorklet);
- this.recorderWorklet.connect(this.recorderContext.destination);
-
- // Dev note: we can't use `addEventListener` for some reason. It just doesn't work.
- this.recorderWorklet.port.onmessage = (ev) => {
- switch (ev.data['ev']) {
- case PayloadEvent.Timekeep:
- this.processAudioUpdate(ev.data['timeSeconds']);
- break;
- case PayloadEvent.AmplitudeMark:
- // Sanity check to make sure we're adding about one sample per second
- if (ev.data['forSecond'] === this.amplitudes.length) {
- this.amplitudes.push(ev.data['amplitude']);
+ try {
+ this.recorderStream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ channelCount: CHANNELS,
+ noiseSuppression: true, // browsers ignore constraints they can't honour
+ deviceId: CallMediaHandler.getAudioInput(),
+ },
+ });
+ this.recorderContext = createAudioContext({
+ // latencyHint: "interactive", // we don't want a latency hint (this causes data smoothing)
+ });
+ this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream);
+ this.recorderFFT = this.recorderContext.createAnalyser();
+
+ // Bring the FFT time domain down a bit. The default is 2048, and this must be a power
+ // of two. We use 64 points because we happen to know down the line we need less than
+ // that, but 32 would be too few. Large numbers are not helpful here and do not add
+ // precision: they introduce higher precision outputs of the FFT (frequency data), but
+ // it makes the time domain less than helpful.
+ this.recorderFFT.fftSize = 64;
+
+ // Set up our worklet. We use this for timing information and waveform analysis: the
+ // web audio API prefers this be done async to avoid holding the main thread with math.
+ const mxRecorderWorkletPath = document.body.dataset.vectorRecorderWorkletScript;
+ if (!mxRecorderWorkletPath) {
+ // noinspection ExceptionCaughtLocallyJS
+ throw new Error("Unable to create recorder: no worklet script registered");
+ }
+
+ // Connect our inputs and outputs
+ this.recorderSource.connect(this.recorderFFT);
+
+ if (this.recorderContext.audioWorklet) {
+ await this.recorderContext.audioWorklet.addModule(mxRecorderWorkletPath);
+ this.recorderWorklet = new AudioWorkletNode(this.recorderContext, WORKLET_NAME);
+ this.recorderSource.connect(this.recorderWorklet);
+ this.recorderWorklet.connect(this.recorderContext.destination);
+
+ // Dev note: we can't use `addEventListener` for some reason. It just doesn't work.
+ this.recorderWorklet.port.onmessage = (ev) => {
+ switch (ev.data['ev']) {
+ case PayloadEvent.Timekeep:
+ this.processAudioUpdate(ev.data['timeSeconds']);
+ break;
+ case PayloadEvent.AmplitudeMark:
+ // Sanity check to make sure we're adding about one sample per second
+ if (ev.data['forSecond'] === this.amplitudes.length) {
+ this.amplitudes.push(ev.data['amplitude']);
+ }
+ break;
  }
- break;
+ };
+ } else {
+ // Safari fallback: use a processor node instead, buffered to 1024 bytes of data
+ // like the worklet is.
+ this.recorderProcessor = this.recorderContext.createScriptProcessor(1024, CHANNELS, CHANNELS);
+ this.recorderSource.connect(this.recorderProcessor);
+ this.recorderProcessor.connect(this.recorderContext.destination);
+ this.recorderProcessor.addEventListener("audioprocess", this.onAudioProcess);
  }
- };
-
- this.recorder = new Recorder({
- encoderPath, // magic from webpack
- encoderSampleRate: SAMPLE_RATE,
- encoderApplication: 2048, // voice (default is "audio")
- streamPages: true, // this speeds up the encoding process by using CPU over time
- encoderFrameSize: 20, // ms, arbitrary frame size we send to the encoder
- numberOfChannels: CHANNELS,
- sourceNode: this.recorderSource,
- encoderBitRate: BITRATE,
-
- // We use low values for the following to ease CPU usage - the resulting waveform
- // is indistinguishable for a voice message. Note that the underlying library will
- // pick defaults which prefer the highest possible quality, CPU be damned.
- encoderComplexity: 3, // 0-10, 10 is slow and high quality.
- resampleQuality: 3, // 0-10, 10 is slow and high quality
- });
- this.recorder.ondataavailable = (a: ArrayBuffer) => {
- const buf = new Uint8Array(a);
- const newBuf = new Uint8Array(this.buffer.length + buf.length);
- newBuf.set(this.buffer, 0);
- newBuf.set(buf, this.buffer.length);
- this.buffer = newBuf;
- };
+
+ this.recorder = new Recorder({
+ encoderPath, // magic from webpack
+ encoderSampleRate: SAMPLE_RATE,
+ encoderApplication: 2048, // voice (default is "audio")
+ streamPages: true, // this speeds up the encoding process by using CPU over time
+ encoderFrameSize: 20, // ms, arbitrary frame size we send to the encoder
+ numberOfChannels: CHANNELS,
+ sourceNode: this.recorderSource,
+ encoderBitRate: BITRATE,
+
+ // We use low values for the following to ease CPU usage - the resulting waveform
+ // is indistinguishable for a voice message. Note that the underlying library will
+ // pick defaults which prefer the highest possible quality, CPU be damned.
+ encoderComplexity: 3, // 0-10, 10 is slow and high quality.
+ resampleQuality: 3, // 0-10, 10 is slow and high quality
+ });
+ this.recorder.ondataavailable = (a: ArrayBuffer) => {
+ const buf = new Uint8Array(a);
+ const newBuf = new Uint8Array(this.buffer.length + buf.length);
+ newBuf.set(this.buffer, 0);
+ newBuf.set(buf, this.buffer.length);
+ this.buffer = newBuf;
+ };
+ } catch (e) {
+ console.error("Error starting recording: ", e);
+ if (e instanceof DOMException) { // Unhelpful DOMExceptions are common - parse them sanely
+ console.error(`${e.name} (${e.code}): ${e.message}`);
+ }
+
+ // Clean up as best as possible
+ if (this.recorderStream) this.recorderStream.getTracks().forEach(t => t.stop());
+ if (this.recorderSource) this.recorderSource.disconnect();
+ if (this.recorder) this.recorder.close();
+ if (this.recorderContext) {
+ // noinspection ES6MissingAwait - not important that we wait
+ this.recorderContext.close();
+ }
+
+ throw e; // rethrow so upstream can handle it
+ }
  }
 
  private get audioBuffer(): Uint8Array {
@@ -190,14 +221,30 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
  return this.mxc;
  }
 
+ private onAudioProcess = (ev: AudioProcessingEvent) => {
+ this.processAudioUpdate(ev.playbackTime);
+
+ // We skip the functionality of the worklet regarding waveform calculations: we
+ // should get that information pretty quick during the playback info.
+ };
+
  private processAudioUpdate = (timeSeconds: number) => {
  if (!this.recording) return;
 
  // The time domain is the input to the FFT, which means we use an array of the same
  // size. The time domain is also known as the audio waveform. We're ignoring the
  // output of the FFT here (frequency data) because we're not interested in it.
  const data = new Float32Array(this.recorderFFT.fftSize);
- this.recorderFFT.getFloatTimeDomainData(data);
+ if (!this.recorderFFT.getFloatTimeDomainData) {
+ // Safari compat
+ const data2 = new Uint8Array(this.recorderFFT.fftSize);
+ this.recorderFFT.getByteTimeDomainData(data2);
+ for (let i = 0; i < data2.length; i++) {
+ data[i] = percentageWithin(percentageOf(data2[i], 0, 256), -1, 1);
+ }
+ } else {
+ this.recorderFFT.getFloatTimeDomainData(data);
+ }
 
  // We can't just `Array.from()` the array because we're dealing with 32bit floats
  // and the built-in function won't consider that when converting between numbers.
@@ -268,7 +315,11 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
  // Disconnect the source early to start shutting down resources
  await this.recorder.stop(); // stop first to flush the last frame
  this.recorderSource.disconnect();
- this.recorderWorklet.disconnect();
+ if (this.recorderWorklet) this.recorderWorklet.disconnect();
+ if (this.recorderProcessor) {
+ this.recorderProcessor.disconnect();
+ this.recorderProcessor.removeEventListener("audioprocess", this.onAudioProcess);
+ }
 
  // close the context after the recorder so the recorder doesn't try to
  // connect anything to the context (this would generate a warning)