Audio integration to front end (#360)

* Audio integration to front end * Add flask end point * Read and decode audio data * refract a typo * fix clang format

Author: nickyfantasy

frontend/src/audio/ui/AudioPanel.vue

@@ -14,8 +14,7 @@
  v-model="currentIndex"
  ></v-slider>
 
- <audio controls>
- <source src="horse.ogg" type="audio/ogg">
+ <audio controls :src="audioData.audioSrc">
  Your browser does not support the audio element.
  </audio>
  </v-card>
@@ -82,7 +81,7 @@ export default {
  if (this.data && this.data[index]) {
  let currentAudioInfo = this.data ? this.data[index] : {};
  let {query, step, wall_time} = currentAudioInfo;
- let url = '/data/plugin/images/individualAudio?ts=' + wall_time;
+ let url = '/data/plugin/audio/individualAudio?ts=' + wall_time;
  let audioSrc = [url, query].join('&');
  this.audioData = {
  audioSrc,

frontend/tool/webpack.config.js

@@ -92,7 +92,7 @@ const config = {
  use: getLoaders(isDev, 'stylus')
  },
  {
- test: /\.(gif|png|jpe?g|svg)$/i,
+ test: /\.(gif|png|jpe?g|svg|wav)$/i,
  loader: 'file-loader',
  options: {
  name: 'assets/[name].[hash].[ext]'

visualdl/server/lib.py

@@ -6,6 +6,7 @@
 import numpy as np
 from PIL import Image
 from .log import logger
+import wave
 
 try:
  from urllib.parse import urlencode
@@ -155,6 +156,86 @@ def get_invididual_image(storage, mode, tag, step_index, max_size=80):
  return tempfile
 
 
+def get_audio_tags(storage):
+ result = {}
+
+ for mode in storage.modes():
+ with storage.mode(mode) as reader:
+ tags = reader.tags('audio')
+ if tags:
+ result[mode] = {}
+ for tag in tags:
+ audio = reader.audio(tag)
+ for i in range(max(1, audio.num_samples())):
+ caption = tag if audio.num_samples(
+ ) <= 1 else '%s/%d' % (tag, i)
+ result[mode][caption] = {
+ 'displayName': caption,
+ 'description': "",
+ 'samples': 1,
+ }
+
+ return result
+
+
+def get_audio_tag_steps(storage, mode, tag):
+ # remove suffix '/x'
+ res = re.search(r".*/([0-9]+$)", tag)
+ sample_index = 0
+ origin_tag = tag
+ if res:
+ tag = tag[:tag.rfind('/')]
+ sample_index = int(res.groups()[0])
+
+ with storage.mode(mode) as reader:
+ audio = reader.audio(tag)
+ res = []
+
+ for step_index in range(audio.num_records()):
+ record = audio.record(step_index, sample_index)
+
+ query = urlencode({
+ 'sample': 0,
+ 'index': step_index,
+ 'tag': origin_tag,
+ 'run': mode,
+ })
+ res.append({
+ 'step': record.step_id(),
+ 'wall_time': audio.timestamp(step_index),
+ 'query': query,
+ })
+
+ return res
+
+
+def get_individual_audio(storage, mode, tag, step_index, max_size=80):
+
+ with storage.mode(mode) as reader:
+ res = re.search(r".*/([0-9]+$)", tag)
+ # remove suffix '/x'
+ offset = 0
+ if res:
+ offset = int(res.groups()[0])
+ tag = tag[:tag.rfind('/')]
+
+ audio = reader.audio(tag)
+ record = audio.record(step_index, offset)
+
+ data = np.array(record.data(), dtype='uint8')
+
+ tempfile = NamedTemporaryFile(mode='w+b', suffix='.wav')
+
+ wavfile = wave.open(tempfile, 'wb')
+ wavfile.setnchannels(2)
+ wavfile.setsampwidth(2)
+ wavfile.writeframes(data.tostring())
+
+ tempfile.seek(0, 0)
+
+ return tempfile
+
+
 def get_histogram_tags(storage):
  return get_tags(storage, 'histogram')
 

visualdl/server/visualDL

@@ -163,6 +163,14 @@ def image_tags():
  return Response(json.dumps(result), mimetype='application/json')
 
 
+@app.route("/data/plugin/audio/tags")
+def audio_tags():
+ data = cache_get("/data/plugin/audio/tags", try_call, lib.get_audio_tags,
+ log_reader)
+ result = gen_result(0, "", data)
+ return Response(json.dumps(result), mimetype='application/json')
+
+
 @app.route("/data/plugin/histograms/tags")
 def histogram_tags():
  data = cache_get("/data/plugin/histograms/tags", try_call,
@@ -245,6 +253,35 @@ def embeddings():
  result = gen_result(0, "", data)
  return Response(json.dumps(result), mimetype='application/json')
 
+@app.route('/data/plugin/audio/audio')
+def audio():
+ mode = request.args.get('run')
+ tag = request.args.get('tag')
+ key = os.path.join('/data/plugin/audio/audio', mode, tag)
+
+ data = cache_get(key, try_call, lib.get_audio_tag_steps, log_reader, mode,
+ tag)
+ result = gen_result(0, "", data)
+
+ return Response(json.dumps(result), mimetype='application/json')
+
+
+@app.route('/data/plugin/audio/individualAudio')
+def individual_audio():
+ mode = request.args.get('run')
+ print mode
+ tag = request.args.get('tag') # include a index
+ step_index = int(request.args.get('index')) # index of step
+
+ key = os.path.join('/data/plugin/audio/individualAudio', mode, tag,
+ str(step_index))
+ data = cache_get(key, try_call, lib.get_individual_audio, log_reader, mode,
+ tag, step_index)
+ response = send_file(
+ data, as_attachment=True, attachment_filename='audio.wav')
+ return response
+
+
 @app.route('/data/plugin/graphs/graph')
 def graph():
  # TODO(ChunweiYan) need to add a config for whether have graph.