Kaldi 5.1.0 (#1342)

Author: danpovey

.gitignore

@@ -88,6 +88,10 @@ GSYMS
 /tools/openfst-1.3.4/
 /tools/openfst-1.4.1.tar.gz
 /tools/openfst-1.4.1/
+/tools/openfst-1.5.4.tar.gz
+/tools/openfst-1.5.4/
+/tools/openfst-1.6.0.tar.gz
+/tools/openfst-1.6.0/
 /tools/pa_stable_v19_20111121.tgz
 /tools/portaudio/
 /tools/sctk-2.4.0-20091110-0958.tar.bz2

.travis.yml

@@ -20,6 +20,7 @@ addons:
 branches:
  only:
  - master
+ - shortcut
 
 before_install:
  - cat /proc/sys/kernel/core_pattern

egs/fisher_swbd/s5/local/nnet3/run_tdnn_discriminative.sh

@@ -8,7 +8,7 @@ set -o pipefail
 # note: this relies on having a cluster that has plenty of CPUs as well as GPUs,
 # since the lattice generation runs in about real-time, so takes of the order of
 # 1000 hours of CPU time.
-# 
+#
 . ./cmd.sh
 
 
@@ -38,27 +38,21 @@ dir=${srcdir}_${criterion}
 ## Egs options
 frames_per_eg=150
 frames_overlap_per_eg=30
-truncate_deriv_weights=10
 
 ## Nnet training options
 effective_learning_rate=0.00000125
 max_param_change=1
 num_jobs_nnet=4
 num_epochs=2
-regularization_opts= # Applicable for providing --xent-regularize and --l2-regularize options 
+regularization_opts= # Applicable for providing --xent-regularize and --l2-regularize options
 minibatch_size=64
-adjust_priors=true # May need to be set to false 
- # because it does not help in some setups
-modify_learning_rates=true
-last_layer_factor=0.1
-
 ## Decode options
 decode_start_epoch=1 # can be used to avoid decoding all epochs, e.g. if we decided to run more.
 
 if $use_gpu; then
  if ! cuda-compiled; then
- cat <<EOF && exit 1 
-This script is intended to be used with GPUs but you have not compiled Kaldi with CUDA 
+ cat <<EOF && exit 1
+This script is intended to be used with GPUs but you have not compiled Kaldi with CUDA
 If you want to use GPUs (and have them), go to src/, and configure and make on a machine
 where "nvcc" is installed. Otherwise, call this script with --use-gpu false
 EOF
@@ -89,7 +83,7 @@ fi
 if [ -z "$lats_dir" ]; then
  lats_dir=${srcdir}_denlats
  if [ $stage -le 2 ]; then
- nj=100 
+ nj=100
  # this doesn't really affect anything strongly, except the num-jobs for one of
  # the phases of get_egs_discriminative.sh below.
  num_threads_denlats=6
@@ -102,8 +96,8 @@ if [ -z "$lats_dir" ]; then
  fi
 fi
 
-model_left_context=`nnet3-am-info $srcdir/final.mdl | grep "left-context:" | awk '{print $2}'` 
-model_right_context=`nnet3-am-info $srcdir/final.mdl | grep "right-context:" | awk '{print $2}'` 
+model_left_context=$(nnet3-am-info $srcdir/final.mdl | grep "^left-context:" | awk '{print $2}')
+model_right_context=$(nnet3-am-info $srcdir/final.mdl | grep "^right-context:" | awk '{print $2}')
 
 left_context=$[model_left_context + extra_left_context]
 right_context=$[model_right_context + extra_right_context]
@@ -113,7 +107,7 @@ if [ -f $srcdir/frame_subsampling_factor ]; then
  frame_subsampling_opt="--frame-subsampling-factor $(cat $srcdir/frame_subsampling_factor)"
 fi
 
-cmvn_opts=`cat $srcdir/cmvn_opts` 
+cmvn_opts=`cat $srcdir/cmvn_opts`
 
 if [ -z "$degs_dir" ]; then
  degs_dir=${srcdir}_degs
@@ -126,15 +120,12 @@ if [ -z "$degs_dir" ]; then
  # have a higher maximum num-jobs if
  if [ -d ${srcdir}_degs/storage ]; then max_jobs=10; else max_jobs=5; fi
 
- degs_opts="--determinize true --minimize true --remove-output-symbols true --remove-epsilons true --collapse-transition-ids true"
-
  steps/nnet3/get_egs_discriminative.sh \
  --cmd "$decode_cmd --max-jobs-run $max_jobs --mem 20G" --stage $get_egs_stage --cmvn-opts "$cmvn_opts" \
- --adjust-priors $adjust_priors \
  --online-ivector-dir $online_ivector_dir \
  --left-context $left_context --right-context $right_context \
  $frame_subsampling_opt \
- --frames-per-eg $frames_per_eg --frames-overlap-per-eg $frames_overlap_per_eg ${degs_opts} \
+ --frames-per-eg $frames_per_eg --frames-overlap-per-eg $frames_overlap_per_eg \
  $train_data_dir data/lang ${srcdir}_ali $lats_dir $srcdir/final.mdl $degs_dir ;
  fi
 fi
@@ -147,9 +138,7 @@ if [ $stage -le 4 ]; then
  --num-epochs $num_epochs --one-silence-class $one_silence_class --minibatch-size $minibatch_size \
  --num-jobs-nnet $num_jobs_nnet --num-threads $num_threads \
  --regularization-opts "$regularization_opts" \
- --truncate-deriv-weights $truncate_deriv_weights --adjust-priors $adjust_priors \
- --modify-learning-rates $modify_learning_rates --last-layer-factor $last_layer_factor \
- ${degs_dir} $dir 
+ ${degs_dir} $dir
 fi
 
 graph_dir=exp/tri5a/graph_fsh_sw1_tg
@@ -158,8 +147,8 @@ if [ $stage -le 5 ]; then
  for decode_set in eval2000 rt03; do
  (
  num_jobs=`cat data/${decode_set}_hires/utt2spk|cut -d' ' -f2|sort -u|wc -l`
- iter=epoch$x.adj
- 
+ iter=epoch${x}_adj
+
  steps/nnet3/decode.sh --nj $num_jobs --cmd "$decode_cmd" --iter $iter \
  --online-ivector-dir exp/nnet3/ivectors_${decode_set} \
  $graph_dir data/${decode_set}_hires $dir/decode_${decode_set}_fsh_sw1_tg_$iter ;
@@ -181,4 +170,3 @@ fi
 
 
 exit 0;
-

egs/fisher_swbd/s5/local/rt03_data_prep.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
-# RT-03 data preparation (conversational telephone speech part only) 
+# RT-03 data preparation (conversational telephone speech part only)
 # Adapted from Arnab Ghoshal's script for Hub-5 Eval 2000 by Peng Qi
 
 # To be run from one directory above this script.
 
 # Expects the standard directory layout for RT-03
 
 if [ $# -ne 1 ]; then
- echo "Usage: "`basename $0`" <rt03-dir>"
+ echo "Usage: $0 <rt03-dir>"
+ echo "e.g.: $0 /export/corpora/LDC/LDC2007S10"
  echo "See comments in the script for more details"
  exit 1
 fi
@@ -19,7 +20,7 @@ sdir=$1
 [ ! -d $sdir/data/references/eval03/english/cts ] \
  && echo Expecting directory $tdir/data/references/eval03/english/cts to be present && exit 1;
 
-. path.sh 
+. path.sh
 
 dir=data/local/rt03
 mkdir -p $dir
@@ -37,7 +38,7 @@ sph2pipe=$KALDI_ROOT/tools/sph2pipe_v2.5/sph2pipe
  && echo "Could not execute the sph2pipe program at $sph2pipe" && exit 1;
 
 awk -v sph2pipe=$sph2pipe '{
- printf("%s-A %s -f wav -p -c 1 %s |\n", $1, sph2pipe, $2); 
+ printf("%s-A %s -f wav -p -c 1 %s |\n", $1, sph2pipe, $2);
  printf("%s-B %s -f wav -p -c 2 %s |\n", $1, sph2pipe, $2);
 }' < $dir/sph.scp | sort > $dir/wav.scp || exit 1;
 #side A - channel 1, side B - channel 2
@@ -47,7 +48,7 @@ awk -v sph2pipe=$sph2pipe '{
 # sw02001-A_000098-001156 sw02001-A 0.98 11.56
 #pem=$sdir/english/hub5e_00.pem
 #[ ! -f $pem ] && echo "No such file $pem" && exit 1;
-# pem file has lines like: 
+# pem file has lines like:
 # en_4156 A unknown_speaker 301.85 302.48
 
 #grep -v ';;' $pem \
@@ -59,7 +60,7 @@ cat $tdir/*.stm | grep -v ';;' | grep -v inter_segment_gap \
  | sort -u > $dir/segments
 
 # stm file has lines like:
-# en_4156 A en_4156_A 357.64 359.64 <O,en,F,en-F> HE IS A POLICE OFFICER 
+# en_4156 A en_4156_A 357.64 359.64 <O,en,F,en-F> HE IS A POLICE OFFICER
 # TODO(arnab): We should really be lowercasing this since the Edinburgh
 # recipe uses lowercase. This is not used in the actual scoring.
 #grep -v ';;' $tdir/reference/hub5e00.english.000405.stm \
@@ -77,7 +78,7 @@ cat $tdir/*.stm | \
  grep -v inter_segment_gap | \
  awk '{
  printf $1; if ($1==";;") printf(" %s",$2); else printf(($2==1)?" A":" B"); for(n=3;n<=NF;n++) printf(" %s", $n); print ""; }'\
- > $dir/stm 
+ > $dir/stm
 #$tdir/reference/hub5e00.english.000405.stm > $dir/stm
 cp $rtroot/data/trans_rules/en20030506.glm $dir/glm
 
@@ -87,10 +88,10 @@ cp $rtroot/data/trans_rules/en20030506.glm $dir/glm
  echo "Segments from pem file and stm file do not match." && exit 1;
 
 grep -v IGNORE_TIME_SEGMENT_ $dir/text.all > $dir/text
- 
+
 # create an utt2spk file that assumes each conversation side is
 # a separate speaker.
-awk '{print $1,$2;}' $dir/segments > $dir/utt2spk 
+awk '{print $1,$2;}' $dir/segments > $dir/utt2spk
 utils/utt2spk_to_spk2utt.pl $dir/utt2spk > $dir/spk2utt
 
 # cp $dir/segments $dir/segments.tmp
@@ -110,4 +111,3 @@ done
 
 echo Data preparation and formatting completed for RT-03
 echo "(but not MFCC extraction)"
-

egs/librispeech/s5/local/chain/run_tdnn_discriminative.sh

@@ -10,7 +10,7 @@ set -e
 # note: this relies on having a cluster that has plenty of CPUs as well as GPUs,
 # since the lattice generation runs in about real-time, so takes of the order of
 # 1000 hours of CPU time.
-# 
+#
 
 
 stage=0
@@ -44,7 +44,6 @@ dir=${srcdir}_${criterion}
 ## Egs options
 frames_per_eg=150
 frames_overlap_per_eg=30
-truncate_deriv_weights=10
 
 ## Nnet training options
 effective_learning_rate=0.000001
@@ -59,8 +58,8 @@ decode_start_epoch=1 # can be used to avoid decoding all epochs, e.g. if we deci
 
 if $use_gpu; then
  if ! cuda-compiled; then
- cat <<EOF && exit 1 
-This script is intended to be used with GPUs but you have not compiled Kaldi with CUDA 
+ cat <<EOF && exit 1
+This script is intended to be used with GPUs but you have not compiled Kaldi with CUDA
 If you want to use GPUs (and have them), go to src/, and configure and make on a machine
 where "nvcc" is installed. Otherwise, call this script with --use-gpu false
 EOF
@@ -95,15 +94,15 @@ if [ $frame_subsampling_factor -ne 1 ]; then
  rm ${train_ivector_dir}_fs/ivector_online.scp 2>/dev/null || true
 
  data_dirs=
- for x in `seq -$[frame_subsampling_factor/2] $[frame_subsampling_factor/2]`; do 
+ for x in `seq -$[frame_subsampling_factor/2] $[frame_subsampling_factor/2]`; do
  steps/shift_feats.sh --cmd "$train_cmd --max-jobs-run 40" --nj 350 \
  $x $train_data_dir exp/shift_hires mfcc_hires
  utils/fix_data_dir.sh ${train_data_dir}_fs$x
  data_dirs="$data_dirs ${train_data_dir}_fs$x"
  awk -v nfs=$x '{print "fs"nfs"-"$0}' $train_ivector_dir/ivector_online.scp >> ${train_ivector_dir}_fs/ivector_online.scp
  done
  utils/combine_data.sh ${train_data_dir}_fs $data_dirs
- for x in `seq -$[frame_subsampling_factor/2] $[frame_subsampling_factor/2]`; do 
+ for x in `seq -$[frame_subsampling_factor/2] $[frame_subsampling_factor/2]`; do
  rm -r ${train_data_dir}_fs$x
  done
  fi
@@ -112,9 +111,9 @@ if [ $frame_subsampling_factor -ne 1 ]; then
 
  affix=_fs
 fi
- 
+
 rm ${train_ivector_dir}_fs/ivector_online.scp 2>/dev/null || true
-for x in `seq -$[frame_subsampling_factor/2] $[frame_subsampling_factor/2]`; do 
+for x in `seq -$[frame_subsampling_factor/2] $[frame_subsampling_factor/2]`; do
  awk -v nfs=$x '{print "fs"nfs"-"$0}' $train_ivector_dir/ivector_online.scp >> ${train_ivector_dir}_fs/ivector_online.scp
 done
 train_ivector_dir=${train_ivector_dir}_fs
@@ -133,7 +132,7 @@ fi
 if [ -z "$lats_dir" ]; then
  lats_dir=${srcdir}_denlats${affix}
  if [ $stage -le 2 ]; then
- nj=50 
+ nj=50
  # this doesn't really affect anything strongly, except the num-jobs for one of
  # the phases of get_egs_discriminative.sh below.
  num_threads_denlats=6
@@ -147,16 +146,13 @@ if [ -z "$lats_dir" ]; then
  fi
 fi
 
-model_left_context=`nnet3-am-info $srcdir/final.mdl | grep "left-context:" | awk '{print $2}'` 
-model_right_context=`nnet3-am-info $srcdir/final.mdl | grep "right-context:" | awk '{print $2}'` 
+model_left_context=`nnet3-am-info $srcdir/final.mdl | grep "left-context:" | awk '{print $2}'`
+model_right_context=`nnet3-am-info $srcdir/final.mdl | grep "right-context:" | awk '{print $2}'`
 
 left_context=$[model_left_context + extra_left_context]
 right_context=$[model_right_context + extra_right_context]
 
-valid_left_context=$[valid_left_context + frames_per_eg]
-valid_right_context=$[valid_right_context + frames_per_eg]
-
-cmvn_opts=`cat $srcdir/cmvn_opts` 
+cmvn_opts=`cat $srcdir/cmvn_opts`
 
 if [ -z "$degs_dir" ]; then
  degs_dir=${srcdir}_degs${affix}
@@ -169,16 +165,13 @@ if [ -z "$degs_dir" ]; then
  # have a higher maximum num-jobs if
  if [ -d ${srcdir}_degs/storage ]; then max_jobs=10; else max_jobs=5; fi
 
- degs_opts="--determinize true --minimize true --remove-output-symbols true --remove-epsilons true --collapse-transition-ids true"
-
  steps/nnet3/get_egs_discriminative.sh \
  --cmd "$decode_cmd --max-jobs-run $max_jobs --mem 20G" --stage $get_egs_stage --cmvn-opts "$cmvn_opts" \
  --adjust-priors false --acwt 1.0 \
  --online-ivector-dir $train_ivector_dir \
  --left-context $left_context --right-context $right_context \
- --valid-left-context $valid_left_context --valid-right-context $valid_right_context \
- --priors-left-context $valid_left_context --priors-right-context $valid_right_context $frame_subsampling_opt \
- --frames-per-eg $frames_per_eg --frames-overlap-per-eg $frames_overlap_per_eg ${degs_opts} \
+ $frame_subsampling_opt \
+ --frames-per-eg $frames_per_eg --frames-overlap-per-eg $frames_overlap_per_eg \
  $train_data_dir $lang ${srcdir}_ali${affix} $lats_dir $srcdir/final.mdl $degs_dir ;
  fi
 fi
@@ -191,7 +184,7 @@ if [ $stage -le 4 ]; then
  --num-epochs $num_epochs --one-silence-class $one_silence_class --minibatch-size $minibatch_size \
  --num-jobs-nnet $num_jobs_nnet --num-threads $num_threads \
  --regularization-opts "$regularization_opts" --use-frame-shift false \
- --truncate-deriv-weights $truncate_deriv_weights --adjust-priors false \
+ --adjust-priors false \
  ${degs_dir} $dir ;
 fi
 
@@ -202,7 +195,7 @@ if [ $stage -le 5 ]; then
  (
  num_jobs=`cat data/${decode_set}_hires/utt2spk|cut -d' ' -f2|sort -u|wc -l`
  iter=epoch$[x*frame_subsampling_factor]
- 
+
  steps/nnet3/decode.sh --nj $num_jobs --cmd "$decode_cmd" --iter $iter \
  --acwt 1.0 --post-decode-acwt 10.0 \
  --online-ivector-dir exp/nnet3${nnet3_affix}/ivectors_${decode_set}_hires \
@@ -219,7 +212,7 @@ if [ $stage -le 5 ]; then
  done
  done
  wait
- [ -f $dir/.error ] && echo "$0: there was a problem while decoding" && exit 1 
+ [ -f $dir/.error ] && echo "$0: there was a problem while decoding" && exit 1
 fi
 
 if [ $stage -le 6 ] && $cleanup; then
@@ -231,4 +224,3 @@ fi
 
 
 exit 0;
-