fix

Author: freewym

egs/ami/s5b/local/rnnlm/tuning/run_lstm_tdnn_b.sh

@@ -13,7 +13,7 @@
 
 # Begin configuration section.
 cmd=run.pl
-affix=b
+dir=exp/rnnlm_lstm_tdnn_b
 embedding_dim=200
 embedding_l2=0.005 # embedding layer l2 regularize
 comp_l2=0.005 # component-level l2 regularize
@@ -22,15 +22,12 @@ epochs=90
 mic=sdm1
 stage=-10
 train_stage=0
-alpha=0.0
-back_interval=1
 
 . utils/parse_options.sh
 train=data/$mic/train/text
 dev=data/$mic/dev/text
 wordlist=data/lang/words.txt
 text_dir=data/rnnlm/text
-dir=exp/rnnlm_lstm_tdnn_$affix
 mkdir -p $dir/config
 set -e
 
@@ -94,10 +91,9 @@ if [ $stage -le 2 ]; then
 fi
 
 if [ $stage -le 3 ]; then
- backstitch_opt="--backstitch-scale $alpha --backstitch-interval $back_interval"
  rnnlm/train_rnnlm.sh --embedding_l2 $embedding_l2 \
  --stage $train_stage \
- --num-epochs $epochs --cmd "queue.pl" $backstitch_opt $dir
+ --num-epochs $epochs --cmd "queue.pl" $dir
 fi
 
 exit 0

egs/ami/s5b/local/rnnlm/tuning/run_lstm_tdnn_bs_1a.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# Copyright 2012 Johns Hopkins University (author: Daniel Povey) Tony Robinson
+# 2017 Hainan Xu
+# 2017 Ke Li
+# 2017 Yiming Wang
+
+# This script is similar to rnnlm_lstm_tdnn_b.sh except for adding backstitch training
+
+# rnnlm/train_rnnlm.sh: best iteration (out of 18) was 17, linking it to final iteration.
+# rnnlm/train_rnnlm.sh: train/dev perplexity was 45.6 / 68.7.
+# Train objf: -651.50 -4.44 -4.26 -4.15 -4.08 -4.03 -4.00 -3.97 -3.94 -3.92 -3.90 -3.89 -3.88 -3.86 -3.85 -3.84 -3.83 -3.82
+# Dev objf: -10.76 -4.68 -4.47 -4.38 -4.33 -4.29 -4.28 -4.27 -4.26 -4.26 -4.25 -4.24 -4.24 -4.24 -4.23 -4.23 -4.23 -4.23
+
+# Begin configuration section.
+cmd=run.pl
+affix=1a
+embedding_dim=200
+embedding_l2=0.005 # embedding layer l2 regularize
+comp_l2=0.005 # component-level l2 regularize
+output_l2=0.005 # output-layer l2 regularize
+epochs=90
+mic=sdm1
+stage=-10
+train_stage=0
+alpha=0.8
+back_interval=1
+
+. utils/parse_options.sh
+train=data/$mic/train/text
+dev=data/$mic/dev/text
+wordlist=data/lang/words.txt
+text_dir=data/rnnlm/text
+dir=exp/rnnlm_lstm_tdnn_bs_$affix
+mkdir -p $dir/config
+set -e
+
+for f in $train $dev $wordlist; do
+ [ ! -f $f ] && \
+ echo "$0: expected file $f to exist; search for run.sh and utils/prepare_lang.sh in run.sh" && exit 1
+done
+
+if [ $stage -le 0 ]; then
+ mkdir -p $text_dir
+ cat $train | cut -d ' ' -f2- > $text_dir/ami.txt
+ cat $dev | cut -d ' ' -f2- > $text_dir/dev.txt
+fi
+
+if [ $stage -le 1 ]; then
+ cp $wordlist $dir/config/
+ n=`cat $dir/config/words.txt | wc -l`
+ echo "<brk> $n" >> $dir/config/words.txt
+
+ # words that are not present in words.txt but are in the training or dev data, will be
+ # mapped to <unk> during training.
+ echo "<unk>" >$dir/config/oov.txt
+
+ cat > $dir/config/data_weights.txt <<EOF
+ami 1 1.0
+EOF
+
+ rnnlm/get_unigram_probs.py --vocab-file=$dir/config/words.txt \
+ --unk-word="<unk>" \
+ --data-weights-file=$dir/config/data_weights.txt \
+ $text_dir | awk 'NF==2' >$dir/config/unigram_probs.txt
+
+ # choose features
+ rnnlm/choose_features.py --unigram-probs=$dir/config/unigram_probs.txt \
+ --use-constant-feature=true \
+ --top-word-features 10000 \
+ --min-frequency 1.0e-03 \
+ --special-words='<s>,</s>,<brk>,<unk>,[noise],[laughter]' \
+ $dir/config/words.txt > $dir/config/features.txt
+
+lstm_opts="l2-regularize=$comp_l2"
+tdnn_opts="l2-regularize=$comp_l2"
+output_opts="l2-regularize=$output_l2"
+
+ cat >$dir/config/xconfig <<EOF
+input dim=$embedding_dim name=input
+lstm-layer name=lstm1 cell-dim=$embedding_dim $lstm_opts
+relu-renorm-layer name=tdnn dim=$embedding_dim $tdnn_opts input=Append(0, IfDefined(-1))
+lstm-layer name=lstm2 cell-dim=$embedding_dim $lstm_opts
+output-layer name=output $output_opts include-log-softmax=false dim=$embedding_dim
+EOF
+ rnnlm/validate_config_dir.sh $text_dir $dir/config
+fi
+
+if [ $stage -le 2 ]; then
+ # the --unigram-factor option is set larger than the default (100)
+ # in order to reduce the size of the sampling LM, because rnnlm-get-egs
+ # was taking up too much CPU (as much as 10 cores).
+ rnnlm/prepare_rnnlm_dir.sh --unigram-factor 200 \
+ $text_dir $dir/config $dir
+fi
+
+if [ $stage -le 3 ]; then
+ backstitch_opt="--rnnlm.backstitch-scale $alpha --rnnlm.backstitch-interval $back_interval --embedding.backstitch-scale $alpha --embedding.backstitch-interval $back_interval"
+ rnnlm/train_rnnlm.sh --embedding_l2 $embedding_l2 \
+ --stage $train_stage \
+ --num-epochs $epochs --cmd "queue.pl" $backstitch_opt $dir
+fi
+
+exit 0

src/rnnlm/rnnlm-core-training.cc

@@ -166,7 +166,8 @@ void RnnlmCoreTrainer::Train(
  ProvideInput(minibatch, derived, word_embedding, &computer);
  computer.Run(); // This is the forward pass.
 
- ProcessOutput(true, minibatch, derived, word_embedding,
+ bool is_backstitch_step1 = true;
+ ProcessOutput(is_backstitch_step1, minibatch, derived, word_embedding,
  &computer, word_embedding_deriv);
 
  computer.Run(); // This is the backward pass.

src/rnnlm/rnnlm-core-training.h

@@ -75,12 +75,12 @@ struct RnnlmCoreTrainerOptions {
  "related to parallelization by model averaging.");
  opts->Register("backstitch-training-scale", &backstitch_training_scale,
  "backstitch training factor. "
- "if 0 then in the normal training mode. It is referred as "
+ "if 0 then in the normal training mode. It is referred to as "
  "'\\alpha' in our publications.");
  opts->Register("backstitch-training-interval",
  &backstitch_training_interval,
  "do backstitch training with the specified interval of "
- "minibatches. It is referred as 'n' in our publications.");
+ "minibatches. It is referred to as 'n' in our publications.");
  }
 };
 

src/rnnlm/rnnlm-embedding-training.h

@@ -92,12 +92,12 @@ struct RnnlmEmbeddingTrainerOptions {
  "training the word-embedding matrix.");
  opts->Register("backstitch-training-scale", &backstitch_training_scale,
  "backstitch training factor. "
- "if 0 then in the normal training mode. It is referred as "
+ "if 0 then in the normal training mode. It is referred to as "
  "'\\alpha' in our publications.");
  opts->Register("backstitch-training-interval",
  &backstitch_training_interval,
  "do backstitch training with the specified interval of "
- "minibatches. It is referred as 'n' in our publications.");
+ "minibatches. It is referred to as 'n' in our publications.");
  opts->Register("use-natural-gradient", &use_natural_gradient,
  "True if you want to use natural gradient to update the "
  "embedding matrix");

src/rnnlm/rnnlm-training.cc

@@ -230,14 +230,9 @@ void RnnlmTrainer::TrainInternal() {
  GetWordEmbedding(&word_embedding_storage, &word_embedding);
 
  CuMatrix<BaseFloat> word_embedding_deriv;
- if (train_embedding_) {
- KALDI_ASSERT(core_config_.backstitch_training_scale ==
- embedding_config_.backstitch_training_scale &&
- core_config_.backstitch_training_interval ==
- embedding_config_.backstitch_training_interval);
+ if (train_embedding_)
  word_embedding_deriv.Resize(word_embedding->NumRows(),
  word_embedding->NumCols());
- }
 
  if (core_config_.backstitch_training_scale > 0.0 &&
  num_minibatches_processed_ % core_config_.backstitch_training_interval ==

src/rnnlmbin/rnnlm-train.cc

@@ -42,9 +42,6 @@ int main(int argc, char *argv[]) {
  std::string embedding_rxfilename;
  std::string embedding_wxfilename;
  std::string word_features_rxfilename;
- // backstitch options
- BaseFloat backstitch_training_scale = 0.0;
- int32 backstitch_training_interval = 1;
  // binary mode for writing output.
  bool binary = true;
 
@@ -91,14 +88,6 @@ int main(int argc, char *argv[]) {
  "letters and other hand-built features; it's not trainable."
  " If present, the embedding matrix read via --read-embedding "
  "will be interpreted as a feature-embedding matrix.");
- po.Register("backstitch-training-scale", &backstitch_training_scale,
- "backstitch training factor. "
- "if 0 then in the normal training mode. It is referred as "
- "'\\alpha' in our publications.");
- po.Register("backstitch-training-interval",
- &backstitch_training_interval,
- "do backstitch training with the specified interval of "
- "minibatches. It is referred as 'n' in our publications.");
  po.Register("binary", &binary,
  "If true, write outputs in binary form.");
 
@@ -121,11 +110,6 @@ int main(int argc, char *argv[]) {
 
  po.Read(argc, argv);
 
- core_config.backstitch_training_scale = backstitch_training_scale;
- core_config.backstitch_training_interval = backstitch_training_interval;
- embedding_config.backstitch_training_scale = backstitch_training_scale;
- embedding_config.backstitch_training_interval = backstitch_training_interval;
-
  if (po.NumArgs() != 1) {
  po.PrintUsage();
  exit(1);