rimes handwriting database

egs/madcat_ar/v1/local/prepare_data.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright 2017 Chun Chieh Chang
+# 2017 Ashish Arora
+# 2017 Hossein Hadian
+# Apache 2.0
+
+# This script downloads the data splits for MADCAT Arabic dataset and prepares the training
+# validation, and test data (i.e text, images.scp, utt2spk and spk2utt) by calling process_data.py.
+# It also uses Arabic Gigaword text corpus for language modeling.
+
+# Eg. local/prepare_data.sh
+# Eg. text file: LDC0001_000399_NHR_ARB_20070113.0052_11_LDC0001_0z11 
+# وهناك تداخل بين الرأسمالية الإسرائيلية
+# utt2spk file: LDC0001_000397_NHR_ARB_20070113.0052_11_LDC0001_00z1 LDC0001
+# images.scp file: LDC0001_000397_NHR_ARB_20070113.0052_11_LDC0001_00z1 
+# data/local/train/1/NHR_ARB_20070113.0052_11_LDC0001_00z1.png
+
+download_dir1=/export/corpora/LDC/LDC2012T15/data
+download_dir2=/export/corpora/LDC/LDC2013T09/data
+download_dir3=/export/corpora/LDC/LDC2013T15/data
+train_split_url=http://www.openslr.org/resources/48/madcat.train.raw.lineid
+test_split_url=http://www.openslr.org/resources/48/madcat.test.raw.lineid
+dev_split_url=http://www.openslr.org/resources/48/madcat.dev.raw.lineid
+data_splits=data/download/data_splits
+stage=0
+download_dir=data/download
+gigacorpus=data/local/gigawordcorpus
+gigaword_loc=/export/corpora5/LDC/LDC2011T11
+use_extra_corpus_text=true
+
+. ./cmd.sh
+. ./path.sh
+. ./utils/parse_options.sh || exit 1;
+
+if [ -d $data_splits ]; then
+ echo "$0: Not downloading the data splits as it is already there."
+else
+ if [ ! -f $data_splits/madcat.train.raw.lineid ]; then
+ mkdir -p $data_splits
+ echo "$0: Downloading the data splits..."
+ wget -P $data_splits $train_split_url || exit 1;
+ wget -P $data_splits $test_split_url || exit 1;
+ wget -P $data_splits $dev_split_url || exit 1;
+ fi
+ echo "$0: Done downloading the data splits"
+fi
+
+if [ -d $download_dir1 ]; then
+ echo "$0: madcat arabic data directory is present."
+else
+ if [ ! -f $download_dir1/madcat/*.madcat.xml ]; then
+ echo "$0: please download madcat data..."
+ fi
+fi
+
+mkdir -p $download_dir data/local
+if $use_extra_corpus_text; then
+ mkdir -p $gigacorpus
+ cp -r $gigaword_loc/. $gigacorpus
+ for newswire in aaw_arb afp_arb ahr_arb asb_arb hyt_arb nhr_arb qds_arb umh_arb xin_arb; do
+ for file in $gigacorpus/arb_gw_5/data/$newswire/*.gz; do
+ gzip -d $file
+ done
+ for file in $gigacorpus/arb_gw_5/data/$newswire/*; do
+ sed -e '/^<[^>]*>$/d; s/``/"/g; s/\x27\x27/"/g' $file >> $gigacorpus/arb_gw_5/data/${newswire}_combined.txt
+ done
+ done
+fi

egs/madcat_ar/v1/run_end2end.sh

@@ -19,6 +19,7 @@ images_scp_dir=data/local
 overwrite=false
 subset=false
 augment=false
+use_extra_corpus_text=true
 . ./cmd.sh ## You'll want to change cmd.sh to something that will work on your system.
  ## This relates to the queue.
 . ./path.sh
@@ -35,9 +36,9 @@ if [ $stage -le 0 ]; then
  echo "Exiting with status 1 to avoid data corruption"
  exit 1;
  fi
- echo "$0: Downloading data splits...$(date)"
- local/download_data.sh --data_splits $data_splits_dir --download_dir1 $download_dir1 \
- --download_dir2 $download_dir2 --download_dir3 $download_dir3
+ local/prepare_data.sh --data_splits $data_splits_dir --download_dir1 $download_dir1 \
+  --download_dir2 $download_dir2 --download_dir3 $download_dir3 \
+ --use_extra_corpus_text $use_extra_corpus_text
 
  for set in test train dev; do
  data_split_file=$data_splits_dir/madcat.$set.raw.lineid
@@ -48,7 +49,7 @@ if [ $stage -le 0 ]; then
  --data data/local/$set --subset $subset --augment $augment || exit 1
  done
 
- echo "$0: Preparing data..."
+ echo "$0: Processing data..."
  for set in dev train test; do
  local/process_data.py $download_dir1 $download_dir2 $download_dir3 \
  $data_splits_dir/madcat.$set.raw.lineid data/$set $images_scp_dir/$set/images.scp \

egs/rimes/README.txt

@@ -0,0 +1,13 @@
+Rimes is a French handwriting recognition database created by A2iA.
+The database was created by asking individuals to write letters on a given scenario like
+a change of personal information, payment difficulty, damage declaration. The
+dataset has been used in several international research including ICFHR 2008,
+ICDAR-2009, ICDAR-2011 competitions for isolated word level and
+line level recognition tasks.
+
+It contains 11333 training lines and 788 test lines. It does not include
+a validation split but in a recent publication a 10% sampling of the total
+training lines for validation purposes were performed
+(http://www.jpuigcerver.net/pubs/jpuigcerver_icdar2017.pdf).
+We have used a similar train, test and validation split.
+More info: http://www.a2ialab.com/doku.php?id=rimes_database:start

egs/rimes/v1/cmd.sh

@@ -0,0 +1,13 @@
+# you can change cmd.sh depending on what type of queue you are using.
+# If you have no queueing system and want to run on a local machine, you
+# can change all instances 'queue.pl' to run.pl (but be careful and run
+# commands one by one: most recipes will exhaust the memory on your
+# machine). queue.pl works with GridEngine (qsub). slurm.pl works
+# with slurm. Different queues are configured differently, with different
+# queue names and different ways of specifying things like memory;
+# to account for these differences you can create and edit the file
+# conf/queue.conf to match your queue's configuration. Search for
+# conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information,
+# or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl.
+
+export cmd="retry.pl queue.pl"

egs/rimes/v1/image

@@ -0,0 +1 @@
+../../cifar/v1/image/

egs/rimes/v1/local/chain/compare_wer.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# this script is used for comparing decoding results between systems.
+# e.g. local/chain/compare_wer.sh exp/chain/cnn{1a,1b}
+
+# Copyright 2017 Chun Chieh Chang
+# 2017 Ashish Arora
+
+if [ $# == 0 ]; then
+ echo "Usage: $0: <dir1> [<dir2> ... ]"
+ echo "e.g.: $0 exp/chain/cnn{1a,1b}"
+ exit 1
+fi
+. ./path.sh
+
+echo "# $0 $*"
+used_epochs=false
+
+echo -n "# System "
+for x in $*; do printf "% 10s" " $(basename $x)"; done
+echo
+
+echo -n "# WER "
+for x in $*; do
+ wer=$(cat $x/decode_test/scoring_kaldi/best_wer | awk '{print $2}')
+ printf "% 10s" $wer
+done
+echo
+
+echo -n "# CER "
+for x in $*; do
+ cer=$(cat $x/decode_test/scoring_kaldi/best_cer | awk '{print $2}')
+ printf "% 10s" $cer
+done
+echo
+
+echo -n "# WER val "
+for x in $*; do
+ wer=$(cat $x/decode_val/scoring_kaldi/best_wer | awk '{print $2}')
+ printf "% 10s" $wer
+done
+echo
+
+echo -n "# CER val "
+for x in $*; do
+ cer=$(cat $x/decode_val/scoring_kaldi/best_cer | awk '{print $2}')
+ printf "% 10s" $cer
+done
+echo
+
+if $used_epochs; then
+ exit 0; # the diagnostics aren't comparable between regular and discriminatively trained systems.
+fi
+
+echo -n "# Final train prob "
+for x in $*; do
+ prob=$(grep Overall $x/log/compute_prob_train.final.log | grep -v xent | awk '{printf("%.4f", $8)}')
+ printf "% 10s" $prob
+done
+echo
+
+echo -n "# Final valid prob "
+for x in $*; do
+ prob=$(grep Overall $x/log/compute_prob_valid.final.log | grep -v xent | awk '{printf("%.4f", $8)}')
+ printf "% 10s" $prob
+done
+echo
+
+echo -n "# Final train prob (xent) "
+for x in $*; do
+ prob=$(grep Overall $x/log/compute_prob_train.final.log | grep -w xent | awk '{printf("%.4f", $8)}')
+ printf "% 10s" $prob
+done
+echo
+
+echo -n "# Final valid prob (xent) "
+for x in $*; do
+ prob=$(grep Overall $x/log/compute_prob_valid.final.log | grep -w xent | awk '{printf("%.4f", $8)}')
+ printf "% 10s" $prob
+done
+echo
+
+echo -n "# Parameters "
+for x in $*; do
+ params=$(nnet3-info $x/final.mdl 2>/dev/null | grep num-parameters | cut -d' ' -f2 | awk '{printf "%0.2fM\n",$1/1000000}')
+ printf "% 10s" $params
+done
+echo

egs/rimes/v1/local/chain/run_cnn_e2eali.sh

@@ -0,0 +1 @@
+tuning/run_cnn_e2eali_1a.sh

egs/rimes/v1/local/chain/run_e2e_cnn.sh

@@ -0,0 +1 @@
+tuning/run_e2e_cnn_1a.sh