[egs,scripts] Byte-pair encoding (BPE) applied MADCAT Arabic OCR (#2434)

Author: babakrkb

egs/madcat_ar/v1/local/prepare_dict.sh

@@ -12,7 +12,7 @@ mkdir -p $dir
 
 local/prepare_lexicon.py $dir
 
-cut -d' ' -f2- $dir/lexicon.txt | tr ' ' '\n' | sort -u >$dir/nonsilence_phones.txt || exit 1;
+cut -d' ' -f2- $dir/lexicon.txt | sed 's/SIL//g' | tr ' ' '\n' | sort -u | sed '/^$/d' >$dir/nonsilence_phones.txt || exit 1;
 
 echo '<sil> SIL' >> $dir/lexicon.txt
 

egs/madcat_ar/v1/local/prepare_lexicon.py

@@ -19,7 +19,8 @@
  line_vect = line.strip().split(' ')
  for i in range(1, len(line_vect)):
  characters = list(line_vect[i])
- characters = " ".join(characters)
+ # Put SIL instead of "|". Because every "|" in the beginning of the words is for initial-space of that word
+ characters = " ".join([ 'SIL' if char == '|' else char for char in characters])
  lex[line_vect[i]] = characters
  if line_vect[i] == '#':
  lex[line_vect[i]] = "<HASH>"

egs/madcat_ar/v1/local/prepend_words.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This script, prepend '|' to every words in the transcript to mark
+# the beginning of the words for finding the initial-space of every word
+# after decoding.
+
+import sys, io
+
+infile = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
+output = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+for line in infile:
+ output.write(' '.join([ "|"+word for word in line.split()]) + '\n')
+
+

egs/madcat_ar/v1/local/reverse.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# This script, reverse all latin and digits sequences
+# (including words like MP3) to put them in the right order in the images. 
+
+import re, os, sys, io
+
+in_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
+out_stream = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+for line in in_stream:
+ out_stream.write( re.sub(r'[a-zA-Z0-9][a-zA-Z0-9\s\.\:]*[a-zA-Z0-9]' , lambda m:m.group(0)[::-1] , line ))
+

egs/madcat_ar/v1/local/wer_output_filter

@@ -12,7 +12,7 @@ binmode STDERR, ":utf8";
 # Arabic-specific normalization
 while (<>) {
  @F = split " ";
- print $F[0] . " ";
+ print $F[0];
  foreach $s (@F[1..$#F]) {
  $s =~ s/\x{0623}/\x{0627}/g;
  $s =~ s/\x{0625}/\x{0627}/g;
@@ -34,12 +34,12 @@ while (<>) {
  $s =~ s/\x{0621}//g;
  $s =~ s/[\x{064b}-\x{0655}]//g;
  $s =~ s/\x{0640}//g;
+ $s =~ s/\|/ /g;
  if ($s ne "") {
  print "$s";
  } else {
  print "";
  }
- print " ";
  }
  print "\n";
 }

egs/madcat_ar/v1/run_end2end.sh

@@ -80,9 +80,19 @@ fi
 
 if [ $stage -le 5 ]; then
  echo "$0: Preparing dictionary and lang..."
+ cut -d' ' -f2- data/train/text | python3 local/reverse.py | python3 local/prepend_words.py | python3 utils/lang/bpe/learn_bpe.py -s 700 > data/train/bpe.out
+ for set in test train dev ; do
+ cut -d' ' -f1 data/$set/text > data/$set/ids
+ cut -d' ' -f2- data/$set/text | python3 local/reverse.py | python3 local/prepend_words.py | python3 utils/lang/bpe/apply_bpe.py -c data/train/bpe.out | sed 's/@@//g' > data/$set/bpe_text
+ mv data/$set/text data/$set/text.old
+ paste -d' ' data/$set/ids data/$set/bpe_text > data/$set/text
+ done
  local/prepare_dict.sh
- utils/prepare_lang.sh --num-sil-states 4 --num-nonsil-states 8 --sil-prob 0.9999 \
+ # This recipe uses byte-pair encoding, the silences are part of the words' pronunciations.
+ # So we set --sil-prob to 0.0
+ utils/prepare_lang.sh --num-sil-states 4 --num-nonsil-states 8 --sil-prob 0.0 --position-dependent-phones false \
  data/local/dict "<sil>" data/lang/temp data/lang
+ utils/lang/bpe/add_final_optional_silence.sh --final-sil-prob 0.5 data/lang
 fi
 
 if [ $stage -le 6 ]; then

egs/wsj/s5/utils/lang/bpe/add_final_optional_silence.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+. ./path.sh
+
+final_sil_prob=0.5
+
+echo "$0 $@" # Print the command line for logging
+
+. utils/parse_options.sh
+
+if [ $# -ne 1 ]; then
+ echo "Usage: $0 <lang>"
+ echo " Add final optional silence to lexicon FSTs (L.fst and L_disambig.fst) in "
+ echo " lang/ directory <lang>."
+ echo " This can be useful in systems with byte-pair encoded (BPE) lexicons, in which"
+ echo " the word-initial silence is part of the lexicon, so we turn off the standard"
+ echo " optional silence in the lexicon"
+ echo "options:"
+ echo " --final-sil-prob <final silence probability>	# default 0.5"
+ exit 1;
+fi
+
+lang=$1
+
+if [ $lang/phones/final_sil_prob -nt $lang/phones/nonsilence.txt ]; then
+ echo "$0 $lang/phones/final_sil_prob exists. Exiting..."
+ exit 1;
+fi
+
+sil_eq_zero=$(echo $(perl -e "if ( $final_sil_prob == 0.0) {print 'true';} else {print 'false';}"))
+sil_eq_one=$(echo $(perl -e "if ( $final_sil_prob == 1.0) {print 'true';} else {print 'false';}"))
+sil_lt_zero=$(echo $(perl -e "if ( $final_sil_prob < 0.0) {print 'true';} else {print 'false';}"))
+sil_gt_one=$(echo $(perl -e "if ( $final_sil_prob > 1.0) {print 'true';} else {print 'false';}"))
+
+if $sil_lt_zero || $sil_gt_one; then
+ echo "$0 final-sil-prob should be between 0.0 and 1.0. Final silence was not added."
+ exit 1;
+else
+ if $sil_eq_zero; then
+ echo "$0 final-sil-prob = 0 => Final silence was not added."
+ exit 0;
+ elif $sil_eq_one; then
+ echo -e "0\t1\t1\t0\n1" | fstcompile > $lang/final_sil.fst
+ else
+ log_silprob=$(echo $(perl -e "print log $final_sil_prob"))
+ echo -e "0\t1\t1\t0\t$log_silprob\n0\t$log_silprob\n1\t0.0" | fstcompile > $lang/final_sil.fst
+ fi
+ mv $lang/L.fst $lang/L.fst.orig
+ mv $lang/L_disambig.fst $lang/L_disambig.fst.orig
+ fstconcat $lang/L.fst.orig $lang/final_sil.fst | fstarcsort --sort_type=olabel > $lang/L.fst
+ fstconcat $lang/L_disambig.fst.orig $lang/final_sil.fst | fstarcsort --sort_type=olabel > $lang/L_disambig.fst
+ echo "$final_sil_prob" > $lang/phones/final_sil_prob
+fi
+