Merge pull request #235 from didi/docker

ci and delta using tf2.3.0

Author: zh794390558

.travis.yml

@@ -6,7 +6,7 @@ services:
 before_install:
  - export DELTA_PATH=`pwd`; echo $DELTA_PATH
  - export DOCKER_DELTA="/home/gitlab-runner/delta"; echo $DOCKER_DELTA
- - export CI_IMAGE=zh794390558/delta:ci-cpu-py3
+ - export CI_IMAGE=zh794390558/delta:2.3.0-ci-cpu-py3
  - docker pull ${CI_IMAGE}
  - docker run -it -d --name travis_con --user root -v ${DELTA_PATH}:${DOCKER_DELTA} ${CI_IMAGE} bash
  - docker exec travis_con bash -c "gcc -v && g++ -v"

README.md

@@ -118,9 +118,9 @@ To verify the installation, run:
 
 ```shell
 # Activate conda environment
-conda activate delta-py3.6-tf2.0.0
+conda activate delta-py3.6-tf2.3.0
 # Or use the following command if your conda version is < 4.6
-# source activate delta-py3.6-tf2.0.0
+# source activate delta-py3.6-tf2.3.0
 
 # Add DELTA environment
 source env.sh

core/ops/Makefile

@@ -8,7 +8,7 @@ endif
 #$(info $(MAKEFILE_DIR))
 #$(info $(MAIN_ROOT))
 
-CXX := g++ 
+CXX := g++-7
 NVCC := nvcc
 PYTHON_BIN_PATH= python3
 CC :=

delta/data/feat/speech_ops_test.py

@@ -209,13 +209,13 @@ def test_splice(self):
  self.assertTupleEqual(out.eval().shape, (1, 3, 2 * ctx))
  self.assertAllEqual(out, tf.ones([1, 3, 2 * ctx]))
 
- with self.assertRaises(ValueError):
+ with self.assertRaises(tf.errors.InvalidArgumentError):
  out = tffeat.splice(feat, left_context=-2, right_context=-2).eval()
 
- with self.assertRaises(ValueError):
+ with self.assertRaises(tf.errors.InvalidArgumentError):
  out = tffeat.splice(feat, left_context=2, right_context=-2).eval()
 
- with self.assertRaises(ValueError):
+ with self.assertRaises(tf.errors.InvalidArgumentError):
  out = tffeat.splice(feat, left_context=-2, right_context=2).eval()
 
 

delta/data/preprocess/base_preparer.py

@@ -144,7 +144,7 @@ def prepare_one_raw_data(self, one_path, one_path_after, mode,
  if self.multi_output:
  for i in range(self.output_num):
  label_ds = label[i].batch(self.batch_size)
- label_iterator = label_ds.make_initializable_iterator()
+ label_iterator = tf.data.make_initializable_iterator(label_ds)
  label_after_arr = self.run_dataset(label_iterator, batch_num)
  label_after_one = [
  one_line.decode("utf-8") for one_line in label_after_arr
@@ -154,7 +154,7 @@ def prepare_one_raw_data(self, one_path, one_path_after, mode,
  else:
  label = label[0]
  label_ds = label.batch(self.batch_size)
- label_iterator = label_ds.make_initializable_iterator()
+ label_iterator = tf.data.make_initializable_iterator(label_ds)
  label_after_arr = self.run_dataset(label_iterator, batch_num)
  one_label_after = [
  one_line.decode("utf-8") for one_line in label_after_arr

delta/data/preprocess/text_ops.py

@@ -101,7 +101,7 @@ def load_textline_dataset(paths, column_num):
  """Load raw data for text task."""
  ds = tf.data.TextLineDataset(paths)
  ds = ds.map(
-  lambda x: tf.strings.split(x, sep="\t", result_type="RaggedTensor"))
+ lambda x: tf.squeeze(tf.strings.split(x, sep="\t", result_type="RaggedTensor"), axis = 0))
  ds = ds.filter(lambda line: tf.equal(tf.size(line), column_num))
  ds_list = []
  for i in range(column_num):

delta/data/preprocess/text_ops_test.py

@@ -121,7 +121,7 @@ def test_process_one_label_dataset(self):
  true_res = [0, 0, 0, 8]
  label_ds = process_one_label_dataset(label_ds, self.config)
 
- iterator = label_ds.make_initializable_iterator()
+ iterator = tf.data.make_initializable_iterator(label_ds)
  label_res = iterator.get_next()
 
  with tf.Session() as sess:
@@ -139,7 +139,7 @@ def test_process_multi_label_dataset(self):
  label_ds = tf.data.TextLineDataset(label_filepath)
  true_res = [[0, 8, 8], [0, 7, 8]]
  label_ds = process_multi_label_dataset(label_ds, self.config)
- iterator = label_ds.make_initializable_iterator()
+ iterator = tf.data.make_initializable_iterator(label_ds)
  label_res = iterator.get_next()
 
  with tf.Session() as sess:

delta/data/preprocess/utils.py

@@ -37,7 +37,7 @@ def get_pre_process_text_ds_iter(
 
  text_ds = text_ds.batch(batch_size)
 
- iterator = text_ds.make_initializable_iterator()
+ iterator = tf.data.make_initializable_iterator(text_ds)
 
  return iterator
 

delta/data/task/text_cls_task.py

@@ -184,7 +184,7 @@ def dataset(self):
 
  data_set = data_set.prefetch(self.num_prefetch_batch)
 
- iterator = data_set.make_initializable_iterator()
+ iterator = tf.data.make_initializable_iterator(data_set)
 
  # pylint: disable=unused-variable
  if self.infer_without_label:

delta/data/task/text_match_task.py

@@ -170,7 +170,7 @@ def dataset(self):
 
  text_ds_left_right = text_ds_left_right.prefetch(self.num_prefetch_batch)
 
- iterator = text_ds_left_right.make_initializable_iterator()
+ iterator = tf.data.make_initializable_iterator(text_ds_left_right)
  # pylint: disable=unused-variable
  if self.infer_without_label:
  (input_x_left, input_x_right), (input_x_left_len,