Updated mnist script.

Author: mandar2812

scripts/mnist.sc

@@ -11,7 +11,7 @@
 
  val dataSet = MNISTLoader.load(Paths.get(tempdir.toString()), MNISTLoader.MNIST)
 
- val dtf_cifar_data = dtfdata.tf_dataset(
+ val dtf_mnist_data = dtfdata.tf_dataset(
  dtfdata.supervised_dataset(
  dataSet.trainImages.unstack(axis = 0),
  dataSet.trainLabels.castTo[Long].unstack(axis = -1)),
@@ -47,16 +47,13 @@
  loss
  )
 
- val data_ops = dtflearn.model.data_ops(
+ val data_ops = dtflearn.model.data_ops[(Output[UByte], Output[Long])](
  shuffleBuffer = 5000,
  batchSize = 128,
- prefetchSize = 10,
- concatOpI = Some(dtfpipe.EagerStack[UByte]()),
- concatOpT = Some(dtfpipe.EagerStack[Long]()),
- concatOpO = Some(dtfpipe.EagerConcatenate[Float]())
+ prefetchSize = 10
  )
 
- val config = dtflearn.model.trainConfig(
+ val train_config = dtflearn.model.trainConfig(
  tempdir/"mnist_summaries",
  data_ops,
  optimizer,
@@ -69,9 +66,43 @@
  checkPointFreq = 100)
  ))
 
+ val pattern_to_tensor =
+ DataPipe[Seq[(Tensor[UByte], Tensor[Long])], (Tensor[UByte], Tensor[Long])](
+ ds => {
+ val (xs, ys) = ds.unzip
+
+ (
+ dtfpipe.EagerStack[UByte](axis = 0).run(xs),
+ dtfpipe.EagerStack[Long](axis = 0).run(ys)
+ )
+ }
+ )
+
+ val data_handle_ops = dtflearn.model.tf_data_handle_ops[
+ (Tensor[UByte], Tensor[Long]),
+ (Tensor[UByte], Tensor[Long]),
+ Tensor[Float],
+ (Output[UByte], Output[Long])
+ ](
+ bufferSize = 500,
+ patternToTensor = Some(pattern_to_tensor),
+ concatOpO = Some(dtfpipe.EagerConcatenate[Float]())
+ )
 
-
- mnist_model.train(dtf_cifar_data.training_dataset, config)
+ val data_handle_ops_infer =
+ dtflearn.model.tf_data_handle_ops[Tensor[UByte], Tensor[UByte], Tensor[
+ Float
+ ], Output[UByte]](
+ bufferSize = 1000,
+ patternToTensor = Some(dtfpipe.EagerStack[UByte](axis = 0)),
+ concatOpO = Some(dtfpipe.EagerConcatenate[Float]())
+ )
+
+ mnist_model.train(
+ dtf_mnist_data.training_dataset,
+ train_config,
+ data_handle_ops
+ )
 
  def accuracy(predictions: Tensor[Long], labels: Tensor[Long]): Float =
  tfi.equal(predictions.argmax[Long](1), labels)
@@ -81,13 +112,26 @@
  .asInstanceOf[Float]
 
  val (trainingPreds, testPreds): (Tensor[Float], Tensor[Float]) = (
- mnist_model.infer_batch(dtf_cifar_data.training_dataset.map(p => p._1), data_ops).left.get,
- mnist_model.infer_batch(dtf_cifar_data.test_dataset.map(p => p._1), data_ops).left.get
+ mnist_model
+ .infer_batch(
+ dtf_mnist_data.training_dataset.map(p => p._1),
+ data_handle_ops_infer
+ )
+ .left
+ .get,
+ mnist_model
+ .infer_batch(
+ dtf_mnist_data.test_dataset.map(p => p._1),
+ data_handle_ops_infer
+ )
+ .left
+ .get
  )
 
  val (trainAccuracy, testAccuracy) = (
  accuracy(trainingPreds.castTo[Long], dataSet.trainLabels.castTo[Long]),
- accuracy(testPreds.castTo[Long], dataSet.testLabels.castTo[Long]))
+ accuracy(testPreds.castTo[Long], dataSet.testLabels.castTo[Long])
+ )
 
  print("Train accuracy = ")
  pprint.pprintln(trainAccuracy)