Fix correlation issues (#691)

* add correlation CI * fix building error and submit correlation.conf Signed-off-by: minmingzhu <minmingzhu@intel.com> * Update correlation.conf Co-authored-by: minmingzhu <minmingzhu@intel.com>

Author: minmingzhu

bin/run_all.sh

@@ -89,6 +89,9 @@ for benchmark in `cat $root_dir/conf/benchmarks.lst`; do
  if [ $benchmark == "ml/gmm" ] && [ $framework == "hadoop" ]; then
  continue
  fi
+ if [ $benchmark == "ml/correlation" ] && [ $framework == "hadoop" ]; then
+ continue
+ fi
 
  echo -e "${UYellow}${BYellow}Run ${Yellow}${UYellow}${benchmark}/${framework}${Color_Off}"
  echo -e "${BCyan}Exec script: ${Cyan}$WORKLOAD/${framework}/run.sh${Color_Off}"

conf/workloads/ml/correlation.conf

@@ -0,0 +1,26 @@
+hibench.correlation.tiny.examples 50000
+hibench.correlation.tiny.features 10000
+
+hibench.correlation.small.examples 100000
+hibench.correlation.small.features 20000
+
+hibench.correlation.large.examples 200000
+hibench.correlation.large.features 30000
+
+hibench.correlation.huge.examples 300000
+hibench.correlation.huge.features 50000
+
+hibench.correlation.gigantic.examples 500000
+hibench.correlation.gigantic.features 80000
+
+hibench.correlation.bigdata.examples 2000000
+hibench.correlation.bigdata.features 20000
+
+hibench.correlation.corrType pearson
+
+hibench.correlation.examples ${hibench.correlation.${hibench.scale.profile}.examples}
+hibench.correlation.features ${hibench.correlation.${hibench.scale.profile}.features}
+hibench.correlation.partitions ${hibench.default.map.parallelism}
+
+hibench.workload.input ${hibench.hdfs.data.dir}/Correlation/Input
+hibench.workload.output ${hibench.hdfs.data.dir}/Correlation/Output

conf/workloads/ml/correlation.info

@@ -1,18 +1,17 @@
 package com.intel.hibench.sparkbench.ml
 
 import org.apache.spark.ml.feature.LabeledPoint
-import scopt.OptionParser
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.ml.linalg.{Matrix, Vector}
 import org.apache.spark.ml.stat.Correlation
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{Row, SparkSession}
+import scopt.OptionParser
 
 object CorrelationExample {
 
  case class Params(
- input: String = null,
- corrType: String = "pearson",
- )
+ input: String = null,
+ corrType: String = "pearson")
 
  def main(args: Array[String]): Unit = {
 
@@ -46,20 +45,17 @@ object CorrelationExample {
  import spark.implicits._
  val training = data.toDF().cache()
 
-
  val numTraining = training.count()
 
  val numFeatures = training.select("features").first().getAs[Vector](0).size
  println(s" numTraining = $numTraining")
  println(s" numFeatures = $numFeatures")
 
-
  println(s"Correlation ${params.corrType} between label and each feature")
  val df = training.toDF("label", "features")
  val Row(coeff1: Matrix) = Correlation.corr(df, "features", params.corrType).head()
  println(s"Pearson correlation matrix:\n $coeff1.")
 
  spark.stop()
  }
-
 }