update breeze test

Author: delphi-xk

src/main/scala/com/hyzs/spark/ml/MatrixOpsInSpark.scala

@@ -14,6 +14,7 @@ import org.apache.spark.ml.classification.{BinaryLogisticRegressionSummary, Logi
  */
 object MatrixOpsInSpark {
 
+ import spark.implicits._
 
  def importDataset(): Unit ={
  val data: Dataset[Row] = spark.read
@@ -62,6 +63,8 @@ object MatrixOpsInSpark {
  println(s"Coefficients: ${lrModel.coefficients} Intercept: ${lrModel.intercept}")
 
  val trainingSummary = lrModel.summary
+ println(s"training summary objectiveHistory: ${trainingSummary.objectiveHistory.mkString(",")}")
+ println(s"training summary totalIterations: ${trainingSummary.totalIterations}")
  val binarySummary = trainingSummary.asInstanceOf[BinaryLogisticRegressionSummary]
  val roc = binarySummary.roc
  roc.show()

src/main/scala/com/hyzs/spark/utils/SparkUtils.scala

@@ -37,8 +37,6 @@ object SparkUtils {
  // NOTE: not serializable, cannot initialize class.
  //mapper.registerModule(DefaultScalaModule)
  val broadMapper: Broadcast[ObjectMapper] = sc.broadcast(mapper)
- import spark.implicits._
- import spark.sqlContext.implicits._
 
  def checkHDFileExist(filePath: String): Boolean = {
  val path = new Path(filePath)

src/test/scala/BreezeTest.scala

@@ -0,0 +1,47 @@
+import org.scalatest.FunSuite
+
+/**
+ * Created by xk on 2018/6/1.
+ */
+
+import breeze.linalg._
+import breeze.numerics._
+import breeze.stats.distributions._
+import breeze.stats.DescriptiveStats._
+import breeze.stats._
+
+class BreezeTest extends FunSuite{
+
+
+
+ test("test breeze algebra"){
+ val v1 = DenseVector.zeros[Double](5)
+ println(v1)
+ v1(1 to 2) := .5
+ println(v1)
+ v1(0 to 1) := DenseVector(.1,.2)
+ println(v1)
+ val m = DenseMatrix.zeros[Int](5,5)
+
+ val v2 = DenseVector(.1, .2, .3, .4, .5)
+ val m2 = DenseMatrix(v1, v2)
+ m2(0 to 1, 0 to 2) := DenseMatrix((1.1,1.2,1.3),(2.1,2.2,2.3))
+
+ }
+
+ test("test distribution"){
+ val gaussian = Gaussian(0.0, 1.0)
+ val g_samples = gaussian.sample(100)
+ println(mean(g_samples))
+ println(meanAndVariance(g_samples))
+ println(stddev(g_samples))
+ val poi = Poisson(3.0)
+ val p_sample = poi.sample(50)
+ println(p_sample)
+
+ val gau_matrix = new DenseMatrix(100, 10, gaussian.sample(1000).toArray)
+ println(gau_matrix)
+ }
+
+
+}