• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• Apache-Spark-2x-Machine-Learning-Cookbook-master
  • Chapter08
    • Chapter 8 Data
      • my_kmeans_data.txt
      • my_lda_data.txt
      • trainingDir
        • trainingKStreaming.txt
      • glass.scale
      • socr_data.txt
      • testDir
        • testKStreaming.txt
    • Chapter 8 Code
      • MyGaussianMixture.scala
      • MyLDA.scala
      • MyBisectingKMeans.scala
      • MyPowerIterationClustering.scala
      • MyKMeansCluster.scala
      • MyStreamingKMeans.scala
  • Chapter13
    • Chapter 13 Data
      • GE.csv
      • iris.data
      • person.json
      • pima-indians-diabetes.data
    • Chapter 13 Code
      • DatasetStreamCSV.scala
      • CountStreamProducer.scala
      • KMeansStreaming.scala
      • VoteCountStream.scala
      • ClickStream.scala
      • RegressionStreaming.scala
      • IrisData.scala
      • LogisticStreaming.scala
      • DataFrameStream.scala
      • ClickGenerator.scala
  • LICENSE
  • Chapter01
    • Screencast for setup
      • ApacheSparkMachineLearingCookbook-GettingStarted.pptx
    • Chapter 1 Code
      • MyFirstSpark20.scala
      • MyChart.scala
  • Chapter12
    • Chapter 12 Code
      • ProcessLDA20.scala
      • ProcessWordCount20.scala
      • ProcessLSA20.scala
      • ProcessWord2Vec20.scala
    • Chapter 12 Data
      • stopwords.txt
  • .gitattributes
  • Chapter10
    • Chapter 10 Data
      • breast-cancer-wisconsin.data
    • Chapter 10 Code
      • MyRandomForestClassification.scala
      • MyDecisionTreeRegression.scala
      • MyGradientBoostingClassification.scala
      • MyRandomForestRegression.scala
      • MyDecisionTreeClassification.scala
      • MyGradientBoostingRegression.scala
  • README.md
  • Chapter02
    • Chapter 2 Code
      • MyVectorMatrix.scala
  • Chapter05
    • Chapter 5 Data
      • hmohiv.csv
      • iris.scale.txt
      • housing8.csv
    • Chapter 5 Code
      • MyRegress01_20.scala
      • OnevsRest.scala
      • MyAFTSurvivalRegression.scala
      • MyRegress02_20.scala
      • MLP.scala
      • GLR.scala
      • MyRegress03_20.scala
      • MyRegress04_20.scala
      • MyIsotonicRegress.scala
  • Chapter03
    • Chapter 3 Code
      • MyRDD.scala
      • MyDataFrame.scala
      • MyDatasetSeq.scala
      • MyDatasetFunc.scala
      • MyDatasetRDD.scala
      • MyDatasetJson.scala
      • MyDatasetData.scala
    • Chapter 3 Data
      • customers13.txt
      • cars.json
      • products13.txt
      • sales13.txt
  • Chapter07
    • Chapter 7 Code
      • MovieData.scala
      • MovieLens.scala
      • RatingsData.scala
    • Chapter 7 Data
      • README
      • users.dat
      • movies.dat
  • Chapter11
    • Chapter 11 Data
      • processed.cleveland.data
    • Chapter 11 Code
      • MySVD.scala
      • MyCSV.scala
      • MyPCA.scala
  • .gitignore
  • Chapter04
    • Chapter 4 Code
      • MyPMMLExport.scala
      • MyMultiLabel.scala
      • MyRegressionMetrics.scala
      • MyBinaryClassification.scala
      • MyAccessSparkCluster20.scala
      • MyDataset.scala
      • DatasetvsRDD.scala
      • MyPipeLine.scala
      • MySummaryStats.scala
      • MyDataRDD.scala
      • MyAccessSparkClusterPre20.scala
      • MyNormalize.scala
      • MyDataSplit.scala
      • MyLabeledPoint.scala
      • MyMulticlass.scala
      • SessionvsContext.scala
      • MyBreezeChart.scala
    • Chapter 4 Data
      • breast-cancer-wisconsin.data
      • teams.csv
      • mySampleCSV.csv
      • my_kmeans_data_sample.txt
      • wine.data
      • beatles.txt
      • myMulticlassIrisData.txt
  • Chapter06
    • Chapter 6 Data
      • sample_libsvm_data.txt
      • iris.data.prepared.txt
      • admission1.csv
      • housing8.csv
    • Chapter 6 Code
      • MyRegress02.scala
      • MyNaiveBayes08.scala
      • MyRegress06.scala
      • MyRegress04.scala
      • MyRegress03.scala
      • MySVM07.scala
      • MyFirstLogistic.scala
      • MyRegress05.scala
  • Chapter09
    • Chapter 9 Data
      • Year_Salary.csv
      • housing8.csv
    • Chapter 9 Code
      • MyGradient.scala
      • MyGDLinearRegression.scala
      • MyRegressNormal.scala

package spark.ml.cookbook.chapter13

import org.apache.log4j.{Level, Logger}
import org.apache.spark.mllib.classification.StreamingLogisticRegressionWithSGD
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.StreamingLinearRegressionWithSGD
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.streaming.{Seconds, StreamingContext}

import scala.collection.mutable.Queue


object LogisticStreaming {

  def main(args: Array[String]) {

    Logger.getLogger("org").setLevel(Level.ERROR)
    Logger.getRootLogger.setLevel(Level.WARN)

    val spark = SparkSession
      .builder
      .master("local[*]")
      .appName("Logistic Streaming App")
      .config("spark.sql.warehouse.dir", ".")
      .getOrCreate()

    import spark.implicits._

    val ssc = new StreamingContext(spark.sparkContext, Seconds(2))

    val rawDF = spark.read
      .text("../data/sparkml2/chapter13/pima-indians-diabetes.data").as[String]

    val buf = rawDF.rdd.map(value => {
      val data = value.split(",")
      (data.init.toSeq, data.last)
    })

    val lps = buf.map{ case (feature: Seq[String], label: String) =>
      val featureVector = feature.map(_.toDouble).toArray[Double]
      LabeledPoint(label.toDouble, Vectors.dense(featureVector))
    }

    val trainQueue = new Queue[RDD[LabeledPoint]]()
    val testQueue = new Queue[RDD[LabeledPoint]]()

    val trainingStream = ssc.queueStream(trainQueue)
    val testStream = ssc.queueStream(testQueue)

    val numFeatures = 8
    val model = new StreamingLogisticRegressionWithSGD()
      .setInitialWeights(Vectors.zeros(numFeatures))
      .setNumIterations(15)
      .setStepSize(0.5)
      .setMiniBatchFraction(0.25)

    model.trainOn(trainingStream)
    val result = model.predictOnValues(testStream.map(lp => (lp.label, lp.features)))

    result.map{ case (label: Double, prediction: Double) =>  (label, prediction) }.print()

    ssc.start()

    val Array(trainData, test) = lps.randomSplit(Array(.80, .20))

    trainQueue +=  trainData
    Thread.sleep(4000)

    val testGroups = test.randomSplit(Array(.50, .50))
    testGroups.foreach(group => {
      testQueue += group
      Thread.sleep(2000)
    })

    ssc.stop()
  }
}