• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• spark-dev-master
  • src
    • main
      • scala
        • examples
          • sql
            • TestParquet.scala
            • TestDataset.scala
          • PurchaseLogAnalysis.scala
          • TestJoins.scala
          • graphx
            • EmployeeRelationship.scala
          • TestPartition.scala
          • TestBroadcastVariables.scala
          • TestAccumulators.scala
          • streaming
            • TestStreamingListener.scala
            • TestReduceByKeyAndWindow.scala
            • TestMapWithState.scala
            • TestVanillaStreaming.scala
            • TestWindowing.scala
            • TestAdditionInWindow.scala
            • TestUpdateStateByKey.scala
            • WordCountWithSQL.scala
          • WordCount.scala
          • TestValueTransformations.scala
          • TestCombineByKey.scala
      • python
        • sql
          • ds_test.py
          • join_test.py
          • parquet
            • parquet_converter.py
            • parquet_test.py
          • api_test.py
        • key_summary
          • keysummary.py
          • keysummary.sh
          • README.md
        • streamtest.py
        • sparktest.py
        • twifeeds.py
        • stream_list_test.py
        • dstream_rdd.py
        • pysparktest
          • pysparktest.py
          • __init__.py
  • LICENSE
  • test
    • python
      • sample_test.py
      • sql_test.py
      • pysparktestcase_test.py
  • data
    • purchases.log
    • store-sales.log
    • countries.csv
    • sample.log
  • README.md
  • build.sbt

package examples.streaming

import org.apache.spark.streaming.StreamingContext
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{ Seconds, State, StateSpec }

/**
 * Listens to socket text stream on host=localhost, port=9999.
 * Tokenizes the incoming stream into (words, no. of occurrences) and tracks the state
 * of the word using the 1.6 API 'mapWithState'.
 * Keys with no updates are removed using StateSpec.timeout API.
 * Checkpoint dir created in HDFS.
 * Chekpointing frequency every 20s.
 */

object TestMapWithState {
  val checkpointDir: String = "hdfs://localhost:9000/user/hduser/spark-chkpt"

  def main(args: Array[String]): Unit = {
    val ssc = StreamingContext.getOrCreate(checkpointDir, createFunc)

    ssc.start()
    ssc.awaitTermination()
  }

  def createFunc(): StreamingContext = {
    val ssc = new StreamingContext(new SparkConf().setAppName("TestMapWithStateJob"),
      Seconds(5))

    ssc.checkpoint(checkpointDir)

    // State specs
    val stateSpec = StateSpec.function(mappingFunc _)
      .numPartitions(4)
      .timeout(Seconds(30)) // idle keys will be removed.

    ssc.socketTextStream("localhost", "9999".toInt)
      .flatMap(_.split(" "))
      .map((_, 1))
      .mapWithState(stateSpec)
      .checkpoint(Seconds(20))
      .print()

    ssc
  }

  /**
   * Mapping function for the 'mapWithState' API.
   */
  def mappingFunc(key: String, value: Option[Int], state: State[Int]): Option[(String, Int)] = {
    val sum = value.getOrElse(0) + state.getOption().getOrElse(0)

    // updating the state of non-idle keys...
    // To call State.update(...) we need to check State.isTimingOut() == false, 
    // else there will be NoSuchElementException("Cannot update the state that is timing out")
    if (state.isTimingOut())
      println(key + " key is timing out...will be removed.")
    else
      state.update(sum)

    Some((key, sum))
  }
}