• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• spark-dev-master
  • src
    • main
      • scala
        • examples
          • sql
            • TestParquet.scala
            • TestDataset.scala
          • PurchaseLogAnalysis.scala
          • TestJoins.scala
          • graphx
            • EmployeeRelationship.scala
          • TestPartition.scala
          • TestBroadcastVariables.scala
          • TestAccumulators.scala
          • streaming
            • TestStreamingListener.scala
            • TestReduceByKeyAndWindow.scala
            • TestMapWithState.scala
            • TestVanillaStreaming.scala
            • TestWindowing.scala
            • TestAdditionInWindow.scala
            • TestUpdateStateByKey.scala
            • WordCountWithSQL.scala
          • WordCount.scala
          • TestValueTransformations.scala
          • TestCombineByKey.scala
      • python
        • sql
          • ds_test.py
          • join_test.py
          • parquet
            • parquet_converter.py
            • parquet_test.py
          • api_test.py
        • key_summary
          • keysummary.py
          • keysummary.sh
          • README.md
        • streamtest.py
        • sparktest.py
        • twifeeds.py
        • stream_list_test.py
        • dstream_rdd.py
        • pysparktest
          • pysparktest.py
          • __init__.py
  • LICENSE
  • test
    • python
      • sample_test.py
      • sql_test.py
      • pysparktestcase_test.py
  • data
    • purchases.log
    • store-sales.log
    • countries.csv
    • sample.log
  • README.md
  • build.sbt

package examples.streaming

/**
 * Using org.apache.spark.streaming.scheduler.StreamListerner
 * trait for listening to streams
 */
import org.apache.spark.streaming.{ StreamingContext, Seconds }
import org.apache.spark.streaming.scheduler.{
	StreamingListener,
	StreamingListenerBatchStarted,
	StreamingListenerBatchCompleted
}
import org.apache.spark.SparkConf

object TestStreamingListener {
	def main(args: Array[String]): Unit = {

		val ssc = new StreamingContext(new SparkConf().setAppName("TestStreamingListenerJob"),
			Seconds(5))

		ssc.addStreamingListener(new MyStreamingListener())

		ssc
			.socketTextStream("localhost", 9999)
			.flatMap(_.split(" "))
			.count()
			.print()

		ssc.start()
		ssc.awaitTermination()
	}
}

class MyStreamingListener extends StreamingListener {

	override def onBatchStarted(batchStarted: StreamingListenerBatchStarted): Unit = {
		println(">>> Batch started...records in batch = " + batchStarted.batchInfo.numRecords)
	}

	override def onBatchCompleted(batchCompleted: StreamingListenerBatchCompleted): Unit = {
		println(">>> Batch completed...time taken (ms) = " + batchCompleted.batchInfo.totalDelay)
	}
}