• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• spark-dev-master
  • src
    • main
      • scala
        • examples
          • sql
            • TestParquet.scala
            • TestDataset.scala
          • PurchaseLogAnalysis.scala
          • TestJoins.scala
          • graphx
            • EmployeeRelationship.scala
          • TestPartition.scala
          • TestBroadcastVariables.scala
          • TestAccumulators.scala
          • streaming
            • TestStreamingListener.scala
            • TestReduceByKeyAndWindow.scala
            • TestMapWithState.scala
            • TestVanillaStreaming.scala
            • TestWindowing.scala
            • TestAdditionInWindow.scala
            • TestUpdateStateByKey.scala
            • WordCountWithSQL.scala
          • WordCount.scala
          • TestValueTransformations.scala
          • TestCombineByKey.scala
      • python
        • sql
          • ds_test.py
          • join_test.py
          • parquet
            • parquet_converter.py
            • parquet_test.py
          • api_test.py
        • key_summary
          • keysummary.py
          • keysummary.sh
          • README.md
        • streamtest.py
        • sparktest.py
        • twifeeds.py
        • stream_list_test.py
        • dstream_rdd.py
        • pysparktest
          • pysparktest.py
          • __init__.py
  • LICENSE
  • test
    • python
      • sample_test.py
      • sql_test.py
      • pysparktestcase_test.py
  • data
    • purchases.log
    • store-sales.log
    • countries.csv
    • sample.log
  • README.md
  • build.sbt

package examples.sql

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.desc

/**
 * Porting /src/main/python/sql/parquet/parquet_test.py
 * to Scala
 */
case class Purchase(date: String, time: String,
	city: String, category: String, amount: Double, method: String)

object TestParquet {
	def main(args: Array[String]): Unit = {

		if (args.length == 0) {
			println("Error: Please mention the parquet file...")
			sys.exit(-1)
		}

		val spark = SparkSession
			.builder()
			.master("local[4]")
			.appName("TestParquet")
			.config(conf = new SparkConf())
			.getOrCreate()

		import spark.implicits._

		val ds = spark.read.parquet(args(0)).as[Purchase]

		ds.printSchema()

		println("Number of Method=\"Discover\": "
			+ ds.filter(data => data.method == "Discover").count())

		// Top 3 methods...
		println(">>> Top 3 Methods using Dataframe API >>>")

		ds.groupBy("Method")
			.sum("Amount")
			.withColumnRenamed("sum(Amount)", "Total")
			.orderBy(desc("Total"))
			.take(3)
			.foreach(println)

		println(">>> Top 3 Methods using SQL >>>")

		ds.createOrReplaceTempView("temp")
		val sqlStr = "select Method, sum(Amount) as Total from temp group by Method order by Total desc"

		spark
			.sql(sqlStr)
			.take(3)
			.foreach(println)
	}
}