• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• Hands-On-Big-Data-Analytics-with-PySpark-master
  • Chapter08
    • ImmutableDataSet.scala
    • ImmutableRDD.scala
    • InheritanceRdd.scala
    • ImmutableDataFrame.scala
    • MultithreadedImmutabilityTest.scala
  • Chapter13
    • PageRankTest.scala
    • VertexAPI.scala
    • EdgeAPI.scala
    • GraphBuilder.scala
    • CreatingGraph.scala
    • CalculateDegreeTest.scala
  • LICENSE
  • Chapter01
    • Chapter 1.ipynb
  • Chapter12
    • IntegrationTesting.scala
    • TestingSparkPre2x.scala
    • MockingDataSources.scala
    • SeparatingLogic.scala
    • BonusVerifier.scala
    • PropertyBasedTesting.scala
  • Chapter10
    • SaveCSV.scala
    • SaveJSON.scala
    • SaveParquet.scala
    • ReUseWithCheckpoint.scala
    • SaveAvro.scala
    • SavePlainText.scala
  • README.md
  • Chapter02
    • Chapter 2.ipynb
  • Chapter05
    • Chapter 5.ipynb
  • Chapter03
    • Chapter 3.ipynb
  • Chapter07
    • ReduceAPI.scala
    • GroupByKey.scala
    • TriggerComputations.scala
    • DeferComputations.scala
    • TriggerComputationsReusingRDD.scala
  • Chapter11
    • ActionsOnPairs.scala
    • AggregateByKey.scala
    • CustomRangePartitioner.scala
    • TransformationsOnPairs.scala
    • UsePartitioner.scala
  • Chapter04
    • Chapter 4.ipynb
  • Chapter06
    • Chapter 6.ipynb
  • Chapter09
    • TestingOperationsThatCausesShuffle.scala
    • DataSetJoins.scala
    • ExecutionPlanForJoins.scala
    • ChangingDesign.scala
    • DetectingShuffle.scala
    • CustomPartitioner.scala
    • DataFrameJoins.scala
    • KeyByTesting.scala

package com.tomekl007.chapter_5

import com.tomekl007.UserTransaction
import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession
import org.scalatest.FunSuite
import org.scalatest.Matchers._

import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer

class TransformationsOnPairs extends FunSuite {
  val spark: SparkContext = SparkSession.builder().master("local[2]").getOrCreate().sparkContext

  test("should use transformation on k/v pair") {
    //given
    val keysWithValuesList =
      Array(
        UserTransaction("A", 100),
        UserTransaction("B", 4),
        UserTransaction("A", 100001),
        UserTransaction("B", 10),
        UserTransaction("C", 10)
      )
    val data = spark.parallelize(keysWithValuesList)
    val keyed = data.keyBy(_.userId)

    //when
    val counted  = keyed.countByKey()
//    keyed.combineByKey()
//    keyed.aggregateByKey()
//    keyed.foldByKey()
//    keyed.groupByKey()

    //then
    counted should contain theSameElementsAs Map("B" -> 2, "A" -> 2, "C" -> 1)

  }
}