org.joda.time.Instant Scala Examples

package org.archive.archivespark.sparkling.warc

import java.nio.charset.Charset
import java.util.UUID

import org.archive.archivespark.sparkling.Sparkling
import org.archive.archivespark.sparkling.util.DigestUtil
import org.joda.time.Instant
import org.joda.time.format.{DateTimeFormat, DateTimeFormatter, ISODateTimeFormat}

object WarcHeaders {
  val UTF8: Charset = Charset.forName(Sparkling.DefaultCharset)
  val ArcDateTimeFormat: DateTimeFormatter = DateTimeFormat.forPattern("yyyyMMddHHmmss").withZoneUTC
  val WarcDateTimeFormat: DateTimeFormatter = ISODateTimeFormat.dateTimeNoMillis

  val Br = "\r\n"

  def arcFile(info: WarcFileMeta, filename: String): Array[Byte] = {
    val header = StringBuilder.newBuilder
    header.append("filedesc://")
    header.append(filename)
    header.append(" 0.0.0.0 ")
    header.append(ArcDateTimeFormat.print(info.created))
    header.append(" text/plain ")

    val headerBody = StringBuilder.newBuilder
    // Internet Archive: Name of gathering organization with no white space (http://archive.org/web/researcher/ArcFileFormat.php)
    headerBody.append("1 0 " + info.publisher.replace(" ", "")).append(Br)
    headerBody.append("URL IP-address Archive-date Content-type Archive-length").append(Br)

    val headerBodyStr: String = headerBody.toString
    val headerBodyBlob: Array[Byte] = headerBodyStr.getBytes(UTF8)

    header.append(headerBodyBlob.length).append(Br)
    header.append(headerBodyStr).append(Br)

    header.toString().getBytes(UTF8)
  }

  def warcFile(meta: WarcFileMeta, filename: String): Array[Byte] = {
    val header = StringBuilder.newBuilder
    header.append("WARC/1.0").append(Br)
    header.append("WARC-Type: warcinfo").append(Br)
    header.append("WARC-Date: " + WarcDateTimeFormat.print(Instant.now)).append(Br)
    header.append("WARC-Filename: " + filename).append(Br)
    header.append("WARC-Record-ID: " + newRecordID()).append(Br)
    header.append("Content-Type: application/warc-fields").append(Br)

    val headerBody = StringBuilder.newBuilder
    headerBody.append("software: " + meta.software).append(Br)
    headerBody.append("format: WARC File Format 1.0").append(Br)
    headerBody.append("conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf").append(Br)
    headerBody.append("publisher: " + meta.publisher).append(Br)
    headerBody.append("created: " + WarcDateTimeFormat.print(meta.created)).append(Br)
    headerBody.append(Br * 3)

    val headerBodyStr = headerBody.toString()
    val headerBodyBlob = headerBodyStr.getBytes(UTF8)

    header.append("Content-Length: " + headerBodyBlob.length).append(Br)
    header.append(Br)
    header.append(headerBodyStr)

    header.toString().getBytes(UTF8)
  }

  def warcResponseRecord(meta: WarcRecordMeta, content: Array[Byte], payload: Array[Byte]): Array[Byte] = {
    val header = StringBuilder.newBuilder
    header.append("WARC/1.0").append(Br)
    header.append("WARC-Type: response").append(Br)
    header.append("WARC-Target-URI: " + meta.url).append(Br)
    header.append("WARC-Date: " + WarcDateTimeFormat.print(meta.timestamp)).append(Br)
    header.append("WARC-Payload-Digest: sha1:" + DigestUtil.sha1Base32(payload)).append(Br)
    if (meta.ip.isDefined) header.append("WARC-IP-Address: " + meta.ip.get).append(Br)
    header.append("WARC-Record-ID: " + meta.recordId.getOrElse(newRecordID())).append(Br)
    header.append("Content-Type: application/http; msgtype=response").append(Br)
    header.append("Content-Length: " + content.length).append(Br)
    header.append(Br)

    header.toString().getBytes(UTF8)
  }

  def http(statusLine: String, headers: Map[String, String]): Array[Byte] = {
    val header = StringBuilder.newBuilder
    header.append(statusLine).append(Br)
    for ((key, value) <- headers) {
      header.append(s"$key: $value").append(Br)
    }
    header.append(Br)
    header.toString().getBytes(UTF8)
  }

  private def newRecordID(): String = "<urn:uuid:" + UUID.randomUUID() + ">"
} 

package com.spotify.scio.bigquery

import com.google.protobuf.ByteString
import com.spotify.scio._
import com.spotify.scio.bigquery.client.BigQuery
import com.spotify.scio.testing._
import magnolify.scalacheck.auto._
import org.apache.beam.sdk.options.PipelineOptionsFactory
import org.joda.time.format.DateTimeFormat
import org.joda.time.{Instant, LocalDate, LocalDateTime, LocalTime}
import org.scalacheck._
import org.scalatest.BeforeAndAfterAll

import scala.util.Random

object TypedBigQueryIT {
  @BigQueryType.toTable
  case class Record(
    bool: Boolean,
    int: Int,
    long: Long,
    float: Float,
    double: Double,
    string: String,
    byteString: ByteString,
    timestamp: Instant,
    date: LocalDate,
    time: LocalTime,
    datetime: LocalDateTime
  )

  // Workaround for millis rounding error
  val epochGen = Gen.chooseNum[Long](0L, 1000000000000L).map(x => x / 1000 * 1000)
  implicit val arbByteString = Arbitrary(Gen.alphaStr.map(ByteString.copyFromUtf8))
  implicit val arbInstant = Arbitrary(epochGen.map(new Instant(_)))
  implicit val arbDate = Arbitrary(epochGen.map(new LocalDate(_)))
  implicit val arbTime = Arbitrary(epochGen.map(new LocalTime(_)))
  implicit val arbDatetime = Arbitrary(epochGen.map(new LocalDateTime(_)))

  private val recordGen = {
    implicitly[Arbitrary[Record]].arbitrary
  }

  private val table = {
    val TIME_FORMATTER = DateTimeFormat.forPattern("yyyyMMddHHmmss")
    val now = Instant.now().toString(TIME_FORMATTER)
    val spec =
      "data-integration-test:bigquery_avro_it.records_" + now + "_" + Random.nextInt(Int.MaxValue)
    Table.Spec(spec)
  }
  private val records = Gen.listOfN(1000, recordGen).sample.get
  private val options = PipelineOptionsFactory
    .fromArgs(
      "--project=data-integration-test",
      "--tempLocation=gs://data-integration-test-eu/temp"
    )
    .create()
}

class TypedBigQueryIT extends PipelineSpec with BeforeAndAfterAll {
  import TypedBigQueryIT._

  override protected def beforeAll(): Unit = {
    val sc = ScioContext(options)
    sc.parallelize(records).saveAsTypedBigQueryTable(table)

    sc.run()
    ()
  }

  override protected def afterAll(): Unit =
    BigQuery.defaultInstance().tables.delete(table.ref)

  "TypedBigQuery" should "read records" in {
    val sc = ScioContext(options)
    sc.typedBigQuery[Record](table) should containInAnyOrder(records)
    sc.run()
  }
} 

// Example: Demonstrates a streaming job with periodically refreshing side input
// Usage:

// `sbt "scio-examples/runMain com.spotify.scio.examples.extra.RefreshingSideInputExample
// --project=[PROJECT] --runner=[RUNNER] --zone=[ZONE] --input=[PUBSUB_SUBSCRIPTION]"`
package com.spotify.scio.examples.extra

import com.spotify.scio._
import com.spotify.scio.values.WindowOptions
import org.apache.beam.sdk.io.GenerateSequence
import org.apache.beam.sdk.options.StreamingOptions
import org.apache.beam.sdk.transforms.windowing.Window.ClosingBehavior
import org.apache.beam.sdk.transforms.windowing.{AfterPane, Repeatedly}
import org.apache.beam.sdk.values.WindowingStrategy.AccumulationMode
import org.joda.time.{Duration, Instant}
import org.slf4j.LoggerFactory

import scala.util.{Random, Success, Try}


object RefreshingSideInputExample {
  case class LotteryTicket(numbers: Seq[Int])
  case class LotteryResult(
    eventTime: Instant,
    processTime: Instant,
    isWinner: Boolean,
    ticket: Seq[Int],
    winningNumbers: Seq[Int]
  )

  private lazy val logger = LoggerFactory.getLogger(this.getClass)

  private val ticketSize = 5

  def main(cmdlineArgs: Array[String]): Unit = {
    val (sc, args) = ContextAndArgs(cmdlineArgs)
    sc.optionsAs[StreamingOptions].setStreaming(true)

    // An unbounded input that produces a sequence of 5 randomly generated winning lottery numbers,
    // refreshed every 10 seconds. Materialized as a singleton `SideInput`.
    val winningLotteryNumbers = sc
      .customInput(
        "winningLotteryNumbers",
        GenerateSequence
          .from(0)
          .withRate(1, Duration.standardSeconds(10))
      )
      .withFixedWindows(
        duration = Duration.standardSeconds(10),
        offset = Duration.ZERO,
        options = WindowOptions(
          trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(1)),
          accumulationMode = AccumulationMode.DISCARDING_FIRED_PANES,
          closingBehavior = ClosingBehavior.FIRE_IF_NON_EMPTY,
          allowedLateness = Duration.standardSeconds(0)
        )
      )
      .map(_ => Seq.fill(ticketSize)(Random.nextInt(100)))
      // A default is needed in case an empty pane is fired
      .asSingletonSideInput(Seq.fill(ticketSize)(-1))

    // Sample PubSub topic modeling lottery tickets as a comma-separated list of numbers.
    // For example, a message might contain the string "10,7,3,1,9"
    sc.pubsubTopic[String](args("input"))
      .flatMap(toLotteryTicket)
      .withFixedWindows(Duration.standardSeconds(5))
      .withTimestamp
      .withSideInputs(winningLotteryNumbers)
      .map {
        case ((lotteryTicket, eventTime), side) =>
          val currentWinningNumbers = side(winningLotteryNumbers)

          val isWinner = lotteryTicket.numbers == currentWinningNumbers
          val result = LotteryResult(
            eventTime,
            Instant.now(),
            isWinner,
            lotteryTicket.numbers,
            currentWinningNumbers
          )

          logger.info(s"Lottery result: $result")
      } // Can save output to PubSub, BigQuery, etc.

    sc.run()
    ()
  }

  private def toLotteryTicket(message: String): Option[LotteryTicket] =
    Try(LotteryTicket(message.split(",").map(_.toInt))) match {
      case Success(s) if s.numbers.size == ticketSize => Some(s)
      case _ =>
        logger.error(s"Malformed message: $message")
        None
    }
} 

package com.spotify.scio.examples.complete.game

import com.spotify.scio.examples.complete.game.UserScore.GameActionInfo
import com.spotify.scio.testing._
import org.apache.beam.sdk.transforms.windowing.IntervalWindow
import org.apache.beam.sdk.values.TimestampedValue
import org.joda.time.{Duration, Instant}

class LeaderBoardTest extends PipelineSpec {
  private val allowedLateness = Duration.standardHours(1)
  private val teamWindowDuration = Duration.standardMinutes(20)
  private val baseTime = new Instant(0)

  case class TestUser(user: String, team: String)
  private val redOne = TestUser("scarlet", "red")
  private val redTwo = TestUser("burgundy", "red")
  private val blueOne = TestUser("navy", "blue")
  private val blueTwo = TestUser("sky", "blue")

  private def event(
    user: TestUser,
    score: Int,
    baseTimeOffset: Duration
  ): TimestampedValue[GameActionInfo] = {
    val t = baseTime.plus(baseTimeOffset)
    TimestampedValue.of(GameActionInfo(user.user, user.team, score, t.getMillis), t)
  }

  "LeaderBoard.calculateTeamScores" should "work with on time elements" in {
    // #LeaderBoardTest_example_1
    val stream = testStreamOf[GameActionInfo]
    // Start at the epoch
      .advanceWatermarkTo(baseTime)
      // add some elements ahead of the watermark
      .addElements(
        event(blueOne, 3, Duration.standardSeconds(3)),
        event(blueOne, 2, Duration.standardMinutes(1)),
        event(redTwo, 3, Duration.standardSeconds(22)),
        event(blueTwo, 5, Duration.standardSeconds(3))
      )
      // #LeaderBoardTest_example_1
      // #LeaderBoardTest_example_2
      // The watermark advances slightly, but not past the end of the window
      .advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3)))
      .addElements(
        event(redOne, 1, Duration.standardMinutes(4)),
        event(blueOne, 2, Duration.standardSeconds(270))
      )
      // The window should close and emit an ON_TIME pane
      .advanceWatermarkToInfinity
    // #LeaderBoardTest_example_2

    // #LeaderBoardTest_example_3
    runWithContext { sc =>
      val teamScores =
        LeaderBoard.calculateTeamScores(sc.testStream(stream), teamWindowDuration, allowedLateness)

      val window = new IntervalWindow(baseTime, teamWindowDuration)
      teamScores should inOnTimePane(window) {
        containInAnyOrder(Seq((blueOne.team, 12), (redOne.team, 4)))
      }
    }
    // #LeaderBoardTest_example_3
  }
} 

package com.spotify.scio.examples.complete

import com.spotify.scio.testing._
import org.joda.time.{Duration, Instant}

class AutoCompleteTest extends PipelineSpec {
  "AutoComplete" should "work" in {
    val data = Seq(
      "apple",
      "apple",
      "apricot",
      "banana",
      "blackberry",
      "blackberry",
      "blackberry",
      "blueberry",
      "blueberry",
      "cherry"
    )
    val expected = Seq(
      ("a", Map("apple" -> 2L, "apricot" -> 1L)),
      ("ap", Map("apple" -> 2L, "apricot" -> 1L)),
      ("b", Map("blackberry" -> 3L, "blueberry" -> 2L)),
      ("ba", Map("banana" -> 1L)),
      ("bl", Map("blackberry" -> 3L, "blueberry" -> 2L)),
      ("c", Map("cherry" -> 1L)),
      ("ch", Map("cherry" -> 1L))
    )
    runWithContext { sc =>
      val in = sc.parallelize(data)
      for (recursive <- Seq(true, false)) {
        val r = AutoComplete
          .computeTopCompletions(in, 2, recursive)
          .filter(_._1.length <= 2)
          .mapValues(_.toMap)
        r should containInAnyOrder(expected)
      }
    }
  }

  it should "work with tiny input" in {
    val data = Seq("x", "x", "x", "xy", "xy", "xyz")
    val expected = Seq(
      ("x", Map("x" -> 3L, "xy" -> 2L)),
      ("xy", Map("xy" -> 2L, "xyz" -> 1L)),
      ("xyz", Map("xyz" -> 1L))
    )
    runWithContext { sc =>
      val in = sc.parallelize(data)
      for (recursive <- Seq(true, false)) {
        val r = AutoComplete
          .computeTopCompletions(in, 2, recursive)
          .mapValues(_.toMap)
        r should containInAnyOrder(expected)
      }
    }
  }

  it should "work with windowed input" in {
    val data = Seq(
      ("xA", new Instant(1)),
      ("xA", new Instant(1)),
      ("xB", new Instant(1)),
      ("xB", new Instant(2)),
      ("xB", new Instant(2))
    )
    val expected = Seq(
      // window [0, 2)
      ("x", Map("xA" -> 2L, "xB" -> 1L)),
      ("xA", Map("xA" -> 2L)),
      ("xB", Map("xB" -> 1L)),
      // window [1, 3)
      ("x", Map("xA" -> 2L, "xB" -> 3L)),
      ("xA", Map("xA" -> 2L)),
      ("xB", Map("xB" -> 3L)),
      // window [2, 3)
      ("x", Map("xB" -> 2L)),
      ("xB", Map("xB" -> 2L))
    )
    runWithContext { sc =>
      val in =
        sc.parallelizeTimestamped(data).withSlidingWindows(new Duration(2))
      for (recursive <- Seq(true, false)) {
        val r = AutoComplete
          .computeTopCompletions(in, 2, recursive)
          .mapValues(_.toMap)
        r should containInAnyOrder(expected)
      }
    }
  }
} 

package com.spotify.scio.examples.extra

import com.spotify.scio.io.TextIO
import com.spotify.scio.testing._
import org.joda.time.{Duration, Instant}

class WindowedWordCountTest extends PipelineSpec {
  private val baseTime = new Instant(0)

  "WindowedWordCount" should "work" in {
    JobTest[com.spotify.scio.examples.WindowedWordCount.type]
      .args(
        "--input=input.txt",
        s"--windowSize=PT0.1S", // 100 ms, in ISO-8601 standard used by Joda for Duration parsing
        "--outputGlobalWindow=true",
        "--output=output.txt"
      )
      .inputStream(
        TextIO("input.txt"),
        testStreamOf[String]
          .advanceWatermarkTo(baseTime)
          .addElements("a b b c")
          .advanceWatermarkTo(baseTime.plus(Duration.millis(150)))
          .addElements("b e")
          .advanceWatermarkToInfinity()
      )
      .output(TextIO("output.txt"))(
        _ should containInAnyOrder(Seq("(a,1)", "(b,2)", "(b,1)", "(c,1)", "(e,1)"))
      )
      .run()
  }
} 

package com.spotify.scio.examples.cookbook

import com.spotify.scio.testing._
import org.joda.time.{Duration, Instant}

class TriggerExampleTest extends PipelineSpec {
  // #TriggerExampleTest_example
  "TriggerExample.extractFlowInfo" should "work" in {
    val data = Seq(
      "01/01/2010 00:00:00,1108302,94,E,ML,36,100,29,0.0065,66,9,1,0.001,74.8,1,9,3,0.0028,71,1,9,"
        + "12,0.0099,67.4,1,9,13,0.0121,99.0,1,,,,,0,,,,,0,,,,,0,,,,,0",
      "01/01/2010 00:00:00,"
        + "1100333,5,N,FR,9,0,39,,,9,,,,0,,,,,0,,,,,0,,,,,0,,,,,0,,,,,0,,,,,0,,,,"
    )
    runWithContext { sc =>
      val r = TriggerExample.extractFlowInfo(sc.parallelize(data))
      r should haveSize(1)
      r should containSingleValue(("94", 29))
    }
  }
  // #TriggerExampleTest_example

  "TriggerExample.totalFlow" should "work" in {
    val data = Seq(
      (
        "01/01/2010 00:00:00,1108302,5,W,ML,36,100,30,0.0065,66,9,1,0.001,"
          + "74.8,1,9,3,0.0028,71,1,9,12,0.0099,87.4,1,9,13,0.0121,99.0,1,,,,,0,,,,,0,,,,,0,,,"
          + ",,0",
        new Instant(60000)
      ),
      (
        "01/01/2010 00:00:00,1108302,110,E,ML,36,100,40,0.0065,66,9,1,0.001,"
          + "74.8,1,9,3,0.0028,71,1,9,12,0.0099,67.4,1,9,13,0.0121,99.0,1,,,,,0,,,,,0,,,,,0,,,"
          + ",,0",
        new Instant(1)
      ),
      (
        "01/01/2010 00:00:00,1108302,110,E,ML,36,100,50,0.0065,66,9,1,"
          + "0.001,74.8,1,9,3,0.0028,71,1,9,12,0.0099,97.4,1,9,13,0.0121,50.0,1,,,,,0,,,,,0"
          + ",,,,,0,,,,,0",
        new Instant(1)
      )
    )
    val expected = Seq(
      TriggerExample.Record(
        "default",
        "5",
        30,
        1,
        "[1970-01-01T00:01:00.000Z..1970-01-01T00:02:00.000Z)",
        true,
        true,
        "ON_TIME",
        new Instant(1),
        new Instant(1)
      ),
      TriggerExample.Record(
        "default",
        "110",
        90,
        2,
        "[1970-01-01T00:00:00.000Z..1970-01-01T00:01:00.000Z)",
        true,
        true,
        "ON_TIME",
        new Instant(1),
        new Instant(1)
      )
    )
    runWithContext { sc =>
      val flowInfo = TriggerExample
        .extractFlowInfo(sc.parallelizeTimestamped(data))
        .withFixedWindows(Duration.standardMinutes(1))
      val r = TriggerExample
        .totalFlow(flowInfo, "default")
        .map(_.copy(event_time = new Instant(1), processing_time = new Instant(1)))
      r should containInAnyOrder(expected)
    }
  }
} 

package com.spotify.scio.bigquery.syntax

// import com.google.api.services.bigquery.model.{TableRow => GTableRow}
import com.spotify.scio.bigquery.{Date, DateTime, TableRow, Time, Timestamp}
import org.joda.time.{Instant, LocalDate, LocalDateTime, LocalTime}

import scala.jdk.CollectionConverters._

import scala.util.Try


final class TableRowOps(private val r: TableRow) extends AnyVal {
  def getBoolean(name: AnyRef): Boolean =
    this.getValue(name, _.toString.toBoolean, false)

  def getBooleanOpt(name: AnyRef): Option[Boolean] =
    this.getValueOpt(name, _.toString.toBoolean)

  def getLong(name: AnyRef): Long = this.getValue(name, _.toString.toLong, 0L)

  def getLongOpt(name: AnyRef): Option[Long] =
    this.getValueOpt(name, _.toString.toLong)

  def getDouble(name: AnyRef): Double =
    this.getValue(name, _.toString.toDouble, 0.0)

  def getDoubleOpt(name: AnyRef): Option[Double] =
    this.getValueOpt(name, _.toString.toDouble)

  def getString(name: AnyRef): String = this.getValue(name, _.toString, null)

  def getStringOpt(name: AnyRef): Option[String] =
    this.getValueOpt(name, _.toString)

  def getTimestamp(name: AnyRef): Instant =
    this.getValue(name, v => Timestamp.parse(v.toString), null)

  def getTimestampOpt(name: AnyRef): Option[Instant] =
    this.getValueOpt(name, v => Timestamp.parse(v.toString))

  def getDate(name: AnyRef): LocalDate =
    this.getValue(name, v => Date.parse(v.toString), null)

  def getDateOpt(name: AnyRef): Option[LocalDate] =
    this.getValueOpt(name, v => Date.parse(v.toString))

  def getTime(name: AnyRef): LocalTime =
    this.getValue(name, v => Time.parse(v.toString), null)

  def getTimeOpt(name: AnyRef): Option[LocalTime] =
    this.getValueOpt(name, v => Time.parse(v.toString))

  def getDateTime(name: AnyRef): LocalDateTime =
    this.getValue(name, v => DateTime.parse(v.toString), null)

  def getDateTimeOpt(name: AnyRef): Option[LocalDateTime] =
    this.getValueOpt(name, v => DateTime.parse(v.toString))

  def getRepeated(name: AnyRef): Seq[AnyRef] =
    this.getValue(name, x => x.asInstanceOf[java.util.List[AnyRef]].iterator().asScala.toSeq, null)

  def getRecord(name: AnyRef): Map[String, AnyRef] =
    r.get(name).asInstanceOf[java.util.Map[String, AnyRef]].asScala.toMap

  private def getValue[T](name: AnyRef, fn: AnyRef => T, default: T): T = {
    val o = r.get(name)
    if (o == null) {
      default
    } else {
      fn(o)
    }
  }

  private def getValueOpt[T](name: AnyRef, fn: AnyRef => T): Option[T] = {
    val o = r.get(name)
    if (o == null) {
      None
    } else {
      Try(fn(o)).toOption
    }
  }
}

trait TableRowSyntax {
  implicit def bigQueryTableRowOps(tr: TableRow): TableRowOps = new TableRowOps(tr)
} 

package com.spotify.scio.bigquery.types

import com.google.protobuf.ByteString
import org.joda.time.{Instant, LocalDate, LocalDateTime, LocalTime}

object Schemas {
  // primitives
  case class Required(
    boolF: Boolean,
    intF: Int,
    longF: Long,
    floatF: Float,
    doubleF: Double,
    stringF: String,
    byteArrayF: Array[Byte],
    byteStringF: ByteString,
    timestampF: Instant,
    dateF: LocalDate,
    timeF: LocalTime,
    datetimeF: LocalDateTime,
    bigDecimalF: BigDecimal,
    geographyF: Geography
  )
  case class Optional(
    boolF: Option[Boolean],
    intF: Option[Int],
    longF: Option[Long],
    floatF: Option[Float],
    doubleF: Option[Double],
    stringF: Option[String],
    byteArrayF: Option[Array[Byte]],
    byteStringF: Option[ByteString],
    timestampF: Option[Instant],
    dateF: Option[LocalDate],
    timeF: Option[LocalTime],
    datetimeF: Option[LocalDateTime],
    bigDecimalF: Option[BigDecimal],
    geographyF: Option[Geography]
  )
  case class Repeated(
    boolF: List[Boolean],
    intF: List[Int],
    longF: List[Long],
    floatF: List[Float],
    doubleF: List[Double],
    stringF: List[String],
    byteArrayF: List[Array[Byte]],
    byteStringF: List[ByteString],
    timestampF: List[Instant],
    dateF: List[LocalDate],
    timeF: List[LocalTime],
    datetimeF: List[LocalDateTime],
    bigDecimalF: List[BigDecimal],
    geographyF: List[Geography]
  )

  // records
  case class RequiredNested(required: Required, optional: Optional, repeated: Repeated)
  case class OptionalNested(
    required: Option[Required],
    optional: Option[Optional],
    repeated: Option[Repeated]
  )
  case class RepeatedNested(
    required: List[Required],
    optional: List[Optional],
    repeated: List[Repeated]
  )

  case class User(@description("user name") name: String, @description("user age") age: Int)
  case class Account(
    @description("account user") user: User,
    @description("in USD") balance: Double
  )
} 

package com.spotify.scio

import org.joda.time.Instant


  final case class Metrics(
    version: String,
    scalaVersion: String,
    appName: String,
    state: String,
    beamMetrics: BeamMetrics
  )

  final case class BeamMetrics(
    counters: Iterable[BeamMetric[Long]],
    distributions: Iterable[BeamMetric[BeamDistribution]],
    gauges: Iterable[BeamMetric[BeamGauge]]
  )
  final case class BeamMetric[T](namespace: String, name: String, value: MetricValue[T])
  final case class BeamDistribution(sum: Long, count: Long, min: Long, max: Long, mean: Double)
  final case class BeamGauge(value: Long, timestamp: Instant)
} 

package com.paypal.gimel.common.query.guard

import java.util.concurrent.atomic.AtomicBoolean

import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd, SparkListenerJobStart, SparkListenerStageCompleted}
import org.apache.spark.sql.SparkSession
import org.joda.time.{DateTime, Instant}

import com.paypal.gimel.common.conf.{QueryGuardConfigs, QueryGuardConstants}
import com.paypal.gimel.common.utilities.GenericUtils
import com.paypal.gimel.logger.Logger

class QueryGuardListener[E >: QueryGuardDelayedEvent](spark: SparkSession,
                                                      discoveryType: String =
                                                        "job")
    extends SparkListener
    with Producer[E] {
  private val logger = new Logger(this.getClass.getName)
  private val stopped = new AtomicBoolean(true)
  private val HEADER: String = "[DISCOVERY] "
  private var _consumers: Seq[Consumer[E]] = Seq.empty

  override def onJobStart(jobStart: SparkListenerJobStart) {
    logger.info(
      s"${HEADER}Job[${jobStart.jobId}] started with ${jobStart.stageInfos.size} stages @ ${Instant.now()}"
    )
    if (!stopped.get) {
      val job = JobSubmitted(
        jobStart.jobId,
        discoveryType,
        System.currentTimeMillis(),
        jobTtl = GenericUtils.getValue(
          spark.conf,
          QueryGuardConfigs.JOB_TTL,
          QueryGuardConstants.DEFAULT_JOB_TTL
        ),
        delayTtl = GenericUtils.getValue(
          spark.conf,
          QueryGuardConfigs.DELAY_TTL,
          QueryGuardConstants.DEFAULT_DELAY_TTL
        )
      )
      logger.info(
        s"${HEADER}Proceeding to queue in Job[${jobStart.jobId}] onto QueryGuard"
      )
      publish(job)
    } else {
      logger.info(
        s"${HEADER}As QueryGuardListener is ${stopped.get()}," +
          s" unable to queue in Job[${jobStart.jobId}]"
      )
    }
  }

  override def publish(queryGuardEvent: E): Unit = {
    for (consumer <- _consumers) {
      consumer.consume(queryGuardEvent)
    }
  }

  override def onStageCompleted(
    stageCompleted: SparkListenerStageCompleted
  ): Unit = {
    logger.info(
      s"Stage ${stageCompleted.stageInfo.stageId} completed with ${stageCompleted.stageInfo.numTasks} tasks."
    )
  }

  override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = {
    logger.info(
      s"Job[${jobEnd.jobId}] completed at ${new DateTime(jobEnd.time)}" +
        s" with result -> ${jobEnd.jobResult}"
    )
    super.onJobEnd(jobEnd)
  }

  override def registerConsumers(consumers: Seq[Consumer[E]]): Unit = {
    _consumers = consumers
  }

  def start(): Unit = {
    // toggle stopped to true
    stopped.set(false)
    logger.info(s"${HEADER}Started QueryGuardListener: $stopped")
  }

  def stop(): Unit = {
    // toggle stopped to true
    stopped.compareAndSet(false, true)
    logger.info(s"${HEADER}Stopped QueryGuardListener: $stopped")
  }
} 

package com.spotify.scio.values

import com.spotify.scio.testing.PipelineSpec
import org.joda.time.Instant

class WindowedSCollectionTest extends PipelineSpec {
  "WindowedSCollection" should "support filter()" in {
    runWithContext { sc =>
      val p =
        sc.parallelizeTimestamped(Seq("a", "b", "c", "d"), Seq(1, 2, 3, 4).map(new Instant(_)))
      val r = p.toWindowed
        .filter(v => v.timestamp.getMillis % 2 == 0)
        .toSCollection
        .withTimestamp
        .map(kv => (kv._1, kv._2.getMillis))
      r should containInAnyOrder(Seq(("b", 2L), ("d", 4L)))
    }
  }

  it should "support flatMap()" in {
    runWithContext { sc =>
      val p =
        sc.parallelizeTimestamped(Seq("a", "b"), Seq(1, 2).map(new Instant(_)))
      val r = p.toWindowed
        .flatMap(v => Seq(v.copy(v.value + "1"), v.copy(v.value + "2")))
        .toSCollection
        .withTimestamp
        .map(kv => (kv._1, kv._2.getMillis))
      r should containInAnyOrder(Seq(("a1", 1L), ("a2", 1L), ("b1", 2L), ("b2", 2L)))
    }
  }

  it should "support keyBy()" in {
    runWithContext { sc =>
      val p =
        sc.parallelizeTimestamped(Seq("a", "b"), Seq(1, 2).map(new Instant(_)))
      val r =
        p.toWindowed.keyBy(v => v.value + v.timestamp.getMillis).toSCollection
      r should containInAnyOrder(Seq(("a1", "a"), ("b2", "b")))
    }
  }

  it should "support map()" in {
    runWithContext { sc =>
      val p =
        sc.parallelizeTimestamped(Seq("a", "b"), Seq(1, 2).map(new Instant(_)))
      val r = p.toWindowed
        .map(v => v.copy(v.value + v.timestamp.getMillis))
        .toSCollection
      r should containInAnyOrder(Seq("a1", "b2"))
    }
  }
} 

package highperfscala.orderbook

import org.joda.time.Instant
import org.scalacheck.Gen

// Model taken from chapter 2

case class Price(value: BigDecimal) extends AnyVal
object Price {
  implicit val genPrice: Gen[Price] = Gen.posNum[Double].map(d =>
    Price(BigDecimal(d)))
  implicit val ordering: Ordering[Price] = new Ordering[Price] {
    def compare(x: Price, y: Price): Int =
      Ordering.BigDecimal.compare(x.value, y.value)
  }
}

case class OrderId(value: Long)
object OrderId {
  implicit val genOrderId: Gen[OrderId] = Gen.posNum[Long].map(OrderId.apply)
}

sealed trait LimitOrder {
  def id: OrderId
  def price: Price
}
object LimitOrder {
  implicit val genLimitOrder: Gen[LimitOrder] = Gen.oneOf(
    BuyLimitOrder.genBuyLimitOrder, SellLimitOrder.genSellLimitOrder)
}
case class BuyLimitOrder(id: OrderId, price: Price) extends LimitOrder
object BuyLimitOrder {
  implicit val genBuyLimitOrder: Gen[BuyLimitOrder] = Gen.zip(
    OrderId.genOrderId, Price.genPrice).map(Function.tupled(BuyLimitOrder.apply))
}
case class SellLimitOrder(id: OrderId, price: Price) extends LimitOrder
object SellLimitOrder {
  implicit val genSellLimitOrder: Gen[SellLimitOrder] = Gen.zip(
    OrderId.genOrderId, Price.genPrice).map(Function.tupled(
    SellLimitOrder.apply))
}

case class Execution(orderId: OrderId, price: Price)

case class CommandInstant(value: Instant) extends AnyVal
object CommandInstant {
  def now(): CommandInstant =
    CommandInstant(new Instant(System.currentTimeMillis()))
  implicit val genCommandInstant: Gen[CommandInstant] =
    Gen.posNum[Long].map(l => CommandInstant(new Instant(l)))
}

case class EventInstant(value: Instant) extends AnyVal
object EventInstant {
  def now(): EventInstant =
    EventInstant(new Instant(System.currentTimeMillis()))
  implicit val genEventInstant: Gen[EventInstant] =
    Gen.posNum[Long].map(l => EventInstant(new Instant(l)))
} 

package highperfscala.clientreports.views

import org.joda.time.Instant

sealed trait LastHourPnL
case object LastHourPositive extends LastHourPnL
case object LastHourNegative extends LastHourPnL

sealed trait LastDayPnL
case object LastDayPositive extends LastDayPnL
case object LastDayNegative extends LastDayPnL

sealed trait LastSevenDayPnL
case object LastSevenDayPositive extends LastSevenDayPnL
case object LastSevenDayNegative extends LastSevenDayPnL

case class Ticker(value: String) extends AnyVal

case class TradingPerformanceTrend(
  ticker: Ticker,
  lastHour: LastHourPnL,
  lastDay: LastDayPnL,
  lastSevenDay: LastSevenDayPnL)

case class Price(value: BigDecimal) extends AnyVal
object Price {
  def average(ps: List[Price]): Price = {
    val prices = ps.map(_.value)
    Price(prices.sum / prices.length)
  }
}
case class OrderId(value: Long) extends AnyVal
case class CreatedTimestamp(value: Instant) extends AnyVal
case class ClientId(value: Long) extends AnyVal
sealed trait Order {
  def created: CreatedTimestamp
  def id: OrderId
  def ticker: Ticker
  def price: Price
  def clientId: ClientId
}
case class BuyOrder(
  created: CreatedTimestamp, id: OrderId, ticker: Ticker, price: Price,
  clientId: ClientId) extends Order
case class SellOrder(
  created: CreatedTimestamp, id: OrderId, ticker: Ticker, price: Price,
  clientId: ClientId) extends Order

case class Execution(created: CreatedTimestamp, id: OrderId, price: Price)

case class PnL(value: BigDecimal) extends AnyVal
object PnL {
  val zero: PnL = PnL(BigDecimal(0))
}

sealed trait PeriodPnL
case object PeriodPositive extends PeriodPnL
case object PeriodNegative extends PeriodPnL

case class GenerateTradingPerformanceTrend(
  tickers: List[Ticker], clientId: ClientId) 

package highperfscala.clientreports.views

import org.joda.time.{Duration, Instant, Interval}

object PerformanceReporting {

  def trend(
    now: () => Instant,
    findOrders: (Interval, Ticker) => List[Order],
    findExecutions: (Interval, Ticker) => List[Execution],
    request: GenerateTradingPerformanceTrend): List[TradingPerformanceTrend] = {
    def periodPnL(
      duration: Duration): Map[Ticker, PeriodPnL] = {
      val currentTime = now()
      val interval = new Interval(currentTime.minus(duration), currentTime)
      (for {
        ticker <- request.tickers
        orders = findOrders(interval, ticker)
        executions = findExecutions(interval, ticker)
        idToExecPrice = executions.groupBy(_.id).mapValues(es =>
          Price.average(es.map(_.price)))
        signedExecutionPrices = for {
          o <- orders
          if o.clientId == request.clientId
          price <- idToExecPrice.get(o.id).map(p => o match {
            case _: BuyOrder => Price(p.value * -1)
            case _: SellOrder => p
          }).toList
        } yield price
        trend = signedExecutionPrices.foldLeft(PnL.zero) {
          case (pnl, p) => PnL(pnl.value + p.value)
        } match {
          case p if p.value >= PnL.zero.value => PeriodPositive
          case _ => PeriodNegative
        }
      } yield ticker -> trend).toMap
    }

    val tickerToLastHour = periodPnL(Duration.standardHours(1)).mapValues {
      case PeriodPositive => LastHourPositive
      case PeriodNegative => LastHourNegative
    }
    val tickerToLastDay = periodPnL(Duration.standardDays(1)).mapValues {
      case PeriodPositive => LastDayPositive
      case PeriodNegative => LastDayNegative
    }
    val tickerToLastSevenDays = periodPnL(Duration.standardDays(7)).mapValues {
      case PeriodPositive => LastSevenDayPositive
      case PeriodNegative => LastSevenDayNegative
    }

    tickerToLastHour.zip(tickerToLastDay).zip(tickerToLastSevenDays).map({
      case (((t, lastHour), (_, lastDay)), (_, lastSevenDays)) =>
        TradingPerformanceTrend(t, lastHour, lastDay, lastSevenDays)
    }).toList
  }
} 

package highperfscala.clientreports.views

import org.joda.time.{Duration, Instant, Interval}

object ViewPerformanceReporting {

  def trend(
    now: () => Instant,
    findOrders: (Interval, Ticker) => List[Order],
    findExecutions: (Interval, Ticker) => List[Execution],
    request: GenerateTradingPerformanceTrend): List[TradingPerformanceTrend] = {

    def periodPnL(
      duration: Duration): Map[Ticker, PeriodPnL] = {
      val currentTime = now()
      val interval = new Interval(currentTime.minus(duration), currentTime)
      (for {
        ticker <- request.tickers
        orders = findOrders(interval, ticker)
        executions = findExecutions(interval, ticker)
        idToExecPrice = executions.groupBy(_.id).mapValues(es =>
          Price.average(es.map(_.price)))
        signedExecutionPrices = for {
          o <- orders.view
          if o.clientId == request.clientId
          price <- idToExecPrice.get(o.id).map(p => o match {
            case _: BuyOrder => Price(p.value * -1)
            case _: SellOrder => p
          }).toList
        } yield price
        trend = signedExecutionPrices.foldLeft(PnL.zero) {
          case (pnl, p) => PnL(pnl.value + p.value)
        } match {
          case p if p.value >= PnL.zero.value => PeriodPositive
          case _ => PeriodNegative
        }
      } yield ticker -> trend).toMap
    }

    val tickerToLastHour = periodPnL(Duration.standardHours(1)).mapValues {
      case PeriodPositive => LastHourPositive
      case PeriodNegative => LastHourNegative
    }
    val tickerToLastDay = periodPnL(Duration.standardDays(1)).mapValues {
      case PeriodPositive => LastDayPositive
      case PeriodNegative => LastDayNegative
    }
    val tickerToLastSevenDays = periodPnL(Duration.standardDays(7)).mapValues {
      case PeriodPositive => LastSevenDayPositive
      case PeriodNegative => LastSevenDayNegative
    }

    tickerToLastHour.zip(tickerToLastDay).zip(tickerToLastSevenDays).map({
      case (((t, lastHour), (_, lastDay)), (_, lastSevenDays)) =>
        TradingPerformanceTrend(t, lastHour, lastDay, lastSevenDays)
    }).toList
  }

} 

package highperfscala.clientreports.streams

import org.joda.time.Instant

case class Ticker(value: String) extends AnyVal
case class Price(value: BigDecimal) extends AnyVal
case class OrderId(value: Long) extends AnyVal
case class EventInstant(value: Instant) extends AnyVal
case class ClientId(value: Long) extends AnyVal

sealed trait Order {
  def created: EventInstant
  def id: OrderId
  def ticker: Ticker
  def price: Price
  def clientId: ClientId
}
case class BuyOrder(
  created: EventInstant, id: OrderId, ticker: Ticker, price: Price,
  clientId: ClientId) extends Order
case class SellOrder(
  created: EventInstant, id: OrderId, ticker: Ticker, price: Price,
  clientId: ClientId) extends Order

case class Execution(created: EventInstant, id: OrderId, price: Price)


sealed trait OrderBookEvent
case class BuyOrderSubmitted(
  created: EventInstant, id: OrderId, ticker: Ticker, price: Price,
  clientId: ClientId) extends OrderBookEvent
case class SellOrderSubmitted(
  created: EventInstant, id: OrderId, ticker: Ticker, price: Price,
  clientId: ClientId) extends OrderBookEvent
case class OrderCanceled(created: EventInstant, id: OrderId)
  extends OrderBookEvent
case class OrderExecuted(created: EventInstant, id: OrderId, price: Price)
  extends OrderBookEvent 

package org.coursera.common.jsonformat

import org.coursera.common.collection.Enum
import org.coursera.common.collection.EnumSymbol
import org.coursera.common.stringkey.StringKey
import org.coursera.common.stringkey.StringKeyFormat
import org.joda.time.DateTime
import org.joda.time.DateTimeZone
import org.joda.time.Duration
import org.joda.time.Instant
import org.junit.Test
import org.scalatest.junit.AssertionsForJUnit
import play.api.libs.json.Format
import play.api.libs.json.JsNumber
import play.api.libs.json.JsString
import play.api.libs.json.JsSuccess
import play.api.libs.json.Json

class JsonFormatsTest extends AssertionsForJUnit {

  import JsonFormatsTest._

  @Test
  def stringKey(): Unit = {
    val id = TestId(2, "test")
    val idString = StringKey.stringify(id)
    assert(JsString(idString) === Json.toJson(id))
    assert(JsSuccess(id) === Json.fromJson[TestId](JsString(idString)))
    assert(JsString(s"invalid stuff $idString").validate[TestId].isError)
  }

  @Test
  def enums(): Unit = {
    assertResult(Color.Amber)(JsString("Amber").as[Color])

    assertResult(JsString("Green"))(Json.toJson(Color.Green))
  }

  @Test
  def instant(): Unit = {
    import JsonFormats.Implicits.instantFormat
    val testInstant = new Instant(137)
    assertResult(JsNumber(137))(Json.toJson(testInstant))
    assertResult(Some(testInstant))(Json.parse("137").asOpt[Instant])
  }


  @Test
  def duration(): Unit = {
    import JsonFormats.Implicits.durationFormat
    val testDuration = Duration.millis(137L)
    assertResult(JsNumber(137))(Json.toJson(testDuration))
    assertResult(Some(testDuration))(Json.parse("137").asOpt[Duration])
  }


  @Test
  def dateTime(): Unit = {
    import JsonFormats.Implicits.dateTimeFormat
    val testDatetime = new DateTime(2010, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC)
    assertResult(JsNumber(1262304000000L))(Json.toJson(testDatetime))
    assertResult(Some(testDatetime))(Json.parse("1262304000000").asOpt[DateTime].map(_.withZone(DateTimeZone.UTC)))
  }
}

object JsonFormatsTest {

  case class TestId(part1: Int, part2: String)

  object TestId {
    implicit val stringKeyFormat: StringKeyFormat[TestId] =
      StringKeyFormat.caseClassFormat((apply _).tupled, unapply)
    implicit val format: Format[TestId] = JsonFormats.stringKeyFormat[TestId]
  }

  sealed trait Color extends EnumSymbol

  object Color extends Enum[Color] {
    case object Red extends Color
    case object Amber extends Color
    case object Green extends Color

    implicit val format: Format[Color] = JsonFormats.enumFormat(Color)
  }

} 

package models

import org.joda.time.{DateTime, Instant, LocalDate}
import play.api.libs.json._
import uk.gov.hmrc.domain.Nino
import _root_.util.DateTimeTools
import play.api.libs.json.JodaWrites._
import play.api.libs.json.JodaReads._
import play.api.libs.functional.syntax._

case class Person(
  firstName: Option[String],
  middleName: Option[String],
  lastName: Option[String],
  initials: Option[String],
  title: Option[String],
  honours: Option[String],
  sex: Option[String],
  dateOfBirth: Option[LocalDate],
  nino: Option[Nino]
) {
  lazy val initialsName =
    initials.getOrElse(List(title, firstName.map(_.take(1)), middleName.map(_.take(1)), lastName).flatten.mkString(" "))
  lazy val shortName = for {
    f <- firstName
    l <- lastName
  } yield List(f, l).mkString(" ")
  lazy val fullName = List(title, firstName, middleName, lastName, honours).flatten.mkString(" ")
}

object Person {

  implicit val localdateFormatDefault = new Format[LocalDate] {
    override def reads(json: JsValue): JsResult[LocalDate] = JodaReads.DefaultJodaLocalDateReads.reads(json)
    override def writes(o: LocalDate): JsValue = JodaWrites.DefaultJodaLocalDateWrites.writes(o)
  }

  implicit val formats = Json.format[Person]

} 

package com.weather.scalacass.joda

import com.datastax.driver.core.{ Cluster, DataType }
import com.google.common.reflect.TypeToken
import com.weather.scalacass.{ CassFormatDecoder, CassFormatEncoder }
import com.weather.scalacass.CassFormatEncoder.sameTypeCassFormatEncoder
import com.weather.scalacass.CassFormatDecoderVersionSpecific.codecCassFormatDecoder
import org.joda.time.{ DateTime, Instant, LocalDate, LocalTime }

object Implicits {
  implicit val timeEncoder: CassFormatEncoder[LocalTime] = sameTypeCassFormatEncoder(DataType.time)
  implicit val timeDecoder: CassFormatDecoder[LocalTime] = codecCassFormatDecoder(TypeToken.of(classOf[LocalTime]))

  implicit val dateEncoder: CassFormatEncoder[LocalDate] = sameTypeCassFormatEncoder(DataType.date)
  implicit val dateDecoder: CassFormatDecoder[LocalDate] = codecCassFormatDecoder(TypeToken.of(classOf[LocalDate]))

  implicit val instantEncoder: CassFormatEncoder[Instant] = sameTypeCassFormatEncoder(DataType.timestamp)
  implicit val instantDecoder: CassFormatDecoder[Instant] = codecCassFormatDecoder(TypeToken.of(classOf[Instant]))

  implicit def timestampEncoder(implicit cluster: Cluster): CassFormatEncoder[DateTime] =
    sameTypeCassFormatEncoder(cluster.getMetadata.newTupleType(DataType.timestamp, DataType.varchar))
  implicit val timestampDecoder: CassFormatDecoder[DateTime] = codecCassFormatDecoder(TypeToken.of(classOf[DateTime]))
} 

package com.spotify.ratatool.io

import java.util

import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.util.Utils
import com.google.api.services.bigquery.model.{Table, TableReference, TableRow, TableSchema}
import com.google.api.services.bigquery.{Bigquery, BigqueryScopes}
import org.apache.beam.sdk.io.gcp.bigquery.{BigQueryOptions,
                                            PatchedBigQueryServicesImpl,
                                            InsertRetryPolicy,
                                            PatchedBigQueryTableRowIterator}
import org.apache.beam.sdk.options.PipelineOptionsFactory
import org.apache.beam.sdk.transforms.windowing.{GlobalWindow, PaneInfo}
import org.apache.beam.sdk.values.ValueInSingleWindow
import org.joda.time.Instant

import scala.jdk.CollectionConverters._
import com.spotify.scio.bigquery.client.BigQuery


  def writeToTable(data: Seq[TableRow], schema: TableSchema, tableSpec: String): Unit =
    writeToTable(data, schema, parseTableSpec(tableSpec))

}

private class TableRowIterator(private val iter: PatchedBigQueryTableRowIterator)
  extends Iterator[TableRow] {
  private var _isOpen = false
  private var _hasNext = false

  private def init(): Unit = if (!_isOpen) {
    iter.open()
    _isOpen = true
    _hasNext = iter.advance()
  }

  override def hasNext: Boolean = {
    init()
    _hasNext
  }

  override def next(): TableRow = {
    init()
    if (_hasNext) {
      val r = iter.getCurrent
      _hasNext = iter.advance()
      r
    } else {
      throw new NoSuchElementException
    }
  }
} 

package pureconfig.module

import org.joda.time.{ DateTimeZone, Duration, Instant, Interval }
import org.joda.time.format.{ DateTimeFormat, DateTimeFormatter }
import pureconfig.{ ConfigConvert, ConfigReader }
import pureconfig.ConfigConvert.{ catchReadError, viaNonEmptyString }

package object joda {
  implicit def instantConfigConvert: ConfigConvert[Instant] =
    ConfigConvert[Long].xmap(new Instant(_), _.getMillis)

  implicit def intervalConfigConvert: ConfigConvert[Interval] =
    viaNonEmptyString[Interval](
      catchReadError(Interval.parseWithOffset), _.toString)

  implicit def durationConfigConvert: ConfigConvert[Duration] =
    viaNonEmptyString[Duration](
      catchReadError(Duration.parse), _.toString)

  implicit def dateTimeFormatterConfigConvert: ConfigReader[DateTimeFormatter] =
    ConfigReader.fromNonEmptyString[DateTimeFormatter](
      catchReadError(DateTimeFormat.forPattern))

  implicit def dateTimeZoneConfigConvert: ConfigConvert[DateTimeZone] =
    viaNonEmptyString[DateTimeZone](
      catchReadError(DateTimeZone.forID), _.getID)
} 

package shapeless.datatype.test

import java.net.URI

import com.google.protobuf.ByteString
import org.joda.time.Instant
import org.scalacheck._

object Records {
  case class Required(
    booleanField: Boolean,
    intField: Int,
    longField: Long,
    floatField: Float,
    doubleField: Double,
    stringField: String,
    byteStringField: ByteString,
    byteArrayField: Array[Byte],
    timestampField: Instant
  )
  case class Optional(
    booleanField: Option[Boolean],
    intField: Option[Int],
    longField: Option[Long],
    floatField: Option[Float],
    doubleField: Option[Double],
    stringField: Option[String],
    byteStringField: Option[ByteString],
    byteArrayField: Option[Array[Byte]],
    timestampField: Option[Instant]
  )
  case class Repeated(
    booleanField: List[Boolean],
    intField: List[Int],
    longField: List[Long],
    floatField: List[Float],
    doubleField: List[Double],
    stringField: List[String],
    byteStringField: List[ByteString],
    byteArrayField: List[Array[Byte]],
    timestampField: List[Instant]
  )
  case class Mixed(
    longField: Long,
    doubleField: Double,
    stringField: String,
    longFieldO: Option[Long],
    doubleFieldO: Option[Double],
    stringFieldO: Option[String],
    longFieldR: List[Long],
    doubleFieldR: List[Double],
    stringFieldR: List[String]
  )
  case class Nested(
    longField: Long,
    longFieldO: Option[Long],
    longFieldR: List[Long],
    mixedField: Mixed,
    mixedFieldO: Option[Mixed],
    mixedFieldR: List[Mixed]
  )
  case class Seqs(array: Array[Int], list: List[Int], vector: Vector[Int])
  case class Custom(uriField: URI, uriFieldO: Option[URI], uriFieldR: List[URI])

  implicit val arbByteString = Arbitrary(Gen.alphaStr.map(ByteString.copyFromUtf8))
  implicit val arbInstant = Arbitrary(Gen.chooseNum(0, Int.MaxValue).map(new Instant(_)))
  implicit val arbUri = Arbitrary(Gen.alphaStr.map(URI.create))
} 

package shapeless.datatype.tensorflow

import java.net.URI

import org.joda.time.Instant
import org.scalacheck.Prop.{all, forAll}
import org.scalacheck.ScalacheckShapeless._
import org.scalacheck._
import org.tensorflow.example.Example
import shapeless._
import shapeless.datatype.record._

object TensorFlowTypeSpec extends Properties("TensorFlowType") {
  import shapeless.datatype.test.Records._
  import shapeless.datatype.test.SerializableUtils._

  implicit def compareByteArrays(x: Array[Byte], y: Array[Byte]) = java.util.Arrays.equals(x, y)
  implicit def compareIntArrays(x: Array[Int], y: Array[Int]) = java.util.Arrays.equals(x, y)
  implicit def compareDouble(x: Double, y: Double) = x.toFloat == y.toFloat

  def roundTrip[A, L <: HList](m: A)(implicit
    gen: LabelledGeneric.Aux[A, L],
    fromL: FromFeatures[L],
    toL: ToFeatures[L],
    mr: MatchRecord[L]
  ): Prop = {
    val t = ensureSerializable(TensorFlowType[A])
    val f1: SerializableFunction[A, Example] =
      new SerializableFunction[A, Example] {
        override def apply(m: A): Example = t.toExample(m)
      }
    val f2: SerializableFunction[Example, Option[A]] =
      new SerializableFunction[Example, Option[A]] {
        override def apply(m: Example): Option[A] = t.fromExample(m)
      }
    val f3: SerializableFunction[A, Example.Builder] =
      new SerializableFunction[A, Example.Builder] {
        override def apply(m: A): Example.Builder = t.toExampleBuilder(m)
      }
    val f4: SerializableFunction[Example.Builder, Option[A]] =
      new SerializableFunction[Example.Builder, Option[A]] {
        override def apply(m: Example.Builder): Option[A] = t.fromExampleBuilder(m)
      }
    val toFn1 = ensureSerializable(f1)
    val fromFn1 = ensureSerializable(f2)
    val toFn2 = ensureSerializable(f3)
    val fromFn2 = ensureSerializable(f4)
    val copy1 = fromFn1(toFn1(m))
    val copy2 = fromFn2(toFn2(m))
    val rm = RecordMatcher[A]
    all(copy1.exists(rm(_, m)), copy2.exists(rm(_, m)))
  }

  implicit val timestampTensorFlowMappableType = TensorFlowType.at[Instant](
    TensorFlowType.toLongs(_).map(new Instant(_)),
    xs => TensorFlowType.fromLongs(xs.map(_.getMillis))
  )
  property("required") = forAll { m: Required => roundTrip(m) }
  property("optional") = forAll { m: Optional => roundTrip(m) }
  property("repeated") = forAll { m: Repeated => roundTrip(m) }
  property("mixed") = forAll { m: Mixed => roundTrip(m) }
  property("seqs") = forAll { m: Seqs => roundTrip(m) }

  implicit val uriTensorFlowType = TensorFlowType.at[URI](
    TensorFlowType.toStrings(_).map(URI.create),
    xs => TensorFlowType.fromStrings(xs.map(_.toString))
  )
  property("custom") = forAll { m: Custom => roundTrip(m) }
} 

package shapeless.datatype.datastore

import com.google.datastore.v1.client.DatastoreHelper._
import com.google.datastore.v1.{Entity, Value}
import com.google.protobuf.{ByteString, Timestamp}
import org.joda.time.Instant
import org.joda.time.DateTimeConstants
import shapeless.datatype.mappable.{BaseMappableType, MappableType}

import scala.collection.JavaConverters._

trait BaseDatastoreMappableType[V] extends MappableType[Entity.Builder, V] {
  def from(value: Value): V
  def to(value: V): Value

  override def get(m: Entity.Builder, key: String): Option[V] =
    Option(m.getPropertiesMap.get(key)).map(from)
  override def getAll(m: Entity.Builder, key: String): Seq[V] =
    Option(m.getPropertiesMap.get(key)).toSeq
      .flatMap(_.getArrayValue.getValuesList.asScala.map(from))

  override def put(key: String, value: V, tail: Entity.Builder): Entity.Builder =
    tail.putProperties(key, to(value))
  override def put(key: String, value: Option[V], tail: Entity.Builder): Entity.Builder =
    value.foldLeft(tail)((b, v) => b.putProperties(key, to(v)))
  override def put(key: String, values: Seq[V], tail: Entity.Builder): Entity.Builder =
    tail.putProperties(key, makeValue(values.map(to).asJava).build())
}

trait DatastoreMappableType extends DatastoreMappableTypes {
  implicit val datastoreBaseMappableType = new BaseMappableType[Entity.Builder] {
    override def base: Entity.Builder = Entity.newBuilder()

    override def get(m: Entity.Builder, key: String): Option[Entity.Builder] =
      Option(m.getPropertiesMap.get(key)).map(_.getEntityValue.toBuilder)
    override def getAll(m: Entity.Builder, key: String): Seq[Entity.Builder] =
      Option(m.getPropertiesMap.get(key)).toSeq
        .flatMap(_.getArrayValue.getValuesList.asScala.map(_.getEntityValue.toBuilder))

    override def put(key: String, value: Entity.Builder, tail: Entity.Builder): Entity.Builder =
      tail.putProperties(key, makeValue(value).build())
    override def put(
      key: String,
      value: Option[Entity.Builder],
      tail: Entity.Builder
    ): Entity.Builder =
      value.foldLeft(tail)((b, v) => b.putProperties(key, makeValue(v).build()))
    override def put(
      key: String,
      values: Seq[Entity.Builder],
      tail: Entity.Builder
    ): Entity.Builder =
      tail.putProperties(key, makeValue(values.map(v => makeValue(v).build()).asJava).build())
  }
} 

package shapeless.datatype.datastore

import com.google.datastore.v1.client.DatastoreHelper.makeValue
import com.google.datastore.v1.{Entity, Value}
import com.google.protobuf.{ByteString, Timestamp}
import org.joda.time.{DateTimeConstants, Instant}
import shapeless._

class DatastoreType[A] extends Serializable {
  def fromEntityBuilder[L <: HList](m: Entity.Builder)(implicit
    gen: LabelledGeneric.Aux[A, L],
    fromL: FromEntity[L]
  ): Option[A] =
    fromL(m).map(gen.from)

  def fromEntity[L <: HList](m: Entity)(implicit
    gen: LabelledGeneric.Aux[A, L],
    fromL: FromEntity[L]
  ): Option[A] =
    fromL(m.toBuilder).map(gen.from)

  def toEntityBuilder[L <: HList](a: A)(implicit
    gen: LabelledGeneric.Aux[A, L],
    toL: ToEntity[L]
  ): Entity.Builder =
    toL(gen.to(a))

  def toEntity[L <: HList](a: A)(implicit
    gen: LabelledGeneric.Aux[A, L],
    toL: ToEntity[L]
  ): Entity =
    toL(gen.to(a)).build()
}

object DatastoreType {
  def apply[A]: DatastoreType[A] = new DatastoreType[A]

  def at[V](fromFn: Value => V, toFn: V => Value): BaseDatastoreMappableType[V] =
    new BaseDatastoreMappableType[V] {
      override def from(value: Value): V = fromFn(value)
      override def to(value: V): Value = toFn(value)
    }
}

trait DatastoreMappableTypes {
  import DatastoreType.at

  implicit val booleanEntityMappableType = at[Boolean](_.getBooleanValue, makeValue(_).build())
  implicit val intDatastoreMappableType = at[Int](_.getIntegerValue.toInt, makeValue(_).build())
  implicit val longEntityMappableType = at[Long](_.getIntegerValue, makeValue(_).build())
  implicit val floatEntityMappableType = at[Float](_.getDoubleValue.toFloat, makeValue(_).build())
  implicit val doubleEntityMappableType = at[Double](_.getDoubleValue, makeValue(_).build())
  implicit val stringEntityMappableType = at[String](_.getStringValue, makeValue(_).build())
  implicit val byteStringEntityMappableType = at[ByteString](_.getBlobValue, makeValue(_).build())
  implicit val byteArrayEntityMappableType =
    at[Array[Byte]](_.getBlobValue.toByteArray, v => makeValue(ByteString.copyFrom(v)).build())
  implicit val timestampEntityMappableType = at[Instant](toInstant, fromInstant)

  private def toInstant(v: Value): Instant = {
    val t = v.getTimestampValue
    new Instant(t.getSeconds * DateTimeConstants.MILLIS_PER_SECOND + t.getNanos / 1000000)
  }
  private def fromInstant(i: Instant): Value = {
    val t = Timestamp
      .newBuilder()
      .setSeconds(i.getMillis / DateTimeConstants.MILLIS_PER_SECOND)
      .setNanos((i.getMillis % 1000).toInt * 1000000)
    Value.newBuilder().setTimestampValue(t).build()
  }
} 

package shapeless.datatype.avro

import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
import java.net.URI
import java.nio.ByteBuffer

import com.google.protobuf.ByteString
import org.apache.avro.Schema
import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.io.{DecoderFactory, EncoderFactory}
import org.joda.time.Instant
import org.scalacheck.Prop.forAll
import org.scalacheck.ScalacheckShapeless._
import org.scalacheck._
import shapeless._
import shapeless.datatype.record._

import scala.reflect.runtime.universe._

object AvroTypeSpec extends Properties("AvroType") {
  import shapeless.datatype.test.Records._
  import shapeless.datatype.test.SerializableUtils._

  implicit def compareByteArrays(x: Array[Byte], y: Array[Byte]) = java.util.Arrays.equals(x, y)
  implicit def compareIntArrays(x: Array[Int], y: Array[Int]) = java.util.Arrays.equals(x, y)

  def roundTrip[A: TypeTag, L <: HList](m: A)(implicit
    gen: LabelledGeneric.Aux[A, L],
    fromL: FromAvroRecord[L],
    toL: ToAvroRecord[L],
    mr: MatchRecord[L]
  ): Boolean = {
    val t = ensureSerializable(AvroType[A])
    val f1: SerializableFunction[A, GenericRecord] =
      new SerializableFunction[A, GenericRecord] {
        override def apply(m: A): GenericRecord = t.toGenericRecord(m)
      }
    val f2: SerializableFunction[GenericRecord, Option[A]] =
      new SerializableFunction[GenericRecord, Option[A]] {
        override def apply(m: GenericRecord): Option[A] = t.fromGenericRecord(m)
      }
    val toFn = ensureSerializable(f1)
    val fromFn = ensureSerializable(f2)
    val copy = fromFn(roundTripRecord(toFn(m)))
    val rm = RecordMatcher[A]
    copy.exists(rm(_, m))
  }

  def roundTripRecord(r: GenericRecord): GenericRecord = {
    val writer = new GenericDatumWriter[GenericRecord](r.getSchema)
    val baos = new ByteArrayOutputStream()
    val encoder = EncoderFactory.get().binaryEncoder(baos, null)
    writer.write(r, encoder)
    encoder.flush()
    baos.close()
    val bytes = baos.toByteArray

    val reader = new GenericDatumReader[GenericRecord](r.getSchema)
    val bais = new ByteArrayInputStream(bytes)
    val decoder = DecoderFactory.get().binaryDecoder(bais, null)
    reader.read(null, decoder)
  }

  implicit val byteStringAvroType = AvroType.at[ByteString](Schema.Type.BYTES)(
    v => ByteString.copyFrom(v.asInstanceOf[ByteBuffer]),
    v => ByteBuffer.wrap(v.toByteArray)
  )
  implicit val instantAvroType =
    AvroType.at[Instant](Schema.Type.LONG)(v => new Instant(v.asInstanceOf[Long]), _.getMillis)
  property("required") = forAll { m: Required => roundTrip(m) }
  property("optional") = forAll { m: Optional => roundTrip(m) }
  property("repeated") = forAll { m: Repeated => roundTrip(m) }
  property("mixed") = forAll { m: Mixed => roundTrip(m) }
  property("nested") = forAll { m: Nested => roundTrip(m) }
  property("seqs") = forAll { m: Seqs => roundTrip(m) }

  implicit val uriAvroType =
    AvroType.at[URI](Schema.Type.STRING)(v => URI.create(v.toString), _.toString)
  property("custom") = forAll { m: Custom => roundTrip(m) }
} 

package shapeless.datatype.bigquery

import java.net.URI

import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
import com.google.api.services.bigquery.model.TableRow
import com.google.common.io.BaseEncoding
import com.google.protobuf.ByteString
import org.joda.time.{Instant, LocalDate, LocalDateTime, LocalTime}
import org.scalacheck.Prop.forAll
import org.scalacheck.ScalacheckShapeless._
import org.scalacheck._
import shapeless._
import shapeless.datatype.record._

import scala.reflect.runtime.universe._

object BigQueryTypeSpec extends Properties("BigQueryType") {
  import shapeless.datatype.test.Records._
  import shapeless.datatype.test.SerializableUtils._

  val mapper = new ObjectMapper().disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)

  implicit def compareByteArrays(x: Array[Byte], y: Array[Byte]) = java.util.Arrays.equals(x, y)
  implicit def compareIntArrays(x: Array[Int], y: Array[Int]) = java.util.Arrays.equals(x, y)

  def roundTrip[A: TypeTag, L <: HList](m: A)(implicit
    gen: LabelledGeneric.Aux[A, L],
    fromL: FromTableRow[L],
    toL: ToTableRow[L],
    mr: MatchRecord[L]
  ): Boolean = {
    BigQuerySchema[A] // FIXME: verify the generated schema
    val t = ensureSerializable(BigQueryType[A])
    val f1: SerializableFunction[A, TableRow] =
      new SerializableFunction[A, TableRow] {
        override def apply(m: A): TableRow = t.toTableRow(m)
      }
    val f2: SerializableFunction[TableRow, Option[A]] =
      new SerializableFunction[TableRow, Option[A]] {
        override def apply(m: TableRow): Option[A] = t.fromTableRow(m)
      }
    val toFn = ensureSerializable(f1)
    val fromFn = ensureSerializable(f2)
    val copy = fromFn(mapper.readValue(mapper.writeValueAsString(toFn(m)), classOf[TableRow]))
    val rm = RecordMatcher[A]
    copy.exists(rm(_, m))
  }

  implicit val byteStringBigQueryMappableType = BigQueryType.at[ByteString]("BYTES")(
    x => ByteString.copyFrom(BaseEncoding.base64().decode(x.toString)),
    x => BaseEncoding.base64().encode(x.toByteArray)
  )
  property("required") = forAll { m: Required => roundTrip(m) }
  property("optional") = forAll { m: Optional => roundTrip(m) }
  property("repeated") = forAll { m: Repeated => roundTrip(m) }
  property("mixed") = forAll { m: Mixed => roundTrip(m) }
  property("nested") = forAll { m: Nested => roundTrip(m) }
  property("seqs") = forAll { m: Seqs => roundTrip(m) }

  implicit val arbDate = Arbitrary(arbInstant.arbitrary.map(i => new LocalDate(i.getMillis)))
  implicit val arbTime = Arbitrary(arbInstant.arbitrary.map(i => new LocalTime(i.getMillis)))
  implicit val arbDateTime = Arbitrary(
    arbInstant.arbitrary.map(i => new LocalDateTime(i.getMillis))
  )

  case class DateTimeTypes(
    instant: Instant,
    date: LocalDate,
    time: LocalTime,
    dateTime: LocalDateTime
  )
  property("date time types") = forAll { m: DateTimeTypes => roundTrip(m) }

  implicit val uriBigQueryType =
    BigQueryType.at[URI]("STRING")(v => URI.create(v.toString), _.toASCIIString)
  property("custom") = forAll { m: Custom => roundTrip(m) }
} 

package shapeless.datatype.bigquery

import com.google.api.services.bigquery.model.{TableFieldSchema, TableSchema}
import org.joda.time.{Instant, LocalDate, LocalDateTime, LocalTime}

import scala.collection.JavaConverters._
import scala.reflect.runtime.universe._

object BigQuerySchema {
  private def isField(s: Symbol): Boolean =
    s.isPublic && s.isMethod && !s.isSynthetic && !s.isConstructor && s.asMethod.isCaseAccessor

  private def isCaseClass(tpe: Type): Boolean =
    !tpe.toString.startsWith("scala.") &&
      List(typeOf[Product], typeOf[Serializable], typeOf[Equals])
        .forall(b => tpe.baseClasses.contains(b.typeSymbol))

  private def rawType(tpe: Type): (String, Iterable[TableFieldSchema]) = tpe match {
    case t if t =:= typeOf[Boolean]       => ("BOOLEAN", Nil)
    case t if t =:= typeOf[Int]           => ("INTEGER", Nil)
    case t if t =:= typeOf[Long]          => ("INTEGER", Nil)
    case t if t =:= typeOf[Float]         => ("FLOAT", Nil)
    case t if t =:= typeOf[Double]        => ("FLOAT", Nil)
    case t if t =:= typeOf[String]        => ("STRING", Nil)
    case t if t =:= typeOf[Array[Byte]]   => ("BYTES", Nil)
    case t if t =:= typeOf[Instant]       => ("TIMESTAMP", Nil)
    case t if t =:= typeOf[LocalDate]     => ("DATE", Nil)
    case t if t =:= typeOf[LocalTime]     => ("TIME", Nil)
    case t if t =:= typeOf[LocalDateTime] => ("DATETIME", Nil)
    case t if isCaseClass(t)              => ("RECORD", toFields(t))
  }

  private def toField(s: Symbol): TableFieldSchema = {
    val name = s.name.toString
    val tpe = s.asMethod.returnType

    val (mode, valType) = tpe match {
      case t if t.erasure =:= typeOf[Option[_]].erasure => ("NULLABLE", t.typeArgs.head)
      case t
          if t.erasure <:< typeOf[Traversable[_]].erasure || (t.erasure <:< typeOf[
            Array[_]
          ] && !(t.typeArgs.head =:= typeOf[
            Byte
          ])) =>
        ("REPEATED", t.typeArgs.head)
      case t => ("REQUIRED", t)
    }
    val (tpeParam, nestedParam) = customTypes.get(valType.toString) match {
      case Some(t) => (t, Nil)
      case None    => rawType(valType)
    }
    val tfs = new TableFieldSchema().setMode(mode).setName(name).setType(tpeParam)
    if (nestedParam.nonEmpty) {
      tfs.setFields(nestedParam.toList.asJava)
    }
    tfs
  }

  private def toFields(t: Type): Iterable[TableFieldSchema] = t.decls.filter(isField).map(toField)

  private val customTypes = scala.collection.mutable.Map[String, String]()
  private val cachedSchemas = scala.collection.concurrent.TrieMap.empty[TypeTag[_], TableSchema]

  private[bigquery] def register(tpe: Type, typeName: String): Unit =
    customTypes += tpe.toString -> typeName

  def apply[T: TypeTag]: TableSchema = {
    val tt = implicitly[TypeTag[T]]
    cachedSchemas.getOrElseUpdate(tt, new TableSchema().setFields(toFields(tt.tpe).toList.asJava))
  }
} 

package com.byteslounge.slickrepo.repository

import java.sql.Timestamp

import com.byteslounge.slickrepo.meta.{Versioned, VersionedEntity}
import org.joda.time.Instant
import slick.ast.BaseTypedType
import com.byteslounge.slickrepo.scalaversion.JdbcProfile
import com.byteslounge.slickrepo.version.JodaTimeVersionImplicits.instantVersionGenerator

case class TestJodaTimeVersionedEntity(override val id: Option[Int], price: Double, override val version: Option[Instant]) extends VersionedEntity[TestJodaTimeVersionedEntity, Int, Instant] {
  def withId(id: Int): TestJodaTimeVersionedEntity = this.copy(id = Some(id))
  def withVersion(version: Instant): TestJodaTimeVersionedEntity = this.copy(version = Some(version))
}

class TestJodaTimeVersionedEntityRepository(override val driver: JdbcProfile) extends VersionedRepository[TestJodaTimeVersionedEntity, Int, Instant](driver) {

  import driver.api._

  implicit val jodaTimeInstantToSqlTimestampMapper = MappedColumnType.base[Instant, Timestamp](
    { instant => new java.sql.Timestamp(instant.getMillis) },
    { sqlTimestamp => new Instant(sqlTimestamp.getTime) })

  val pkType = implicitly[BaseTypedType[Int]]
  val versionType = implicitly[BaseTypedType[Instant]]
  val tableQuery = TableQuery[TestJodaTimeVersionedEntities]
  type TableType = TestJodaTimeVersionedEntities

  class TestJodaTimeVersionedEntities(tag: slick.lifted.Tag) extends Table[TestJodaTimeVersionedEntity](tag, "TJTV_ENTITY") with Versioned[Int, Instant] {
    def id = column[Int]("ID", O.PrimaryKey)
    def price = column[Double]("PRICE")
    def version = column[Instant]("VERSION")

    def * = (id.?, price, version.?) <> ((TestJodaTimeVersionedEntity.apply _).tupled, TestJodaTimeVersionedEntity.unapply)
  }

} 

package com.byteslounge.slickrepo.version

import com.byteslounge.slickrepo.datetime.DateTimeHelper
import org.joda.time.Instant

object JodaTimeVersionImplicits {

  implicit val instantVersionGenerator = new VersionGenerator[Instant]{
    def initialVersion(): Instant = {
      currentInstant()
    }

    def nextVersion(currentVersion: Instant): Instant = {
      currentInstant()
    }

    private def currentInstant(): Instant = {
      new Instant(DateTimeHelper.currentInstant.toEpochMilli)
    }
  }

} 

package org.apache.beam.examples

import java.util.concurrent.ThreadLocalRandom

import org.apache.beam.sdk.Pipeline
import org.apache.beam.sdk.io.TextIO
import org.apache.beam.sdk.options._
import org.apache.beam.sdk.transforms.DoFn.ProcessElement
import org.apache.beam.sdk.transforms.windowing.{FixedWindows, Window}
import org.apache.beam.sdk.transforms.{Count, DoFn, MapElements, ParDo}
import org.joda.time.{Duration, Instant}

object WindowedWordCount {

  def main(args: Array[String]): Unit = {

    val options = PipelineOptionsFactory
      .fromArgs(args: _*)
      .withValidation()
      .as(classOf[WindowedWordCountOptions])

    val minTimestamp = new Instant(options.getMinTimestampMillis)
    val maxTimestamp = new Instant(options.getMaxTimestampMillis)

    val pipeline = Pipeline.create(options)

    pipeline.apply("ReadFiles", TextIO.read().from(options.getInputFile))
      .apply(ParDo.of(new AddTimestampFn(minTimestamp, maxTimestamp)))
      .apply(Window.into[String](FixedWindows.of(Duration.standardMinutes(options.getWindowSize))))
      .apply(ParDo.of(new ExtractWords))
      .apply(Count.perElement())
      .apply(MapElements.via(new FormatResult))
      .apply("WriteWords", TextIO.write()
        .to(options.getOutput)
        .withWindowedWrites()
        .withNumShards(options.getNumShards))

    pipeline.run().waitUntilFinish()

  }
}

// ======================================= Options =============================================

trait WindowedWordCountOptions extends WordCountOptions {
  @Description("Fixed window duration, in minutes")
  @Default.Long(1)
  def getWindowSize: Long
  def setWindowSize(value: Long): Unit

  @Description("Minimum randomly assigned timestamp, in milliseconds-since-epoch")
  @Default.InstanceFactory(classOf[DefaultToCurrentSystemTime])
  def getMinTimestampMillis: Long
  def setMinTimestampMillis(value: Long): Unit

  @Description("Maximum randomly assigned timestamp, in milliseconds-since-epoch")
  @Default.InstanceFactory(classOf[DefaultToMinTimestampPlusOneHour])
  def getMaxTimestampMillis: Long
  def setMaxTimestampMillis(value: Long): Unit

  @Description("Fixed number of shards to produce per window, or null for runner-chosen sharding")
  @Default.Integer(1)
  def getNumShards: Integer
  def setNumShards(numShards: Integer): Unit
}

// ======================================== UDFs ================================================

class AddTimestampFn(minTimestamp: Instant, maxTimestamp: Instant) extends DoFn[String, String] {
  @ProcessElement
  def processElement(c: ProcessContext): Unit = {
    val randomTS = new Instant(ThreadLocalRandom.current.nextLong(minTimestamp.getMillis, maxTimestamp.getMillis))
    c.outputWithTimestamp(c.element(), new Instant(randomTS))
  }
}

// ====================================== Defaults ==============================================

class DefaultToCurrentSystemTime extends DefaultValueFactory[Long] {
  override def create(options: PipelineOptions) = {
    System.currentTimeMillis()
  }
}

class DefaultToMinTimestampPlusOneHour extends DefaultValueFactory[Long] {
  override def create(options: PipelineOptions): Long = {
    options.as(classOf[WindowedWordCountOptions])
      .getMinTimestampMillis + Duration.standardHours(1).getMillis
  }
} 

package models

import org.joda.time.{DateTime, Instant, LocalDate}
import play.api.libs.json._
import _root_.util.DateTimeTools
import play.api.libs.json.JodaWrites._
import play.api.libs.json.JodaReads._

case class Address(
  line1: Option[String],
  line2: Option[String],
  line3: Option[String],
  line4: Option[String],
  line5: Option[String],
  postcode: Option[String],
  country: Option[String],
  startDate: Option[LocalDate],
  endDate: Option[LocalDate],
  `type`: Option[String]
) {
  lazy val lines = List(line1, line2, line3, line4, line5).flatten
  lazy val fullAddress =
    List(line1, line2, line3, line4, line5, postcode.map(_.toUpperCase), internationalAddressCountry(country)).flatten

  val excludedCountries = List(
    Country("GREAT BRITAIN"),
    Country("SCOTLAND"),
    Country("ENGLAND"),
    Country("WALES"),
    Country("NORTHERN IRELAND")
  )

  def internationalAddressCountry(country: Option[String]): Option[String] =
    excludedCountries.contains(Country(country.getOrElse(""))) match {
      case false => country
      case _     => None
    }

  def isWelshLanguageUnit: Boolean = {
    val welshLanguageUnitPostcodes = Set("CF145SH", "CF145TS", "LL499BF", "BX55AB", "LL499AB")
    welshLanguageUnitPostcodes.contains(postcode.getOrElse("").toUpperCase.trim.replace(" ", ""))
  }
}

object Address {
  implicit val localdateFormatDefault = new Format[LocalDate] {
    override def reads(json: JsValue): JsResult[LocalDate] = JodaReads.DefaultJodaLocalDateReads.reads(json)
    override def writes(o: LocalDate): JsValue = JodaWrites.DefaultJodaLocalDateWrites.writes(o)
  }
  implicit val formats = Json.format[Address]
}