scala.reflect.ClassTag Scala Examples

package com.hazelcast.Scala.serialization

import com.hazelcast.nio.serialization.ByteArraySerializer
import com.hazelcast.map.MapInterceptor
import scala.reflect.{ ClassTag, classTag }

private[Scala] final class ByteArrayInterceptor[T <: AnyRef: ByteArraySerializer: ClassTag] extends MapInterceptor {
  private def ser = implicitly[ByteArraySerializer[T]]
  private def tag = classTag[T]

  def interceptGet(value: Object): Object = value match {
    case arr: Array[Byte] => ser.read(arr)
    case _ => value
  }
  def interceptPut(oldVal: Object, newVal: Object): Object = newVal match {
    case t: T => ser.write(t)
    case _ => newVal
  }
  def interceptRemove(value: Object): Object = value match {
    case arr: Array[Byte] => ser.read(arr)
    case _ => value
  }

  def afterGet(v: Object): Unit = ()
  def afterPut(v: Object): Unit = ()
  def afterRemove(v: Object): Unit = ()

  override def hashCode = ser.getTypeId * 37 + classTag[T].hashCode
  override def equals(obj: Any): Boolean = obj match {
    case that: ByteArrayInterceptor[T] =>
      this.ser.getTypeId == that.ser.getTypeId &&
        this.tag == that.tag
    case _ => false
  }

} 

package sigma

import scalan.RType
import scala.reflect.classTag
package types {
  import scalan.{Internal, Nullable}

  import scala.reflect.ClassTag

  trait ViewType[T, Val] extends RType[T] {
    def tVal: RType[Val]
  }

  case class PrimViewType[T, Val](classTag: ClassTag[T], tVal: RType[Val]) extends ViewType[T, Val] {
    override def name: String = tVal.name
    override def isConstantSize: scala.Boolean = tVal.isConstantSize
  }

  object IsPrimView {
    def unapply(pv: PrimView[_]): Nullable[Any] = (pv match {
      case pv: PrimView[_]  => Nullable(pv.value)
      case _ => Nullable.None
    })
  }

}

package object types {
  implicit val booleanRType: RType[Boolean] = PrimViewType[Boolean, scala.Boolean](classTag[Boolean], RType[scala.Boolean])
  implicit val byteRType: RType[Byte] = PrimViewType[Byte, scala.Byte](classTag[Byte], RType[scala.Byte])
  implicit val intRType: RType[Int] = PrimViewType[Int, scala.Int](classTag[Int], RType[scala.Int])
} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{OneToOneDependency, HashPartitioner}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

import org.apache.spark.graphx._

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package tofu.logging.atom
import java.time.Instant
import java.util.concurrent.TimeUnit

import cats.effect.Clock
import cats.{Applicative, FlatMap}
import tofu.concurrent.Atom
import tofu.higherKind.Embed
import tofu.logging.{LoggedValue, Logging, Logs}
import tofu.syntax.monadic._

import scala.reflect.{ClassTag, classTag}

final case class LogLine(
    loggerName: String,
    level: Logging.Level,
    message: String,
    timestamp: Instant,
    values: Vector[LoggedValue],
)

class AtomLogging[F[_]: FlatMap: Clock](log: Atom[F, Vector[LogLine]], name: String) extends Logging[F] {
  override def write(level: Logging.Level, message: String, values: LoggedValue*): F[Unit] =
    Clock[F].realTime(TimeUnit.MILLISECONDS).flatMap { time =>
      log.update(
        _ :+ LogLine(
          loggerName = name,
          level = level,
          message = message,
          timestamp = Instant.ofEpochMilli(time),
          values = values.toVector
        )
      )
    }

}

final case class AtomLogs[I[_]: Applicative, F[_]: FlatMap: Clock](flog: F[Atom[F, Vector[LogLine]]])
    extends Logs[I, F] {
  def forService[Svc: ClassTag]: I[Logging[F]] = byName(classTag[Svc].runtimeClass.getName)
  def byName(name: String): I[Logging[F]]      =
    Embed.of(flog.map[Logging[F]](new AtomLogging[F](_, name))).pure[I]
} 

package tofu.logging.impl
import cats.Monad
import tofu.Guarantee
import tofu.concurrent.QVar
import tofu.logging.{LoggedValue, Logging, Logs}
import tofu.syntax.bracket._
import tofu.syntax.monadic._

import scala.reflect.{ClassTag, classTag}

class CachedLogs[I[_]: Monad: Guarantee, F[_]](
    underlying: Logs[I, F],
    nameCache: QVar[I, Map[String, Logging[F]]],
    tagCache: QVar[I, Map[ClassTag[_], Logging[F]]]
) extends Logs[I, F] {
  private[this] case object NoneLogging extends Logging[F] {
    def write(level: Logging.Level, message: String, values: LoggedValue*): F[Unit] =
      throw new UnsupportedOperationException("CachedLogs.NonLogging should be never used")
  }

  private[this] def safeGet[K](mvar: QVar[I, Map[K, Logging[F]]], create: => I[Logging[F]], key: K): I[Logging[F]] =
    mvar.read.flatMap(_.getOrElse(key, NoneLogging) match {
      case NoneLogging =>
        mvar.take.bracketIncomplete { map =>
          map.getOrElse(key, NoneLogging) match {
            case NoneLogging => create.flatTap(logging => mvar.put(map.updated(key, logging)))
            case logging     => mvar.put(map) as logging
          }
        }(mvar.put)
      case logging     => logging.pure[I]
    })

  def forService[Svc: ClassTag]: I[Logging[F]] = safeGet(tagCache, underlying.forService[Svc], classTag[Svc])
  def byName(name: String): I[Logging[F]]      = safeGet(nameCache, underlying.byName(name), name)
} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

package org.apache.spark.status

import java.io.File

import scala.annotation.meta.getter
import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.{classTag, ClassTag}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.module.scala.DefaultScalaModule

import org.apache.spark.internal.Logging
import org.apache.spark.util.kvstore._

private[spark] object KVUtils extends Logging {

  
  def viewToSeq[T](
      view: KVStoreView[T],
      max: Int)
      (filter: T => Boolean): Seq[T] = {
    val iter = view.closeableIterator()
    try {
      iter.asScala.filter(filter).take(max).toList
    } finally {
      iter.close()
    }
  }

  private[spark] class MetadataMismatchException extends Exception

} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{HashPartitioner, OneToOneDependency}
import org.apache.spark.graphx._
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package akka.grpc.internal

import scala.concurrent.ExecutionContext
import scala.reflect.{ classTag, ClassTag }
import akka.grpc.GrpcClientSettings
import akka.grpc.scaladsl.AkkaGrpcClient
import akka.stream.Materializer

object AkkaGrpcClientFactory {
  def create[T <: AkkaGrpcClient: ClassTag](
      settings: GrpcClientSettings)(implicit mat: Materializer, ex: ExecutionContext): T = {
    // this reflection requires:
    //    object @{service.name}Client {
    //      def apply(GrpcClientSettings)(Materializer, ExecutionContext): @{service.name}Client
    //    }
    val classT: Class[_] = classTag[T].runtimeClass
    val module: AnyRef = getClass.getClassLoader.loadClass(classT.getName + "$").getField("MODULE$").get(null)
    val instance = module
      .asInstanceOf[{ def apply(settings: GrpcClientSettings)(implicit mat: Materializer, ex: ExecutionContext): T }]
    instance(settings)(mat, ex)
  }

  
  def configure[T <: AkkaGrpcClient: ClassTag](
      clientSettings: GrpcClientSettings)(implicit mat: Materializer, ec: ExecutionContext): Configured[T] =
    new Configured[T] {
      def create() = AkkaGrpcClientFactory.create[T](clientSettings)
    }
} 

package special

import java.math.BigInteger

import org.bouncycastle.math.ec.ECPoint
import scalan.RType
import scalan.RType.GeneralType

import scala.reflect.{ClassTag, classTag}

package sigma {

  case class ArgType(override val name: String) extends RType[Any] {
    override def classTag: ClassTag[Any] = ClassTag.Any
    override def isConstantSize: Boolean = false  // pessimistic but safe default
  }
}

package object sigma {

  implicit val BigIntRType: RType[BigInt] = new GeneralType(classTag[BigInt]) {
    override def isConstantSize: Boolean = true
  }
  implicit val GroupElementRType: RType[GroupElement] = new GeneralType(classTag[GroupElement]) {
    override def isConstantSize: Boolean = true
  }
  implicit val SigmaPropRType: RType[SigmaProp] = new GeneralType(classTag[SigmaProp]) {
    override def isConstantSize: Boolean = true
  }
  implicit val AvlTreeRType:   RType[AvlTree]   = new GeneralType(classTag[AvlTree]) {
    override def isConstantSize: Boolean = true
  }

  implicit val BoxRType:       RType[Box]       = GeneralType(classTag[Box])
  implicit val ContextRType:   RType[Context]   = GeneralType(classTag[Context])

  // these are not wrapper types since they are used directly in ErgoTree values (e.g. Constants)
  // and no conversion is necessary
  implicit val HeaderRType: RType[Header] = new GeneralType(classTag[Header]) {
    override def isConstantSize: Boolean = true
  }
  implicit val PreHeaderRType: RType[PreHeader] = new GeneralType(classTag[PreHeader]) {
    override def isConstantSize: Boolean = true
  }

  implicit val AnyValueRType: RType[AnyValue] = RType.fromClassTag(classTag[AnyValue])
  implicit val CostModelRType: RType[CostModel] = RType.fromClassTag(classTag[CostModel])


  implicit val SigmaContractRType: RType[SigmaContract] = RType.fromClassTag(classTag[SigmaContract])
  implicit val SigmaDslBuilderRType: RType[SigmaDslBuilder] = RType.fromClassTag(classTag[SigmaDslBuilder])

  implicit val BigIntegerRType: RType[BigInteger] = new GeneralType(classTag[BigInteger]) {
    override def isConstantSize: Boolean = true
  }
  implicit val ECPointRType: RType[ECPoint] = new GeneralType(classTag[ECPoint]) {
    override def isConstantSize: Boolean = true
  }


  implicit val SizeAnyValueRType: RType[SizeAnyValue] = RType.fromClassTag(classTag[SizeAnyValue])
  implicit val SizeSigmaPropRType: RType[SizeSigmaProp] = RType.fromClassTag(classTag[SizeSigmaProp])
  implicit val SizeBoxRType: RType[SizeBox] = RType.fromClassTag(classTag[SizeBox])
  implicit val SizeContextRType: RType[SizeContext] = RType.fromClassTag(classTag[SizeContext])
  implicit val SizeBuilderRType: RType[SizeBuilder] = RType.fromClassTag(classTag[SizeBuilder])

  def argRType(name: String): RType[Any] = ArgType(name)
} 

package com.hazelcast.Scala.serialization

import com.hazelcast.config.SerializationConfig
import scala.reflect.{ ClassTag, classTag }
import com.hazelcast.nio.serialization.Serializer
import com.hazelcast.config.SerializerConfig
import com.hazelcast.nio.ObjectDataOutput
import com.hazelcast.nio.ObjectDataInput

abstract class SerializerEnum private (offsetOrExtends: Either[Int, Option[SerializerEnum]]) extends Enumeration {
  def this(offset: Int) = this(Left(offset))
  def this(extendFrom: SerializerEnum) = this(Right(Option(extendFrom)))
  def this() = this(Right(None))

  type Value = ClassSerializer[_]

  private val (offset: Int, extendFrom) = offsetOrExtends match {
    case Left(offset) =>
      offset -> None
    case Right(None) =>
      0 -> None
    case Right(Some(extendFrom)) =>
      math.max(0, extendFrom.maxId + 1 + extendFrom.offset) -> Some(extendFrom)
  }

  sealed abstract class ClassSerializer[T: ClassTag] extends Val with Serializer {
    val theClass = classTag[T].runtimeClass.asInstanceOf[Class[T]]
    def register(conf: SerializationConfig): Unit = {
      val serConf = new SerializerConfig
      serConf.setImplementation(this).setTypeClass(theClass)
      conf.addSerializerConfig(serConf)
    }
    final def getTypeId = this.id + 1 + offset
    def destroy = ()
  }

  abstract class ByteArraySerializer[T: ClassTag] extends ClassSerializer[T] with com.hazelcast.nio.serialization.ByteArraySerializer[T]
  abstract class StreamSerializer[T: ClassTag] extends ClassSerializer[T] with com.hazelcast.nio.serialization.StreamSerializer[T]
  class SingletonSerializer[T: ClassTag](singleton: T) extends ClassSerializer[T] with com.hazelcast.nio.serialization.StreamSerializer[T] {
    def write(out: ObjectDataOutput, obj: T) = assert(obj == singleton)
    def read(inp: ObjectDataInput): T = singleton
  }

  def register(conf: SerializationConfig): Unit = {
    extendFrom.foreach(_.register(conf))
    serializers.foreach(_.register(conf))
  }

  def serializers: Iterator[ClassSerializer[_]] = this.values.iterator.map(_.asInstanceOf[ClassSerializer[_]])
} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{OneToOneDependency, HashPartitioner}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

import org.apache.spark.graphx._

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package com.twilio.open.streaming.trend.discovery

import java.io.{ByteArrayInputStream, InputStream}
import java.nio.charset.StandardCharsets

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.google.protobuf.Message
import com.googlecode.protobuf.format.JsonFormat
import com.holdenkarau.spark.testing.{LocalSparkContext, SparkContextProvider}
import com.twilio.open.protocol.Calls.CallEvent
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers, Suite}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.Seq
import scala.io.Source
import scala.reflect.ClassTag
import scala.reflect.classTag

object TestHelper {
  val log: Logger = LoggerFactory.getLogger("com.twilio.open.streaming.trend.discovery.TestHelper")
  val mapper: ObjectMapper = {
    val m = new ObjectMapper()
    m.registerModule(DefaultScalaModule)
  }

  val jsonFormat: JsonFormat = new JsonFormat

  def loadScenario[T<: Message : ClassTag](file: String): Seq[T] = {
    val fileString = Source.fromFile(file).mkString
    val parsed = mapper.readValue(fileString, classOf[Sceanario])
    parsed.input.map { data =>
      val json = mapper.writeValueAsString(data)
      convert[T](json)
    }
  }

  def convert[T<: Message : ClassTag](json: String): T = {
    val clazz = classTag[T].runtimeClass
    val builder = clazz.getMethod("newBuilder").invoke(clazz).asInstanceOf[Message.Builder]
    try {
      val input: InputStream = new ByteArrayInputStream(json.getBytes())
      jsonFormat.merge(input, builder)
      builder.build().asInstanceOf[T]
    } catch {
      case e: Exception =>
        throw e
    }
  }

  def asMockKafkaDataFrame(event: CallEvent): MockKafkaDataFrame = {
    val key = event.getEventId.getBytes(StandardCharsets.UTF_8)
    val value = event.toByteArray
    MockKafkaDataFrame(key, value)
  }

}

case class MockKafkaDataFrame(key: Array[Byte], value: Array[Byte])


@SerialVersionUID(1L)
case class KafkaDataFrame(key: Array[Byte], topic: Array[Byte], value: Array[Byte]) extends Serializable

case class Sceanario(input: Seq[Any], expected: Option[Any] = None)

trait SparkSqlTest extends BeforeAndAfterAll with SparkContextProvider {
  self: Suite =>

  @transient var _sparkSql: SparkSession = _
  @transient private var _sc: SparkContext = _

  override def sc: SparkContext = _sc

  def conf: SparkConf

  def sparkSql: SparkSession = _sparkSql

  override def beforeAll() {
    _sparkSql = SparkSession.builder().config(conf).getOrCreate()

    _sc = _sparkSql.sparkContext
    setup(_sc)
    super.beforeAll()
  }

  override def afterAll() {
    try {
      _sparkSql.close()
      _sparkSql = null
      LocalSparkContext.stop(_sc)
      _sc = null
    } finally {
      super.afterAll()
    }
  }

} 

package io.getquill.codegen.dag

import com.typesafe.scalalogging.Logger
import io.getquill.codegen.dag.dag.ClassAncestry
import io.getquill.codegen.util.MapExtensions._
import org.slf4j.LoggerFactory

import scala.language.implicitConversions
import scala.reflect.{ ClassTag, classTag }

class DagNode(val cls: ClassTag[_], val parent: Option[DagNode])

trait NodeCatalog {
  def lookup(cls: ClassTag[_]): DagNode
}
object DefaultNodeCatalog extends NodeCatalog {

  private val logger = Logger(LoggerFactory.getLogger(this.getClass))

  implicit def nodeToOpt(dagNode: DagNode) = Some(dagNode)

  object StringNode extends DagNode(classTag[String], None)

  object BigDecimalNode extends DagNode(classTag[BigDecimal], StringNode)
  object DoubleNode extends DagNode(classTag[Double], BigDecimalNode)
  object FloatNode extends DagNode(classTag[Float], DoubleNode)

  object LongNode extends DagNode(classTag[Long], BigDecimalNode)
  object IntNode extends DagNode(classTag[Int], LongNode)
  object ShortNode extends DagNode(classTag[Short], IntNode)
  object ByteNode extends DagNode(classTag[Byte], ShortNode)
  object BooleanNode extends DagNode(classTag[Boolean], ByteNode)

  object TimestampNode extends DagNode(classTag[java.time.LocalDateTime], StringNode)
  object DateNode extends DagNode(classTag[java.time.LocalDate], TimestampNode)

  protected[codegen] val nodeCatalogNodes: Seq[DagNode] = Seq(
    StringNode,
    BigDecimalNode,
    DoubleNode,
    FloatNode,
    LongNode,
    IntNode,
    ByteNode,
    ShortNode,
    BooleanNode,
    TimestampNode,
    DateNode
  )

  override def lookup(cls: ClassTag[_]): DagNode = nodeCatalogNodes.find(_.cls == cls).getOrElse({
    logger.warn(s"Could not find type hiearchy node for: ${cls} Must assume it's a string")
    StringNode
  })
}

package object dag {
  type ClassAncestry = (ClassTag[_], ClassTag[_]) => ClassTag[_]
}

class CatalogBasedAncestry(ancestryCatalog: NodeCatalog = DefaultNodeCatalog) extends ClassAncestry {

  def apply(one: ClassTag[_], two: ClassTag[_]): ClassTag[_] = {

    def getAncestry(node: DagNode): List[DagNode] = node.parent match {
      case Some(parent) => node :: getAncestry(parent)
      case None         => node :: Nil
    }

    def commonAncestry = {
      val oneAncestry = getAncestry(ancestryCatalog.lookup(one))
      val twoAncestry = getAncestry(ancestryCatalog.lookup(two))

      val (node, _) =
        oneAncestry.zipWithIndex.toMap.zipOnKeys(twoAncestry.zipWithIndex.toMap)
          .collect { case (key, (Some(i), Some(j))) => (key, i + j) }
          .toList
          .sortBy { case (node, order) => order }
          .head

      node.cls
    }

    // If the two nodes are exactly the same thing, just return the type. Otherwise look up the DAG.
    if (one == two)
      one
    else
      commonAncestry
  }
} 

package io.getquill.codegen

import java.time.LocalDateTime

import io.getquill.codegen.dag.CatalogBasedAncestry
import org.scalatest.BeforeAndAfter
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers._

import scala.reflect.{ ClassTag, classTag }

// I.e. something the type-ancestry does not know about
class UnknownClass

class CodeGeneratorRunnerDagTest extends AnyFunSuite with BeforeAndAfter {

  case class TestCase[O](one: ClassTag[_], twos: Seq[ClassTag[_]], result: ClassTag[_])

  val cases = Seq(
    TestCase(classTag[Int], Seq(classTag[Long]), classTag[Long]),
    TestCase(classTag[Long], Seq(classTag[Boolean], classTag[Int], classTag[Byte], classTag[Long]), classTag[Long]),
    TestCase(classTag[Int], Seq(classTag[Boolean], classTag[Int], classTag[Byte]), classTag[Int]),
    TestCase(
      classTag[BigDecimal],
      Seq(
        classTag[Boolean], classTag[Int], classTag[Byte], classTag[Long], classTag[BigDecimal]
      ),
      classTag[BigDecimal]
    ),
    TestCase(
      classTag[String],
      Seq(
        classTag[Boolean], classTag[Int], classTag[Long], classTag[Byte],
        classTag[BigDecimal], classTag[java.time.LocalDate], classTag[java.time.LocalDateTime]
      ),
      classTag[String]
    ),
    TestCase(classTag[java.time.LocalDate], Seq(classTag[LocalDateTime]), classTag[LocalDateTime]),
    TestCase(classTag[Short], Seq(classTag[Boolean], classTag[Byte]), classTag[Short]),
    TestCase(classTag[Short], Seq(classTag[Int]), classTag[Int]),
    TestCase(classTag[Int], Seq(classTag[Short]), classTag[Int]),
    TestCase(classTag[UnknownClass], Seq(classTag[String]), classTag[String]),
    TestCase(classTag[UnknownClass], Seq(classTag[UnknownClass]), classTag[UnknownClass]),
    // Don't know ancestry of unknown class to an Int (or any kind) so go directly to root of the ancestry i.e. String.
    TestCase(classTag[UnknownClass], Seq(classTag[Int]), classTag[String])
  )

  val casesIter = for {
    cas <- cases
    two <- cas.twos
  } yield (cas.one, two, cas.result)

  casesIter.foreach({
    case (one, two, expected) =>
      test(s"Common Ancestry between ${one} and ${two} should be ${expected}") {
        new CatalogBasedAncestry().apply(one, two) should equal(expected)
      }
  })
} 

package io.getquill.codegen.jdbc.model

import java.sql.Types._

import io.getquill.codegen.model._
import io.getquill.util.ContextLogger

import scala.reflect.{ ClassTag, classTag }

class DefaultJdbcTyper(
  strategy:          UnrecognizedTypeStrategy,
  numericPreference: NumericPreference
) extends (JdbcTypeInfo => Option[ClassTag[_]]) {

  private val logger = ContextLogger(classOf[DefaultJdbcTyper])
  private[getquill] val MaxIntDigits = 9
  private[getquill] val MaxLongDigits = 18

  def unresolvedType(jdbcType: Int, tag: ClassTag[_]): Option[ClassTag[_]] =
    unresolvedType(jdbcType, Some(tag))

  def unresolvedType(jdbcType: Int, tag: Option[ClassTag[_]]): Option[ClassTag[_]] = {
    strategy match {
      case AssumeString => Some(classTag[String])
      case SkipColumn   => None
      case ThrowTypingError =>
        throw new TypingError(s"Could not resolve jdbc type: ${jdbcType}" + tag.map(t => s" class: `${t}`.").getOrElse(""))
    }
  }

  def apply(jdbcTypeInfo: JdbcTypeInfo): Option[ClassTag[_]] = {

    val jdbcType = jdbcTypeInfo.jdbcType

    jdbcType match {
      case CHAR | VARCHAR | LONGVARCHAR | NCHAR | NVARCHAR | LONGNVARCHAR => Some(classTag[String])
      case NUMERIC => numericPreference match {
        case PreferPrimitivesWhenPossible if (jdbcTypeInfo.size <= MaxIntDigits) => Some(classTag[Int])
        case PreferPrimitivesWhenPossible if (jdbcTypeInfo.size <= MaxLongDigits) => Some(classTag[Long])
        case _ => Some(classTag[BigDecimal])
      }
      case DECIMAL                                   => Some(classTag[BigDecimal])
      case BIT | BOOLEAN                             => Some(classTag[Boolean])
      case TINYINT                                   => Some(classTag[Byte])
      case SMALLINT                                  => Some(classTag[Short])
      case INTEGER                                   => Some(classTag[Int])
      case BIGINT                                    => Some(classTag[Long])
      case REAL                                      => Some(classTag[Float])
      case FLOAT | DOUBLE                            => Some(classTag[Double])
      case DATE                                      => Some(classTag[java.time.LocalDate])
      case TIME                                      => Some(classTag[java.time.LocalDateTime])
      case TIMESTAMP                                 => Some(classTag[java.time.LocalDateTime])
      case ARRAY                                     => Some(classTag[java.sql.Array])

      case BINARY | VARBINARY | LONGVARBINARY | BLOB => unresolvedType(jdbcType, classTag[java.sql.Blob])
      case STRUCT                                    => unresolvedType(jdbcType, classTag[java.sql.Struct])
      case REF                                       => unresolvedType(jdbcType, classTag[java.sql.Ref])
      case DATALINK                                  => unresolvedType(jdbcType, classTag[java.net.URL])
      case ROWID                                     => unresolvedType(jdbcType, classTag[java.sql.RowId])
      case NCLOB                                     => unresolvedType(jdbcType, classTag[java.sql.NClob])
      case SQLXML                                    => unresolvedType(jdbcType, classTag[java.sql.SQLXML])
      case NULL                                      => unresolvedType(jdbcType, classTag[Null])

      case CLOB =>
        unresolvedType(jdbcType, classTag[java.sql.Clob])

      case other =>
        unresolvedType(other, None)
    }
  }
} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{OneToOneDependency, HashPartitioner}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

import org.apache.spark.graphx._

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package io.vamp.common.akka

import akka.actor.{ ActorRef, ActorSystem, PoisonPill }
import akka.util.Timeout
import com.typesafe.scalalogging.Logger
import io.vamp.common.{ ClassProvider, Namespace }
import org.slf4j.{ LoggerFactory, MDC }

import scala.concurrent.Future
import scala.reflect.{ ClassTag, classTag }

trait Bootstrap extends BootstrapLogger {

  def start(): Future[Unit] = Future.successful(())

  def stop(): Future[Unit] = Future.successful(())
}

trait ActorBootstrap extends BootstrapLogger {

  private var actors: Future[List[ActorRef]] = Future.successful(Nil)

  def createActors(implicit actorSystem: ActorSystem, namespace: Namespace, timeout: Timeout): Future[List[ActorRef]]

  def start(implicit actorSystem: ActorSystem, namespace: Namespace, timeout: Timeout): Future[Unit] = {
    info(s"Starting ${getClass.getSimpleName}")
    actors = createActors(actorSystem, namespace, timeout)
    actors.map(_ ⇒ ())(actorSystem.dispatcher)
  }

  def restart(implicit actorSystem: ActorSystem, namespace: Namespace, timeout: Timeout): Future[Unit] = {
    stop.flatMap(_ ⇒ start)(actorSystem.dispatcher)
  }

  def stop(implicit actorSystem: ActorSystem, namespace: Namespace): Future[Unit] = {
    info(s"Stopping ${getClass.getSimpleName}")
    actors.map(_.reverse.foreach(_ ! PoisonPill))(actorSystem.dispatcher)
  }

  def alias[T: ClassTag](name: String, default: String ⇒ Future[ActorRef])(implicit actorSystem: ActorSystem, namespace: Namespace, timeout: Timeout): Future[ActorRef] = {
    ClassProvider.find[T](name).map { clazz ⇒
      IoC.alias(classTag[T].runtimeClass, clazz)
      IoC.createActor(clazz)
    } getOrElse default(name)
  }
}

trait BootstrapLogger {

  protected val logger = Logger(LoggerFactory.getLogger(getClass))

  protected def info(message: String)(implicit namespace: Namespace): Unit = {
    MDC.put("namespace", namespace.name)
    try logger.info(message) finally MDC.remove("namespace")
  }
} 

package com.bloomberg.sparkflow.dc


import org.apache.hadoop.fs.Path
import org.apache.spark.SparkContext
import org.apache.spark.sql._

import scala.reflect.{ClassTag, classTag}


object Util {


  private[dc] def saveCheckpoint[T: ClassTag](checkpointPath: String, dataset: Dataset[T]) = {
    assert(dataset != null)
    dataset.write.mode(SaveMode.Overwrite).parquet(checkpointPath)
  }

  private[dc] def loadCheckpoint[T: ClassTag](checkpointPath: String, spark: SparkSession)(implicit tEncoder: Encoder[T]): Option[Dataset[T]] = {
    if (pathExists(checkpointPath, spark.sparkContext)) {
      val dataFrame = spark.read.parquet(checkpointPath)
      val dataset = if (tEncoder.clsTag.equals(classTag[Row])) {
        dataFrame.asInstanceOf[Dataset[T]]
      } else {
        dataFrame.as[T]
      }
      dataset.count()
      Some(dataset)
    } else {
      None
    }
  }

  def pathExists(dir: String, sc: SparkContext) = {
    val path = new Path(dir)
    val fs = path.getFileSystem(sc.hadoopConfiguration)
    fs.exists(path)
  }

  def deletePath(dir: String, sc: SparkContext) = {
    val path = new Path(dir)
    val fs = path.getFileSystem(sc.hadoopConfiguration)
    fs.delete(path, true)
  }

} 

package com.spotify.scio.schemas.instances

import com.spotify.scio.schemas.{RawRecord, Schema}
import org.apache.avro.specific.SpecificRecord
import org.apache.avro.generic.{GenericRecord, IndexedRecord}
import org.apache.beam.sdk.schemas.utils.AvroUtils
import org.apache.beam.sdk.schemas.{AvroRecordSchema, Schema => BSchema}
import org.apache.beam.sdk.transforms.SerializableFunction
import org.apache.beam.sdk.values.{Row, TypeDescriptor}

import scala.jdk.CollectionConverters._
import scala.reflect.{classTag, ClassTag}

trait AvroInstances {
  implicit def avroSchema[T <: SpecificRecord: ClassTag]: Schema[T] = {
    // TODO: broken because of a bug upstream https://issues.apache.org/jira/browse/BEAM-6742
    // RawRecord[T](new AvroRecordSchema())
    import org.apache.avro.reflect.ReflectData
    val rc = classTag[T].runtimeClass.asInstanceOf[Class[T]]
    val provider = new AvroRecordSchema()
    val td = TypeDescriptor.of(rc)
    val schema = provider.schemaFor(td)
    val avroSchema =
      new AvroInstances.SerializableSchema(ReflectData.get().getSchema(td.getRawType))

    def fromRow = provider.fromRowFunction(td)

    val toRow: SerializableFunction[T, Row] =
      new SerializableFunction[T, Row] {
        def apply(t: T): Row =
          AvroInstances.recordtoRow(schema, avroSchema, t)
      }
    RawRecord[T](schema, fromRow, toRow)
  }

  def fromAvroSchema(schema: org.apache.avro.Schema): Schema[GenericRecord] = {
    val beamSchema = AvroUtils.toBeamSchema(schema)
    val avroSchema = new AvroInstances.SerializableSchema(schema)
    val toRow = new SerializableFunction[GenericRecord, Row] {
      def apply(t: GenericRecord): Row =
        AvroInstances.recordtoRow[GenericRecord](beamSchema, avroSchema, t)
    }

    val fromRow = new SerializableFunction[Row, GenericRecord] {
      def apply(t: Row): GenericRecord =
        AvroUtils.toGenericRecord(t, avroSchema.get)
    }

    RawRecord[GenericRecord](beamSchema, fromRow, toRow)
  }
}

object AvroInstances {
  private class SerializableSchema(@transient private val schema: org.apache.avro.Schema)
      extends Serializable {
    private[this] val stringSchema = schema.toString
    def get: org.apache.avro.Schema = new org.apache.avro.Schema.Parser().parse(stringSchema)
  }

  // Workaround BEAM-6742
  private def recordtoRow[T <: IndexedRecord](
    schema: BSchema,
    avroSchema: SerializableSchema,
    t: T
  ): Row = {
    val row = Row.withSchema(schema)
    schema.getFields.asScala.zip(avroSchema.get.getFields.asScala).zipWithIndex.foreach {
      case ((f, a), i) =>
        val value = t.get(i)
        val v = AvroUtils.convertAvroFieldStrict(value, a.schema, f.getType)
        row.addValue(v)
    }
    row.build()
  }
} 

package com.spotify.scio.coders.instances

import java.io.{InputStream, OutputStream}

import com.spotify.scio.coders.{AvroCoderMacros, Coder}
import org.apache.avro.Schema
import org.apache.avro.generic.GenericRecord
import org.apache.avro.specific.{SpecificData, SpecificFixed}
import org.apache.beam.sdk.coders.Coder.NonDeterministicException
import org.apache.beam.sdk.coders.{AtomicCoder, AvroCoder, StringUtf8Coder}
import org.apache.beam.sdk.util.common.ElementByteSizeObserver

import scala.reflect.{classTag, ClassTag}

final private class SlowGenericRecordCoder extends AtomicCoder[GenericRecord] {
  // TODO: can we find something more efficient than String ?
  private[this] val sc = StringUtf8Coder.of()

  override def encode(value: GenericRecord, os: OutputStream): Unit = {
    val schema = value.getSchema
    val coder = AvroCoder.of(schema)
    sc.encode(schema.toString, os)
    coder.encode(value, os)
  }

  override def decode(is: InputStream): GenericRecord = {
    val schemaStr = sc.decode(is)
    val schema = new Schema.Parser().parse(schemaStr)
    val coder = AvroCoder.of(schema)
    coder.decode(is)
  }

  // delegate methods for determinism and equality checks
  override def verifyDeterministic(): Unit =
    throw new NonDeterministicException(
      this,
      "Coder[GenericRecord] without schema is non-deterministic"
    )
  override def consistentWithEquals(): Boolean = false
  override def structuralValue(value: GenericRecord): AnyRef =
    AvroCoder.of(value.getSchema).structuralValue(value)

  // delegate methods for byte size estimation
  override def isRegisterByteSizeObserverCheap(value: GenericRecord): Boolean =
    AvroCoder.of(value.getSchema).isRegisterByteSizeObserverCheap(value)
  override def registerByteSizeObserver(
    value: GenericRecord,
    observer: ElementByteSizeObserver
  ): Unit =
    AvroCoder.of(value.getSchema).registerByteSizeObserver(value, observer)
}


  // TODO: Use a coder that does not serialize the schema
  def avroGenericRecordCoder(schema: Schema): Coder[GenericRecord] =
    Coder.beam(AvroCoder.of(schema))

  // XXX: similar to GenericAvroSerializer
  def avroGenericRecordCoder: Coder[GenericRecord] =
    Coder.beam(new SlowGenericRecordCoder)

  import org.apache.avro.specific.SpecificRecordBase
  implicit def genAvro[T <: SpecificRecordBase]: Coder[T] =
    macro AvroCoderMacros.staticInvokeCoder[T]

  implicit def avroSpecificFixedCoder[T <: SpecificFixed: ClassTag]: Coder[T] =
    SpecificFixedCoder[T]
} 

package org.coursera.common.jsonformat

import play.api.libs.json.Format
import play.api.libs.json.JsError
import play.api.libs.json.OFormat
import play.api.libs.json.OWrites
import play.api.libs.json.Reads
import play.api.libs.json.Writes

import scala.reflect.ClassTag
import scala.reflect.classTag

object OrFormats {

  def unimplementedReads[T: ClassTag]: Reads[T] = {
    Reads(_ => JsError(s"Invoked `unimplementedReads` for ${classTag[T]}"))
  }

  def unimplementedWrites[T: ClassTag]: Writes[T] = Writes { _ =>
    throw new UnsupportedOperationException(s"Invoked `unimplementedOWrites for ${classTag[T]}")
  }

  def unimplementedOWrites[T: ClassTag]: OWrites[T] = OWrites { _ =>
    throw new UnsupportedOperationException(s"Invoked `unimplementedOWrites for ${classTag[T]}")
  }

  def unimplementedFormat[T: ClassTag]: Format[T] = Format(unimplementedReads, unimplementedWrites)

  def unimplementedOFormat[T: ClassTag]: OFormat[T] =
    OFormat(unimplementedReads[T], unimplementedOWrites[T])

  implicit class OrReads[A](reads: Reads[A]) {
    def orReads[B <: A: Reads]: Reads[A] = {
      import play.api.libs.functional.syntax._
      reads or implicitly[Reads[B]].map(b => b: A)
    }
  }

  implicit class OrWrites[A](writes: Writes[A]) {
    def orWrites[B <: A: Writes: ClassTag](implicit classTag: ClassTag[A]): Writes[A] = Writes {
      case b: B => implicitly[Writes[B]].writes(b)
      case a: A => writes.writes(a)
    }
  }

  implicit class OrOWrites[A](oWrites: OWrites[A]) {
    def orOWrites[B <: A: OWrites: ClassTag](implicit classTag: ClassTag[A]): OWrites[A] = OWrites {
      case b: B => implicitly[OWrites[B]].writes(b)
      case a: A => oWrites.writes(a)
    }
  }

  implicit class OrFormat[A](format: Format[A]) {
    def orFormat[B <: A: Format: ClassTag](implicit classTag: ClassTag[A]): Format[A] = {
      Format(format.orReads[B], format.orWrites[B])
    }
  }

  implicit class OrOFormat[A](oFormat: OFormat[A]) {
    def orOFormat[B <: A: OFormat: ClassTag](implicit classTag: ClassTag[A]): OFormat[A] = {
      OFormat(oFormat.orReads[B], oFormat.orOWrites[B])
    }
  }

} 

package org.apache.spark.graphx2.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{OneToOneDependency, HashPartitioner}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

import org.apache.spark.graphx2._

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package com.eharmony.aloha.reflect

import com.eharmony.aloha
import org.reflections.Reflections

import scala.reflect.{classTag, ClassTag}
import scala.util.Try


  protected[this] def scanObjects[OBJ: ClassTag, A: ClassTag](
      methodName: String,
      packageToSearch: String = aloha.pkgName
  ): Seq[A] = {
    val reflections = new Reflections(aloha.pkgName)
    import scala.collection.JavaConversions.asScalaSet
    val objects = reflections.getSubTypesOf(classTag[OBJ].runtimeClass).toSeq

    val suffixLength = objectSuffix.length

    objects.flatMap {
      case o if isObject(o) =>
        Try {
          // This may have some classloading issues.
          val classObj = Class.forName(o.getCanonicalName.dropRight(suffixLength))
          classObj.getMethod(methodName).invoke(null) match {
            case a: A => a
            case _ => throw new IllegalStateException()
          }
        }.toOption
      case _ => None
    }
  }
} 

package com.eharmony.aloha.dataset

import com.eharmony.aloha.dataset.density.{Sparse, Dense}
import com.eharmony.aloha.semantics.func.GenAggFunc

import scala.collection.{immutable => sci}
import scala.reflect.{ClassTag, classTag}

final case class SparseFeatureExtractorFunction[-A](features: sci.IndexedSeq[(String, GenAggFunc[A, Iterable[(String, Double)]])])
extends FeatureExtractorFunction[A, Iterable[(String, Double)]] {
    protected[this] val postProcessingFunction = (name: String, b: Sparse) => b.map(p => (name + p._1, p._2))
    protected[this] implicit def ctB(): ClassTag[Sparse] = classTag[Sparse]
}

final case class DenseFeatureExtractorFunction[-A](features: sci.IndexedSeq[(String, GenAggFunc[A, Double])])
    extends FeatureExtractorFunction[A, Double] {
    protected[this] val postProcessingFunction = (_: String, b: Dense) => b
    protected[this] implicit def ctB(): ClassTag[Dense] = classTag[Dense]
}

final case class StringFeatureExtractorFunction[-A](features: sci.IndexedSeq[(String, GenAggFunc[A, String])])
    extends FeatureExtractorFunction[A, String] {
    protected[this] val postProcessingFunction = (_: String, b: String) => b
    protected[this] implicit def ctB(): ClassTag[String] = classTag[String]
}

final case class StringSeqFeatureExtractorFunction[-A](features: sci.IndexedSeq[(String, GenAggFunc[A, Seq[String]])])
  extends FeatureExtractorFunction[A, Seq[String]] {
  protected[this] val postProcessingFunction = (_: String, b: Seq[String]) => b
  protected[this] implicit def ctB(): ClassTag[Seq[String]] = classTag[Seq[String]]
}


    def apply(a: A): (MissingAndErroneousFeatureInfo, IndexedSeq[Density]) = {
        def h(l: Array[Density], i: Int, n: Int, missing: List[String], erring: List[String]): (MissingAndErroneousFeatureInfo, IndexedSeq[Density]) = {
            if (i >= n)
                (MissingAndErroneousFeatureInfo(missing.reverse, erring.reverse), l)
            else {
                val (name, feature) = features(i)
                // Prefix the feature value tuple key by the feature name.
                l(i) = postProcessingFunction(name, feature(a))
                val problems = feature.accessorOutputProblems(a)
                h(l, i + 1, n,
                    if (problems.missing.nonEmpty) name :: missing else missing,
                    if (problems.errors.nonEmpty) name :: erring else erring)
            }
        }

        val n = features.size
        implicit val b = ctB() // Need this so that the class tag can be found so that the array can be created.
        h(new Array[Density](n), 0, n, Nil, Nil)
    }
} 

package almond.api

import java.util.UUID

import almond.interpreter.api.{CommHandler, OutputHandler}
import jupyter.{Displayer, Displayers}

import scala.reflect.{ClassTag, classTag}

abstract class JupyterApi { api =>

  
  final implicit lazy val publish: OutputHandler =
    new OutputHandler.StableOutputHandler(changingPublish)

  implicit def commHandler: CommHandler =
    throw new Exception("Comm handler not available (not supported)")
  final def comm: CommHandler = commHandler

  protected def updatableResults0: JupyterApi.UpdatableResults

  final lazy val updatableResults: JupyterApi.UpdatableResults =
    updatableResults0

  def register[T: ClassTag](f: T => Map[String, String]): Unit =
    Displayers.register(
      classTag[T].runtimeClass.asInstanceOf[Class[T]],
      new Displayer[T] {
        import scala.collection.JavaConverters._
        def display(t: T) = f(t).asJava
      }
    )
}

object JupyterApi {

  abstract class UpdatableResults {

    @deprecated("Use updatable instead", "0.4.1")
    def addVariable(k: String, v: String): Unit =
      updatable(k, v)
    @deprecated("Use update instead", "0.4.1")
    def updateVariable(k: String, v: String, last: Boolean): Unit =
      update(k, v, last)

    def updatable(k: String, v: String): Unit = {
      // temporary dummy implementation for binary compatibility
    }
    def updatable(v: String): String = {
      val id = UUID.randomUUID().toString
      updatable(id, v)
      id
    }
    def update(k: String, v: String, last: Boolean): Unit = {
      // temporary dummy implementation for binary compatibility
    }
  }

} 

package mass.db.slick

import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.node.NullNode
import com.github.tminglei.slickpg.json.PgJsonExtensions
import com.github.tminglei.slickpg.utils.{ PgCommonJdbcTypes, SimpleArrayUtils }
import com.github.tminglei.slickpg.{ ArraySupport, ExPostgresProfile }
import fusion.json.jackson.ScalaObjectMapper
import helloscala.common.data.NameValue
import mass.model.job.{ JobItem, JobTrigger }
import slick.jdbc._

import scala.language.implicitConversions
import scala.reflect.{ ClassTag, classTag }
import scala.util.Try

trait PgJacksonJsonSupport extends PgJsonExtensions with PgCommonJdbcTypes {
  driver: PostgresProfile with ArraySupport =>
  import driver.api._

  def pgjson: String
  def objectMapper: ScalaObjectMapper

  trait JacksonCodeGenSupport {
    driver match {
      case profile: ExPostgresProfile =>
        profile.bindPgTypeToScala("json", classTag[JsonNode])
        profile.bindPgTypeToScala("jsonb", classTag[JsonNode])
      case _ =>
    }
  }

  trait JsonImplicits extends JacksonImplicits

  trait JacksonImplicits extends JacksonCodeGenSupport {
    implicit val jacksonJsonTypeMapper: JdbcType[JsonNode] = new GenericJdbcType[JsonNode](
      pgjson,
      v => Try(objectMapper.readTree(v)).getOrElse(NullNode.instance),
      v => objectMapper.stringify(v))

    implicit val jacksonArrayTypeMapper: AdvancedArrayJdbcType[JsonNode] =
      new AdvancedArrayJdbcType[JsonNode](
        pgjson,
        s => SimpleArrayUtils.fromString[JsonNode](jstr => objectMapper.readTree(jstr))(s).orNull,
        v => SimpleArrayUtils.mkString[JsonNode](jnode => objectMapper.stringify(jnode))(v))

    implicit val passportColumnType: JdbcType[NameValue] = mkJsonColumnType[NameValue]
    implicit val triggerConfTypeMapper: JdbcType[JobTrigger] = mkJsonColumnType[JobTrigger]
    implicit val jobItemTypeMapper: JdbcType[JobItem] = mkJsonColumnType[JobItem]

    def mkJsonColumnType[T](implicit ev1: ClassTag[T]): JdbcType[T] =
      MappedColumnType
        .base[T, JsonNode](objectMapper.valueToTree, objectMapper.treeToValue(_, ev1.runtimeClass).asInstanceOf[T])

    implicit def jacksonJsonColumnExtensionMethods(c: Rep[JsonNode]): JsonColumnExtensionMethods[JsonNode, JsonNode] =
      new JsonColumnExtensionMethods[JsonNode, JsonNode](c)

    implicit def jacksonJsonOptionColumnExtensionMethods(
        c: Rep[Option[JsonNode]]): JsonColumnExtensionMethods[JsonNode, Option[JsonNode]] =
      new JsonColumnExtensionMethods[JsonNode, Option[JsonNode]](c)
  }

  trait JacksonJsonPlainImplicits extends JacksonCodeGenSupport {
    import com.github.tminglei.slickpg.utils.PlainSQLUtils._
    implicit class PgJacksonJsonPositionResult(r: PositionedResult) {
      def nextJson(): JsonNode = nextJsonOption().getOrElse(NullNode.instance)
      def nextJsonOption(): Option[JsonNode] =
        r.nextStringOption().map(s => Try(objectMapper.readTree(s)).getOrElse(NullNode.instance))
    }

    implicit val getJacksonJson: GetResult[JsonNode] = mkGetResult(_.nextJson())
    implicit val getJacksonJsonOption: GetResult[Option[JsonNode]] = mkGetResult(_.nextJsonOption())
    implicit val setJacksonJson: SetParameter[JsonNode] = mkSetParameter(pgjson, jnode => objectMapper.stringify(jnode))
    implicit val setJacksonJsonOption: SetParameter[Option[JsonNode]] =
      mkOptionSetParameter[JsonNode](pgjson, jnode => objectMapper.stringify(jnode))
  }
} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

import breeze.linalg.DenseVector
import com.github.nearbydelta.deepspark.data._
import com.github.nearbydelta.deepspark.layer.{BasicLayer, NetworkConcatLayer}
import com.github.nearbydelta.deepspark.network.{GeneralNetwork, SimpleNetwork}
import com.github.nearbydelta.deepspark.train.{TrainerBuilder, TrainingParam}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.{SparkConf, SparkContext}

import scala.reflect.{ClassTag, classTag}


object TestConcat {
  def main(args: Array[String]) {
    val conf = new SparkConf().setMaster("local[5]").setAppName("TestXOR")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .set("spark.broadcast.blockSize", "40960")
      .set("spark.akka.frameSize", "50")
    val sc = new SparkContext(conf)

    val data = (0 to 10).collect {
      case i if i > 7 || i < 3 ⇒
        (0 to 10).collect {
          case j if j > 7 || j < 3 ⇒
            val xor =
              if (i > 7 && j > 7) true
              else if (i < 3 && j < 3) true
              else false
            (0 to 10).collect {
              case k if k > 7 || k < 3 ⇒
                (0 to 10).collect {
                  case l if l > 7 || l < 3 ⇒
                    val xor2 =
                      if (i > 7 && j > 7) true
                      else if (i < 3 && j < 3) true
                      else false
                    (Array(DenseVector(i / 10.0, j / 10.0), DenseVector(k / 10.0, l / 10.0)),
                      xor && xor2)
                }
            }.flatMap(x ⇒ x)
        }.flatMap(x ⇒ x)
    }.flatMap(x ⇒ x)

    val train = sc.makeRDD(data)
    val test = train

    try {
      val builder = new AdaGrad(l2decay = 0.00001, rate = 0.01)
      val input1 = new SimpleNetwork[DataVec]()
        .add(new BasicLayer withInput 2 withOutput 4)
        .add(new BasicLayer withInput 4 withOutput 1)
      val input2 = new SimpleNetwork[DataVec]()
        .add(new BasicLayer withInput 2 withOutput 4)
        .add(new BasicLayer withInput 4 withOutput 1)
      val concat = new ConcatLayer().addNetwork(input1).addNetwork(input2)
      val network = new GeneralNetwork[Array[DataVec], Boolean](concat)
        .add(new BasicLayer withInput 2 withOutput 4)
        .add(new BasicLayer withInput 4 withOutput 1)
        .initiateBy(builder)

      require(network.NOut == 1)

      val trained = new TrainerBuilder(TrainingParam(miniBatch = 10, maxIter = 1000, storageLevel = StorageLevel.MEMORY_ONLY))
        .build(network, train, test, SquaredErr, (x: Boolean) ⇒ if (x) DenseVector(1.0) else DenseVector(0.0), "XORTest")
        .getTrainedNetwork

      (0 until 10).foreach { _ ⇒
        val (in, exp) = data(Math.floor(Math.random() * data.length).toInt)
        val out = trained.predictSoft(in)
        println(s"IN : $in, EXPECTED: $exp, OUTPUT $out")
      }
    } finally {
      sc.stop()
    }
  }

  class ConcatLayer extends NetworkConcatLayer[DataVec] {
    override implicit protected val evidenceI: ClassTag[Array[DataVec]] = classTag[Array[DataVec]]
  }

} 

package com.github.nearbydelta.deepspark.word.layer

import breeze.linalg.DenseVector
import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.io.{Input, Output}
import com.github.nearbydelta.deepspark.data._
import com.github.nearbydelta.deepspark.layer.InputLayer
import com.github.nearbydelta.deepspark.word._
import org.apache.spark.SparkContext
import org.apache.spark.broadcast.Broadcast

import scala.reflect.{ClassTag, classTag}


trait Ledger[OutInfo] extends InputLayer[Array[Int], OutInfo] {
  @transient implicit override protected val evidenceI: ClassTag[Array[Int]] = classTag[Array[Int]]
  @transient var algorithm: LedgerAlgorithm = _
  var bcModel: Broadcast[LedgerModel] = _
  @transient var builder: LedgerBuilder = _
  var dimension: Int = 0
  @transient var model: LedgerModel = _
  protected var padID = -1

  def withModel(model: LedgerModel, builder: LedgerBuilder): this.type = {
    this.model = model
    this.builder = builder
    this.padID = model.padID
    this.dimension = model.dimension
    this.algorithm = builder.getUpdater(this.model.vectors)
    this
  }

  protected def pad =
    if (padID == -1) null
    else if (bcModel != null) vectorOf(bcModel.value.padID)
    else vectorOf(padID)

  protected def updateWord(word: Int, dx: DataVec): Unit =
    if (word != -1 && algorithm != null) {
      val vec = algorithm.delta.getOrElseUpdate(word, DenseVector.zeros[Double](dimension))
      vec += dx
    }

  protected def vectorOf(str: Int) =
    if (bcModel != null) bcModel.value.vectorAt(str)
    else model.vectorAt(str)

  override def broadcast(sc: SparkContext): Unit = {
    bcModel = sc.broadcast(model)
  }

  override def loss: Double = algorithm.loss

  override def read(kryo: Kryo, input: Input): Unit = {
    builder = kryo.readClassAndObject(input).asInstanceOf[LedgerBuilder]
    val model = new LedgerModel
    model.read(kryo, input)

    require(model.size > 0, "Model is empty!")
    withModel(model, builder)
    super.read(kryo, input)
  }

  override def unbroadcast(): Unit = {
    bcModel.unpersist(blocking = false)
  }

  @deprecated
  override def withInput(in: Int): this.type = this

  @deprecated
  override def withOutput(out: Int): this.type = this

  override def write(kryo: Kryo, output: Output): Unit = {
    kryo.writeClassAndObject(output, builder)
    model.write(kryo, output)
    super.write(kryo, output)
  }
} 

package com.github.nearbydelta.deepspark.word.layer

import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.io.{Input, Output}
import com.github.nearbydelta.deepspark.data._
import com.github.nearbydelta.deepspark.layer.InputLayer
import com.github.nearbydelta.deepspark.word._
import org.apache.spark.SparkContext
import org.apache.spark.broadcast.Broadcast

import scala.collection.parallel.ParSeq
import scala.reflect.{ClassTag, classTag}


trait FixedLedger[OutInfo] extends InputLayer[Array[Int], OutInfo] {
  @transient implicit override protected val evidenceI: ClassTag[Array[Int]] = classTag[Array[Int]]
  var bcModel: Broadcast[LedgerModel] = _
  @transient var model: LedgerModel = _
  protected var padID = -1

  def withModel(model: LedgerModel): this.type = {
    this.model = model
    this.padID = model.padID
    this
  }

  protected def pad =
    if (padID == -1) null
    else if (bcModel != null) vectorOf(bcModel.value.padID)
    else vectorOf(padID)

  protected def vectorOf(str: Int) =
    if (bcModel != null) bcModel.value.vectorAt(str)
    else model.vectorAt(str)

  override def backprop(seq: ParSeq[((Array[Int], OutInfo), DataVec)]): (ParSeq[DataVec], ParSeq[() ⇒ Unit]) =
    (null, ParSeq())

  override def broadcast(sc: SparkContext): Unit = {
    bcModel = sc.broadcast(model)
  }

  override def loss: Double = 0.0

  override def read(kryo: Kryo, input: Input): Unit = {
    val model = new LedgerModel
    model.read(kryo, input)
    withModel(model)
    super.read(kryo, input)
  }

  override def unbroadcast(): Unit = {
    bcModel.unpersist(blocking = false)
  }

  @deprecated
  override def withInput(in: Int): this.type = this

  @deprecated
  override def withOutput(out: Int): this.type = this

  override def write(kryo: Kryo, output: Output): Unit = {
    model.write(kryo, output)
    super.write(kryo, output)
  }
} 

package com.spotify.featran

import com.spotify.scio.coders.Coder
import com.spotify.scio.values.SCollection

import scala.reflect.{classTag, ClassTag}

package object scio {

  
  implicit object ScioCollectionType extends CollectionType[SCollection] {
    override def map[A, B: ClassTag](ma: SCollection[A])(f: A => B): SCollection[B] = {
      implicit val coder: Coder[B] = Coder.kryo
      ma.map(f)
    }

    override def reduce[A](ma: SCollection[A])(f: (A, A) => A): SCollection[A] = {
      implicit val ct: ClassTag[A] = classTag[Any].asInstanceOf[ClassTag[A]]
      implicit val coder: Coder[A] = Coder.kryo
      ma.reduce(f)
    }

    override def cross[A, B: ClassTag](
      ma: SCollection[A]
    )(mb: SCollection[B]): SCollection[(A, B)] = {
      implicit val ct: ClassTag[A] = classTag[Any].asInstanceOf[ClassTag[A]]
      implicit val coderA: Coder[A] = Coder.kryo
      implicit val coderB: Coder[B] = Coder.kryo
      ma.cross(mb)
    }

    override def pure[A, B: ClassTag](ma: SCollection[A])(b: B): SCollection[B] = {
      implicit val coder: Coder[B] = Coder.kryo
      ma.context.parallelize(Seq(b))
    }
  }
} 

package com.mesosphere.cosmos.circe

import cats.syntax.either._
import com.mesosphere.cosmos.error.JsonDecodingError
import com.mesosphere.cosmos.error.JsonParsingError
import com.mesosphere.cosmos.finch.MediaTypedDecoder
import com.mesosphere.error.Result
import com.mesosphere.error.ResultOps
import com.mesosphere.http.MediaType
import io.lemonlabs.uri.Uri
import io.circe.Decoder
import io.circe.DecodingFailure
import io.circe.Error
import io.circe.Json
import io.circe.ParsingFailure
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.util.Base64
import scala.reflect.ClassTag
import scala.reflect.classTag

object Decoders {

  implicit val decodeUri: Decoder[Uri] = Decoder.decodeString.map(Uri.parse)

  implicit val decodeString: Decoder[String] = {
    Decoder.decodeString.withErrorMessage("String value expected")
  }

  implicit val decodeByteBuffer: Decoder[ByteBuffer] = Decoder.instance { c =>
    c.as[String].bimap(
      { _ => DecodingFailure("Base64 string value expected", c.history) },
      { s => ByteBuffer.wrap(Base64.getDecoder.decode(s)) }
    )
  }

  def decode[T: Decoder: ClassTag](value: String): Result[T] = {
    convertToCosmosError(io.circe.jawn.decode[T](value), value)
  }

  def mediaTypedDecode[T: ClassTag](
    value: String,
    mediaType: MediaType
  )(
    implicit decoder: MediaTypedDecoder[T]
  ): Result[T] = {
    for {
      json <- parse(value)
      result <- decoder(
        json.hcursor,
        mediaType
      )
    } yield result
  }

  def parse(value: String): Result[Json] = {
    convertToCosmosError(io.circe.jawn.parse(value), value)
  }

  def decode64[T: Decoder: ClassTag](value: String): T = {
    decode[T](base64DecodeString(value)).getOrThrow
  }

  def parse64(value: String): Json = {
    parse(base64DecodeString(value)).getOrThrow
  }

  def convertToCosmosError[T: ClassTag](
    result: Either[Error, T],
    inputValue: String
  ): Result[T] = result match {
    case Right(value) => Right(value)
    case Left(ParsingFailure(message, underlying)) =>
      Left(
        JsonParsingError(
          underlying.getClass.getName,
          message,
          inputValue
        )
      )
    case Left(DecodingFailure(message, _)) =>
      Left(
        JsonDecodingError(
          classTag[T].runtimeClass.getName,
          message,
          inputValue
        )
      )
  }

  private[this] def base64DecodeString(value: String): String = {
    new String(Base64.getDecoder.decode(value), StandardCharsets.UTF_8)
  }
} 

package special.collections

import scalan.RType
import special.collection.{Coll, CollBuilder}

import scala.reflect.{ClassTag, classTag}

class Examples(builder: CollBuilder) {
  import builder._
  import Examples._
  import special.collection.ExtensionMethods._
  implicit val longMonoid = Monoids.longPlusMonoid

  def checkTokenBalance(ctx: Context): Boolean = {
    val input = ctx.inputs.flatMap(_.tokens).sumByKey
    val output = ctx.outputs.flatMap(_.tokens).sumByKey
    val flagged = outerJoin(input, output)(
      onlyIn => false,
      onlyOut => false,
      { case (tokenId, (inV, outV)) => inV == outV })
    flagged.forall { case (tokenId, ok) => ok }
  }

  def tokenTotal1(ctx: Context, tokenId: TokenId): Long = {
    val tokenValues = ctx.inputs.flatMap(box => box.tokens.filter(t => t._1 == tokenId).map(t => t._2))
    tokenValues.sum(longMonoid)
  }
  def tokenTotal2(ctx: Context, tokenId: TokenId): Long = {
    val tokenRecords = ctx.inputs.flatMap(box => box.tokens)
    val selectedTokens = tokenRecords.filter(t => t._1 == tokenId)
    selectedTokens.map(_._2).sum(longMonoid)
  }
  def tokenTotal3(ctx: Context, tokenId: TokenId): Long = {
    val tokenValues = ctx.inputs.map { box =>
      val optToken = box.tokens.find(t => t._1 == tokenId)
      optToken.map(t => t._2).getOrElse(0L)
    }
    tokenValues.sum(longMonoid)
  }
}

object Examples {
  type IdType = Coll[Byte]
  type TokenId = IdType
  type TokenData = (TokenId, Long)
  case class Box(id: IdType, tokens: Coll[TokenData]) {
    override def toString: String = {
      val idVal = id.toArray(0)
      val tokenStr = tokens.map(t => s"${t._1.toArray(0)}->${t._2}").toArray.mkString("[", ";", "]")
      s"Box{id=$idVal; tokens=$tokenStr}"
    }
  }
  implicit val boxType = RType.fromClassTag(classTag[Box])
  case class Context(inputs: Coll[Box], outputs: Coll[Box], selfBoxIndex: Int) {
    override def toString: String = {
      s"""Ctx {
        |  inputs=[
        |    ${inputs.toArray.mkString(";\n    ")}
        |  ]
        |  outputs=[
        |    ${outputs.toArray.mkString(";\n    ")}
        |  ]
        |  self=$selfBoxIndex
        |}
       """.stripMargin
    }
  }
} 

package special

import scalan.RType
import scala.reflect.{ClassTag, classTag}

package collection {
  case class CollType[A](tItem: RType[A]) extends RType[Coll[A]] {
    val classTag: ClassTag[Coll[A]] = ClassTag[Coll[A]](classOf[Coll[A]])
    override def name: String = s"Coll[${tItem.name}]"
    override def isConstantSize: Boolean = false
  }
  case class ReplCollType[A](tItem: RType[A]) extends RType[ReplColl[A]] {
    val classTag: ClassTag[ReplColl[A]] = ClassTag[ReplColl[A]](classOf[ReplColl[A]])
    override def name: String = s"ReplColl[${tItem.name}]"
    override def isConstantSize: Boolean = false
  }

  case class SizeType[A](tVal: RType[A]) extends RType[Size[A]] {
    val classTag: ClassTag[Size[A]] = ClassTag[Size[A]](classOf[Size[A]])
    override def name: String = s"Size[${tVal.name}]"
    override def isConstantSize: Boolean = tVal.isConstantSize
  }
  case class SizePrimType[A](tVal: RType[A]) extends RType[SizePrim[A]] {
    val classTag: ClassTag[SizePrim[A]] = ClassTag[SizePrim[A]](classOf[SizePrim[A]])
    override def name: String = s"SizePrim[${tVal.name}]"
    override def isConstantSize: Boolean = tVal.isConstantSize
  }
  case class SizePairType[A,B](tFst: RType[A], tSnd: RType[B]) extends RType[SizePair[A, B]] {
    val classTag: ClassTag[SizePair[A, B]] = ClassTag[SizePair[A, B]](classOf[SizePair[A, B]])
    override def name: String = s"SizePair[${tFst.name},${tSnd.name}]"
    override def isConstantSize: Boolean = tFst.isConstantSize && tSnd.isConstantSize
  }
  case class SizeCollType[A](tItem: RType[A]) extends RType[SizeColl[A]] {
    val classTag: ClassTag[SizeColl[A]] = ClassTag[SizeColl[A]](classOf[SizeColl[A]])
    override def name: String = s"SizeColl[${tItem.name}]"
    override def isConstantSize: Boolean = tItem.isConstantSize
  }
  case class SizeFuncType[E,A,B](tEnv: RType[E], tDom: RType[A], tRange: RType[B]) extends RType[SizeFunc[E, A, B]] {
    val classTag: ClassTag[SizeFunc[E, A, B]] = ClassTag[SizeFunc[E, A, B]](classOf[SizeFunc[E, A, B]])
    override def name: String = s"SizeFunc[${tEnv.name},${tDom.name},${tRange.name}]"
    override def isConstantSize: Boolean = false
  }
  case class SizeOptionType[A](tItem: RType[A]) extends RType[SizeOption[A]] {
    val classTag: ClassTag[SizeOption[A]] = ClassTag[SizeOption[A]](classOf[SizeOption[A]])
    override def name: String = s"SizeOption[${tItem.name}]"
    override def isConstantSize: Boolean = tItem.isConstantSize
  }
}

package object collection {
  implicit def collRType[A](implicit tA: RType[A]): RType[Coll[A]] = CollType[A](tA)
  implicit def extendCollType[A](ct: RType[Coll[A]]): CollType[A] = ct.asInstanceOf[CollType[A]]

  implicit def replCollRType[A](implicit tA: RType[A]): RType[ReplColl[A]] = ReplCollType[A](tA)

  implicit val collBuilderRType: RType[CollBuilder] = RType.fromClassTag(classTag[CollBuilder])

  implicit def sizeRType[A](implicit tA: RType[A]): RType[Size[A]] = SizeType[A](tA)
  implicit def extendSizeType[A](ct: RType[Size[A]]): SizeType[A] = ct.asInstanceOf[SizeType[A]]

  implicit def sizePrimRType[A](implicit tA: RType[A]): RType[SizePrim[A]] = SizePrimType[A](tA)
  implicit def extendSizePrimType[A](ct: RType[SizePrim[A]]): SizePrimType[A] = ct.asInstanceOf[SizePrimType[A]]
  
  implicit def sizePairRType[A, B](implicit tA: RType[A], tB: RType[B]): RType[SizePair[A, B]] = SizePairType[A, B](tA, tB)
  implicit def extendSizePairType[A, B](ct: RType[SizePair[A,B]]): SizePairType[A, B] = ct.asInstanceOf[SizePairType[A, B]]

  implicit def sizeCollRType[A](implicit tA: RType[A]): RType[SizeColl[A]] = SizeCollType[A](tA)
  implicit def extendSizeCollType[A](ct: RType[SizeColl[A]]): SizeCollType[A] = ct.asInstanceOf[SizeCollType[A]]

  implicit def sizeFuncType[E, A, B](implicit tE: RType[E], tA: RType[A], tB: RType[B]): RType[SizeFunc[E, A, B]] = SizeFuncType[E, A, B](tE, tA, tB)
  implicit def extendSizeFuncType[E, A, B](ct: RType[SizeFunc[E,A,B]]): SizeFuncType[E, A, B] = ct.asInstanceOf[SizeFuncType[E, A, B]]
  
  implicit def sizeOptionRType[A](implicit tA: RType[A]): RType[SizeOption[A]] = SizeOptionType[A](tA)
  implicit def extendSizeOptionType[A](ct: RType[SizeOption[A]]): SizeOptionType[A] = ct.asInstanceOf[SizeOptionType[A]]

} 

package scalan.util

import java.lang.reflect.{Method, AnnotatedElement}

import scala.reflect.{classTag, ClassTag}
import scalan.OverloadId

object ReflectionUtil {
  def jAnnotation[A <: java.lang.annotation.Annotation : ClassTag](element: AnnotatedElement) =
    Option(element.getAnnotation(classTag[A].runtimeClass.asInstanceOf[Class[A]]))

  def overloadId(method: Method) = jAnnotation[OverloadId](method).map(_.value)

  
  def namedSuperclass(clazz: Class[_]) = {
    if (clazz.getSimpleName.contains("$anon$")) {
      val superclass = clazz.getSuperclass
      if (superclass == classOf[Object]) {
        // clazz is composed of traits only, return the first one
        clazz.getInterfaces.head
      } else
        superclass
    } else
      clazz
  }

  implicit class ClassOps(val cl: Class[_]) extends AnyVal {
    private def isSpecialChar(c: Char): Boolean = {
      ('0' <= c && c <= '9') || c == '$'
    }
    def safeSimpleName: String = {
      if (cl.getEnclosingClass == null) return cl.getSimpleName
      val simpleName = cl.getName.substring(cl.getEnclosingClass.getName.length)
      val length = simpleName.length
      var index = 0
      while (index < length && isSpecialChar(simpleName.charAt(index))) { index += 1 }
      // Eventually, this is the empty string iff this is an anonymous class
      simpleName.substring(index)
    }
  }
} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{OneToOneDependency, HashPartitioner, TaskContext}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

import org.apache.spark.graphx._

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package org.apache.livy.server.recovery

import scala.reflect.{classTag, ClassTag}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule

import org.apache.livy.{LivyConf, Logging}
import org.apache.livy.server.recovery.ZooKeeperManager
import org.apache.livy.sessions.SessionKindModule
import org.apache.livy.sessions.SessionManager._

protected trait JsonMapper {
  protected val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .registerModule(new SessionKindModule())

  def serializeToBytes(value: Object): Array[Byte] = mapper.writeValueAsBytes(value)

  def deserialize[T: ClassTag](json: Array[Byte]): T =
    mapper.readValue(json, classTag[T].runtimeClass.asInstanceOf[Class[T]])
}


object StateStore extends Logging {
  private[this] var stateStore: Option[StateStore] = None

  def init(livyConf: LivyConf, zkManager: Option[ZooKeeperManager] = None): Unit = synchronized {
    if (stateStore.isEmpty) {
      val fileStateStoreClassTag = pickStateStore(livyConf)
      if (fileStateStoreClassTag == classOf[ZooKeeperStateStore]) {
        stateStore = Option(fileStateStoreClassTag.
          getDeclaredConstructor(classOf[LivyConf], classOf[ZooKeeperManager])
          .newInstance(livyConf, zkManager.get).asInstanceOf[StateStore])
      } else {
        stateStore = Option(fileStateStoreClassTag.getDeclaredConstructor(classOf[LivyConf])
          .newInstance(livyConf).asInstanceOf[StateStore])
      }
      info(s"Using ${stateStore.get.getClass.getSimpleName} for recovery.")
    }
  }

  def cleanup(): Unit = synchronized {
    stateStore = None
  }

  def get: StateStore = {
    assert(stateStore.isDefined, "StateStore hasn't been initialized.")
    stateStore.get
  }

  private[recovery] def pickStateStore(livyConf: LivyConf): Class[_] = {
    livyConf.get(LivyConf.RECOVERY_MODE) match {
      case SESSION_RECOVERY_MODE_OFF => classOf[BlackholeStateStore]
      case SESSION_RECOVERY_MODE_RECOVERY =>
        livyConf.get(LivyConf.RECOVERY_STATE_STORE) match {
          case "filesystem" => classOf[FileSystemStateStore]
          case "zookeeper" => classOf[ZooKeeperStateStore]
          case ss => throw new IllegalArgumentException(s"Unsupported state store $ss")
        }
      case rm => throw new IllegalArgumentException(s"Unsupported recovery mode $rm")
    }
  }
} 

package com.github.aseigneurin.kafka.serialization.scala

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.typesafe.scalalogging.LazyLogging

import scala.reflect.{ClassTag, classTag}

class JsonSerde[T >: Null : ClassTag] extends BaseSerde[T] with LazyLogging {

  val mapper = new ObjectMapper
  mapper.registerModule(DefaultScalaModule)

  override def deserialize(topic: String, data: Array[Byte]): T = data match {
    case null => null
    case _ =>
      try {
        mapper.readValue(data, classTag[T].runtimeClass.asInstanceOf[Class[T]])
      } catch {
        case e: Exception =>
          val jsonStr = new String(data, "UTF-8")
          logger.warn(s"Failed parsing ${jsonStr}", e)
          null
      }
  }

  override def serialize(topic: String, obj: T): Array[Byte] = {
    mapper.writeValueAsBytes(obj)
  }

} 

package io.github.netvl.picopickle.backends.mongodb

import java.util.Date

import _root_.io.github.netvl.picopickle.{TypesComponent, DefaultPickler, ExceptionsComponent, BackendComponent}
import org.bson._
import org.bson.types.ObjectId

import scala.reflect.{ClassTag, classTag}

trait MongodbBsonBackendComponent extends BackendComponent {
  override val backend = MongodbBsonBackend
}

trait MongodbBsonSerializersComponent {
  this: MongodbBsonBackendComponent with TypesComponent =>

  private def identityBsonReadWriter[T <: backend.BValue : ClassTag] =
    ReadWriter.writing[T](identity).reading { case value: T => value }
      .orThrowing(whenReading = classTag[T].runtimeClass.getSimpleName, expected = classTag[T].runtimeClass.getSimpleName)

  implicit val bsonValueReadWriter: ReadWriter[BsonValue] =
    ReadWriter.writing[BsonValue](identity).reading(PartialFunction(identity))

  implicit val bsonDocumentReadWriter: ReadWriter[BsonDocument] = identityBsonReadWriter[BsonDocument]
  implicit val bsonArrayReadWriter: ReadWriter[BsonArray] = identityBsonReadWriter[BsonArray]
  implicit val bsonStringReadWriter: ReadWriter[BsonString] = identityBsonReadWriter[BsonString]
  implicit val bsonNumberReadWriter: ReadWriter[BsonNumber] = identityBsonReadWriter[BsonNumber]
  implicit val bsonBooleanReadWriter: ReadWriter[BsonBoolean] = identityBsonReadWriter[BsonBoolean]
  implicit val bsonNullReadWriter: ReadWriter[BsonNull] = identityBsonReadWriter[BsonNull]
  implicit val bsonObjectIdReadWriter: ReadWriter[BsonObjectId] = identityBsonReadWriter[BsonObjectId]
  implicit val bsonInt32ReadWriter: ReadWriter[BsonInt32] = identityBsonReadWriter[BsonInt32]
  implicit val bsonInt64ReadWriter: ReadWriter[BsonInt64] = identityBsonReadWriter[BsonInt64]
  implicit val bsonDoubleReadWriter: ReadWriter[BsonDouble] = identityBsonReadWriter[BsonDouble]
  implicit val bsonDateTimeReadWriter: ReadWriter[BsonDateTime] = identityBsonReadWriter[BsonDateTime]
  implicit val bsonBinaryReadWriter: ReadWriter[BsonBinary] = identityBsonReadWriter[BsonBinary]
  implicit val bsonSymbolReadWriter: ReadWriter[BsonSymbol] = identityBsonReadWriter[BsonSymbol]

  // TODO: add a test for this
  implicit val dateReadWriter: ReadWriter[Date] = ReadWriter.writing[Date](d => backend.makeDateTime(d.getTime))
    .reading {
      case backend.BsonExtract.DateTime(ts) => new Date(ts)
    }.orThrowing(whenReading = "date", expected = "datetime")

  implicit val symbolReadWriter: ReadWriter[Symbol] = ReadWriter.writing(backend.makeSymbol)
    .reading {
      case backend.BsonExtract.Symbol(sym) => sym
    }.orThrowing(whenReading = "symbol", expected = "symbol")

  implicit val binaryReadWriter: ReadWriter[Array[Byte]] = ReadWriter.writing(backend.makeBinary)
    .reading {
      case backend.BsonExtract.Binary(arr) => arr
    }.orThrowing(whenReading = "array of bytes", expected = "binary")

  implicit val intReadWriter: ReadWriter[Int] = ReadWriter.writing(backend.makeInt32)
    .reading {
      case backend.BsonExtract.Int32(n) => n
    }.orThrowing(whenReading = "int", expected = "32-bit integer")

  implicit val longReadWriter: ReadWriter[Long] = ReadWriter.writing(backend.makeInt64)
    .reading {
      case backend.BsonExtract.Int64(n) => n
    }.orThrowing(whenReading = "long", expected = "64-bit integer")

  implicit val doubleReadWriter: ReadWriter[Double] = ReadWriter.writing(backend.makeDouble)
    .reading {
      case backend.BsonExtract.Double(n) => n
    }.orThrowing(whenReading = "double", expected = "double")

  implicit val objectIdReadWriter: ReadWriter[ObjectId] = ReadWriter.writing(backend.makeObjectId)
    .reading {
      case backend.BsonExtract.ObjectId(oid) => oid
    }.orThrowing(whenReading = "object id", expected = "object id")
}

trait MongodbBsonPickler
  extends DefaultPickler
  with MongodbBsonBackendComponent
  with MongodbBsonSerializersComponent

object MongodbBsonPickler extends MongodbBsonPickler 

package com.amazon.milan.compiler.flink.runtime

import java.util

import com.amazon.milan.compiler.flink.generator.FlinkGeneratorException
import com.amazon.milan.serialization.MilanObjectMapper
import com.fasterxml.jackson.databind.`type`.TypeFactory
import org.apache.flink.api.common.typeinfo.TypeInformation

import scala.collection.JavaConverters._
import scala.reflect.{ClassTag, classTag}


object RuntimeUtil {
  val typeName: String = getClass.getTypeName.stripSuffix("$")

  def loadJsonList[TElement: ClassTag](listJson: String): List[TElement] = {
    this.loadJsonArrayList[TElement](listJson).asScala.toList
  }

  def loadJsonArrayList[TElement: ClassTag](listJson: String): util.ArrayList[TElement] = {
    val typeFactory = TypeFactory.defaultInstance()
    val itemClass = classTag[TElement].runtimeClass.asInstanceOf[Class[TElement]]
    val javaType = typeFactory.constructCollectionType(classOf[util.ArrayList[TElement]], itemClass)
    MilanObjectMapper.readValue[util.ArrayList[TElement]](listJson, javaType)
  }

  def preventGenericTypeInformation[T](typeInfo: TypeInformation[T]): TypeInformation[T] = {
    if (typeInfo.getClass.getName.contains("__wrapper")) {
      throw new FlinkGeneratorException(s"Creating TypeInformation for '${typeInfo.getTypeClass.getName}' produced a GenericTypeInformation.")
    }

    typeInfo
  }
} 

package com.amazon.milan.compiler.flink.serialization

import com.amazon.milan.serialization.MilanObjectMapper
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.TypeExtractor
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema

import scala.reflect.{ClassTag, classTag}


object JsonDeserializationSchema {
  private val objectMapper = new MilanObjectMapper()
}



class JsonDeserializationSchema[T: ClassTag] extends KinesisDeserializationSchema[T] with Serializable {
  override def deserialize(bytes: Array[Byte],
                           partitionKey: String,
                           seqNum: String,
                           approxArrivalTimestamp: Long,
                           stream: String,
                           shardId: String): T = {
    JsonDeserializationSchema.objectMapper.readValue[T](bytes, classTag[T].runtimeClass.asInstanceOf[Class[T]])
  }

  override def getProducedType: TypeInformation[T] = {
    TypeExtractor.getForClass(classTag[T].runtimeClass.asInstanceOf[Class[T]])
  }
} 

package org.apache.openwhisk.core.database.cosmosdb

import org.scalatest.FlatSpec
import org.apache.openwhisk.core.database.test.behavior.ArtifactStoreBehaviorBase
import org.apache.openwhisk.core.database.{ArtifactStore, AttachmentStore, DocumentSerializer}
import org.apache.openwhisk.core.entity.{
  DocumentReader,
  WhiskActivation,
  WhiskAuth,
  WhiskDocumentReader,
  WhiskEntity,
  WhiskEntityJsonFormat
}

import scala.reflect.{classTag, ClassTag}
import scala.util.Try

trait CosmosDBStoreBehaviorBase extends FlatSpec with ArtifactStoreBehaviorBase with CosmosDBTestSupport {
  override def storeType = "CosmosDB"

  override lazy val storeAvailableCheck: Try[Any] = storeConfigTry

  protected lazy val config: CosmosDBConfig = adaptCosmosDBConfig(storeConfig.copy(db = createTestDB().getId))

  override lazy val authStore = {
    implicit val docReader: DocumentReader = WhiskDocumentReader
    CosmosDBArtifactStoreProvider.makeArtifactStore[WhiskAuth](config, getAttachmentStore[WhiskAuth]())
  }

  override lazy val entityStore =
    CosmosDBArtifactStoreProvider.makeArtifactStore[WhiskEntity](config, getAttachmentStore[WhiskEntity]())(
      classTag[WhiskEntity],
      WhiskEntityJsonFormat,
      WhiskDocumentReader,
      actorSystem,
      logging,
      materializer)

  override lazy val activationStore = {
    implicit val docReader: DocumentReader = WhiskDocumentReader
    CosmosDBArtifactStoreProvider.makeArtifactStore[WhiskActivation](config, getAttachmentStore[WhiskActivation]())
  }

  override protected def getAttachmentStore(store: ArtifactStore[_]) =
    store.asInstanceOf[CosmosDBArtifactStore[_]].attachmentStore

  protected def getAttachmentStore[D <: DocumentSerializer: ClassTag](): Option[AttachmentStore] = None

  protected def adaptCosmosDBConfig(config: CosmosDBConfig): CosmosDBConfig = config
} 

package org.apache.openwhisk.core.database

import org.scalatest.FlatSpec
import org.apache.openwhisk.core.database.test.behavior.ArtifactStoreBehaviorBase
import org.apache.openwhisk.core.entity.{
  DocumentReader,
  WhiskActivation,
  WhiskAuth,
  WhiskDocumentReader,
  WhiskEntity,
  WhiskEntityJsonFormat
}

import scala.reflect.{classTag, ClassTag}

trait CouchDBStoreBehaviorBase extends FlatSpec with ArtifactStoreBehaviorBase {
  override def storeType = "CouchDB"

  override val authStore = {
    implicit val docReader: DocumentReader = WhiskDocumentReader
    CouchDbStoreProvider.makeArtifactStore[WhiskAuth](useBatching = false, getAttachmentStore[WhiskAuth]())
  }

  override val entityStore =
    CouchDbStoreProvider.makeArtifactStore[WhiskEntity](useBatching = false, getAttachmentStore[WhiskEntity]())(
      classTag[WhiskEntity],
      WhiskEntityJsonFormat,
      WhiskDocumentReader,
      actorSystem,
      logging,
      materializer)

  override val activationStore = {
    implicit val docReader: DocumentReader = WhiskDocumentReader
    CouchDbStoreProvider.makeArtifactStore[WhiskActivation](useBatching = true, getAttachmentStore[WhiskActivation]())
  }

  override protected def getAttachmentStore(store: ArtifactStore[_]) =
    store.asInstanceOf[CouchDbRestStore[_]].attachmentStore

  protected def getAttachmentStore[D <: DocumentSerializer: ClassTag](): Option[AttachmentStore] = None
} 

package org.apache.openwhisk.core.database.memory

import org.scalatest.FlatSpec
import org.apache.openwhisk.core.database.{ArtifactStore, AttachmentStore, DocumentSerializer}
import org.apache.openwhisk.core.database.test.behavior.ArtifactStoreBehaviorBase
import org.apache.openwhisk.core.entity.{
  DocumentReader,
  WhiskActivation,
  WhiskAuth,
  WhiskDocumentReader,
  WhiskEntity,
  WhiskEntityJsonFormat
}

import scala.reflect.{classTag, ClassTag}

trait MemoryArtifactStoreBehaviorBase extends FlatSpec with ArtifactStoreBehaviorBase {
  override def storeType = "Memory"

  override lazy val authStore = {
    implicit val docReader: DocumentReader = WhiskDocumentReader
    MemoryArtifactStoreProvider.makeArtifactStore[WhiskAuth](getAttachmentStore[WhiskAuth]())
  }

  override protected def beforeAll(): Unit = {
    MemoryArtifactStoreProvider.purgeAll()
    super.beforeAll()
  }

  override lazy val entityStore =
    MemoryArtifactStoreProvider.makeArtifactStore[WhiskEntity](getAttachmentStore[WhiskEntity]())(
      classTag[WhiskEntity],
      WhiskEntityJsonFormat,
      WhiskDocumentReader,
      actorSystem,
      logging,
      materializer)

  override lazy val activationStore = {
    implicit val docReader: DocumentReader = WhiskDocumentReader
    MemoryArtifactStoreProvider.makeArtifactStore[WhiskActivation](getAttachmentStore[WhiskActivation]())
  }

  override protected def getAttachmentStore(store: ArtifactStore[_]) =
    Some(store.asInstanceOf[MemoryArtifactStore[_]].attachmentStore)

  protected def getAttachmentStore[D <: DocumentSerializer: ClassTag](): AttachmentStore =
    MemoryAttachmentStoreProvider.makeStore()
} 

package com.kakao.shaded.jackson
package module.scala
package deser

import com.kakao.shaded.jackson.core.JsonToken.{START_ARRAY, VALUE_NUMBER_FLOAT, VALUE_NUMBER_INT, VALUE_STRING}
import com.kakao.shaded.jackson.core.{JsonParser, JsonToken}
import com.kakao.shaded.jackson.databind.{JsonDeserializer, BeanDescription, DeserializationConfig, JavaType, DeserializationContext}
import com.kakao.shaded.jackson.databind.DeserializationFeature.UNWRAP_SINGLE_VALUE_ARRAYS
import com.kakao.shaded.jackson.databind.deser.Deserializers
import com.kakao.shaded.jackson.databind.deser.std.StdScalarDeserializer

import scala.reflect.ClassTag
import scala.reflect.classTag

private abstract class BigNumberDeserializer[T >: Null : ClassTag](creator: (String) => T)
  extends StdScalarDeserializer[T](classTag[T].runtimeClass)
{
  override def deserialize(jp: JsonParser, ctxt: DeserializationContext): T = {
    val t = jp.getCurrentToken
    t match {
      case VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT => creator(jp.getText.trim)
      case VALUE_STRING =>
        val text = jp.getText.trim
        if (text.isEmpty) null else try {
          creator(text)
        }
        catch {
          case e: IllegalArgumentException => throw ctxt.weirdStringException(text, _valueClass, "not a valid representation")
        }
      case START_ARRAY if ctxt.isEnabled(UNWRAP_SINGLE_VALUE_ARRAYS) =>
        jp.nextToken()
        val value = deserialize(jp, ctxt)
        if (jp.nextToken() != JsonToken.END_ARRAY) {
          throw ctxt.wrongTokenException(jp, JsonToken.END_ARRAY, "Attempted to unwrap array for single value but there was more than a single value in the array")
        }
        value
      case _ =>
        throw ctxt.mappingException(_valueClass, t)
    }
  }
}

private object BigDecimalDeserializer extends BigNumberDeserializer(BigDecimal.apply)

private object BigIntDeserializer extends BigNumberDeserializer(BigInt.apply)

private object NumberDeserializers extends Deserializers.Base
{
  val BigDecimalClass = BigDecimalDeserializer.handledType()
  val BigIntClass = BigIntDeserializer.handledType()

  override def findBeanDeserializer(tpe: JavaType, config: DeserializationConfig, beanDesc: BeanDescription): JsonDeserializer[_] =
    tpe.getRawClass match {
      case BigDecimalClass => BigDecimalDeserializer
      case BigIntClass => BigIntDeserializer
      case _ => null
    }
}

private [scala] trait ScalaNumberDeserializersModule extends JacksonModule {
  this += NumberDeserializers
} 

package org.apache.livy.server.recovery

import scala.reflect.classTag

import org.scalatest.{BeforeAndAfter, FunSpec}
import org.scalatest.Matchers._

import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
import org.apache.livy.sessions.SessionManager

class StateStoreSpec extends FunSpec with BeforeAndAfter with LivyBaseUnitTestSuite {
  describe("StateStore") {
    after {
      StateStore.cleanup()
    }

    def createConf(stateStore: String): LivyConf = {
      val conf = new LivyConf()
      conf.set(LivyConf.RECOVERY_MODE.key, SessionManager.SESSION_RECOVERY_MODE_RECOVERY)
      conf.set(LivyConf.RECOVERY_STATE_STORE.key, stateStore)
      conf
    }

    it("should throw an error on get if it's not initialized") {
      intercept[AssertionError] { StateStore.get }
    }

    it("should initialize blackhole state store if recovery is disabled") {
      StateStore.init(new LivyConf())
      StateStore.get shouldBe a[BlackholeStateStore]
    }

    it("should pick the correct store according to state store config") {
      StateStore.pickStateStore(createConf("filesystem")) shouldBe classOf[FileSystemStateStore]
      StateStore.pickStateStore(createConf("zookeeper")) shouldBe classOf[ZooKeeperStateStore]
    }

    it("should return error if an unknown recovery mode is set") {
      val conf = new LivyConf()
      conf.set(LivyConf.RECOVERY_MODE.key, "unknown")
      intercept[IllegalArgumentException] { StateStore.init(conf) }
    }

    it("should return error if an unknown state store is set") {
      intercept[IllegalArgumentException] { StateStore.init(createConf("unknown")) }
    }
  }
} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.SparkContext._
import org.apache.spark.rdd.RDD

import org.apache.spark.graphx._


  def updateVertices(updates: VertexRDD[VD]): ReplicatedVertexView[VD, ED] = {
    val shippedVerts = updates.shipVertexAttributes(hasSrcId, hasDstId)
      .setName("ReplicatedVertexView.updateVertices - shippedVerts %s %s (broadcast)".format(
        hasSrcId, hasDstId))
      .partitionBy(edges.partitioner.get)

    val newEdges = edges.withPartitionsRDD(edges.partitionsRDD.zipPartitions(shippedVerts) {
      (ePartIter, shippedVertsIter) => ePartIter.map {
        case (pid, edgePartition) =>
          (pid, edgePartition.updateVertices(shippedVertsIter.flatMap(_._2.iterator)))
      }
    })
    new ReplicatedVertexView(newEdges, hasSrcId, hasDstId)
  }
} 

package com.redislabs.provider.redis.streaming

import com.redislabs.provider.redis.RedisConfig
import org.apache.curator.utils.ThreadUtils
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.streaming.dstream.ReceiverInputDStream

import redis.clients.jedis._

import scala.reflect.{ClassTag, classTag}
import scala.util.control.NonFatal


      keys.foreach{ key =>
        executorPool.submit(new MessageHandler(redisConfig.connectionForKey(key), key))
      }
    } finally {
      executorPool.shutdown()
    }
  }

  def onStop() {
  }

  private class MessageHandler(conn: Jedis, key: String) extends Runnable {
    def run() {
      try {
        while(!isStopped) {
          val response = conn.blpop(2, key)
          if (response == null || response.isEmpty) {
            // no-op
          } else if (classTag[T] == classTag[String]) {
            store(response.get(1).asInstanceOf[T])
          } else if (classTag[T] == classTag[(String, String)]) {
            store((response.get(0), response.get(1)).asInstanceOf[T])
          } else {
            throw new scala.Exception("Unknown Redis Streaming type")
          }
        }
      } catch {
        case NonFatal(e) =>
          restart("Error receiving data", e)
      } finally {
        onStop()
      }
    }
  }
} 

package com.twosigma.flint.timeseries

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.types._

import scala.reflect.{ ClassTag, classTag }

package object summarize {

  def asDoubleExtractor(dataType: DataType, columnIndex: Int): InternalRow => Double = dataType match {
    case DoubleType => { row: InternalRow => row.getDouble(columnIndex) }
    case LongType => { row: InternalRow => row.getLong(columnIndex).toDouble }
    case IntegerType => { row: InternalRow => row.getInt(columnIndex).toDouble }
    case FloatType => { row: InternalRow => row.getFloat(columnIndex).toDouble }
    case _ => throw new IllegalArgumentException(s"Cannot cast $dataType to DoubleType")
  }

  def toClassTag(dataType: DataType): ClassTag[_] = dataType match {
    case IntegerType => classTag[Int]
    case LongType => classTag[Long]
    case FloatType => classTag[Float]
    case DoubleType => classTag[Double]
    case _ => throw new IllegalArgumentException(s"Unsupported data type: $dataType")
  }

  def toOrdering(dataType: DataType): Ordering[_] = dataType match {
    case IntegerType => Ordering[Int]
    case LongType => Ordering[Long]
    case FloatType => Ordering[Float]
    case DoubleType => Ordering[Double]
    case _ => throw new IllegalArgumentException(s"Unsupported data type: $dataType")
  }
} 

package com.twosigma.flint.hadoop

import java.io.{ DataInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream }
import java.io.IOException

import scala.reflect.{ classTag, ClassTag }

import org.apache.hadoop.io.Writable

// Note: we could make this implement InputSplit, but we do not because many input splits do a
// cast to their specific InputSplit, so we do not want to risk it. Further, this currently works
// for any Writable.
case class WriSer[T <: Writable: ClassTag](@transient var get: T) extends Serializable {
  def this() = this(null.asInstanceOf[T])

  @throws(classOf[IOException])
  private def writeObject(out: ObjectOutputStream) {
    out.writeObject(classTag[T])
    get.write(new DataOutputStream(out))
  }

  @throws(classOf[IOException])
  @throws(classOf[ClassNotFoundException])
  private def readObject(in: ObjectInputStream) {
    get = in.readObject.asInstanceOf[ClassTag[T]].runtimeClass.newInstance.asInstanceOf[T]
    get.readFields(new DataInputStream(in))
  }
} 

package emul
import scala.reflect.{ClassTag, classTag}

class Memory[T:ClassTag](name: String) {
  var data: Array[T] = _
  var bits: Int = -1
  private var needsInit: Boolean = true
  def initMem(size: Int, zero: T): Unit = if (needsInit) {
    data = Array.fill(size)(zero)
    needsInit = false
    bits = zero match {
      case k: FixedPoint => k.fmt.bits
      case _ => -1
    }
  }

  def apply(i: Int): T = {
    DRAMTracker.accessMap((classTag[T], bits, "read")) += 1
    data.apply(i)
  }
  def update(i: Int, x: T): Unit = {
    DRAMTracker.accessMap((classTag[T], bits, "write")) += 1
    data.update(i, x)
  }
} 

package argon

import argon.schedule.SimpleScheduler
import utils.io.CaptureStream
import utils.isSubtype

import scala.reflect.{ClassTag, classTag}

trait DSLTestbench extends utils.Testbench with DSLRunnable { self =>

  def req[A,B](res: A, gold: B, msg: => String)(implicit ctx: SrcCtx): Unit = {
    // if (!(res equals gold)) res shouldBe gold
  }
  def reqOp[O:ClassTag](x: Sym[_], msg: => String)(implicit ctx: SrcCtx): Unit = {
    val res = x.op.map(_.getClass).getOrElse(x.getClass)
    val gold = classTag[O].runtimeClass
    require(isSubtype(res,gold), msg)
  }

  def reqWarn(calc: => Any, expect: String, msg: => String)(implicit ctx: SrcCtx): Unit = {
    val capture = new CaptureStream(state.out)
    withOut(capture){ calc }
    val lines = capture.dump.split("\n")
    require(lines.exists{line => line.contains("warn") && line.contains(expect)}, s"$msg. Expected warning $expect")
  }

  def checks(): Unit = { }

  // Having this as a should statement makes it lazily evaluated (even though its part of the constructor)
  s"$name" should "check without requirement failures" in { checks() }
} 

package org.camunda.feel.impl

import java.util.ServiceLoader

import org.camunda.feel.context.{CustomFunctionProvider, FunctionProvider}
import org.camunda.feel.valuemapper.{CustomValueMapper, ValueMapper}

import scala.reflect.{ClassTag, classTag}
import scala.collection.JavaConverters._

object SpiServiceLoader {

  def loadValueMapper: ValueMapper = {
    val customValueMappers = loadServiceProvider[CustomValueMapper]()
    ValueMapper.CompositeValueMapper(customValueMappers)
  }

  def loadFunctionProvider: FunctionProvider =
    loadServiceProvider[CustomFunctionProvider]() match {
      case Nil      => FunctionProvider.EmptyFunctionProvider
      case p :: Nil => p
      case ps       => FunctionProvider.CompositeFunctionProvider(ps)
    }

  private def loadServiceProvider[T: ClassTag](): List[T] =
    try {
      val loader =
        ServiceLoader.load(classTag[T].runtimeClass.asInstanceOf[Class[T]])
      loader.iterator.asScala.toList
    } catch {
      case t: Throwable =>
        System.err.println(
          s"Failed to load service provider: ${classTag[T].runtimeClass.getSimpleName}")
        t.printStackTrace()
        throw t
    }

} 

package org.apache.spark.rdd

import scala.reflect.{classTag, ClassTag}

import org.apache.hadoop.io.Writable
import org.apache.hadoop.io.compress.CompressionCodec
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapred.SequenceFileOutputFormat

import org.apache.spark.internal.Logging


  def saveAsSequenceFile(
      path: String,
      codec: Option[Class[_ <: CompressionCodec]] = None): Unit = self.withScope {
    def anyToWritable[U <% Writable](u: U): Writable = u

    // TODO We cannot force the return type of `anyToWritable` be same as keyWritableClass and
    // valueWritableClass at the compile time. To implement that, we need to add type parameters to
    // SequenceFileRDDFunctions. however, SequenceFileRDDFunctions is a public class so it will be a
    // breaking change.
    val convertKey = self.keyClass != keyWritableClass
    val convertValue = self.valueClass != valueWritableClass

    logInfo("Saving as sequence file of type (" + keyWritableClass.getSimpleName + "," +
      valueWritableClass.getSimpleName + ")" )
    val format = classOf[SequenceFileOutputFormat[Writable, Writable]]
    val jobConf = new JobConf(self.context.hadoopConfiguration)
    if (!convertKey && !convertValue) {
      self.saveAsHadoopFile(path, keyWritableClass, valueWritableClass, format, jobConf, codec)
    } else if (!convertKey && convertValue) {
      self.map(x => (x._1, anyToWritable(x._2))).saveAsHadoopFile(
        path, keyWritableClass, valueWritableClass, format, jobConf, codec)
    } else if (convertKey && !convertValue) {
      self.map(x => (anyToWritable(x._1), x._2)).saveAsHadoopFile(
        path, keyWritableClass, valueWritableClass, format, jobConf, codec)
    } else if (convertKey && convertValue) {
      self.map(x => (anyToWritable(x._1), anyToWritable(x._2))).saveAsHadoopFile(
        path, keyWritableClass, valueWritableClass, format, jobConf, codec)
    }
  }
} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

package is.hail.utils

import java.io.{ObjectInputStream, ObjectOutputStream}

import is.hail.backend.spark.SparkBackend
import is.hail.expr.ir.ExecuteContext
import is.hail.io.fs.FS
import org.apache.spark.rdd.RDD

import scala.reflect.ClassTag
import scala.reflect.classTag

object SpillingCollectIterator {
  def apply[T: ClassTag](localTmpdir: String, fs: FS, rdd: RDD[T], sizeLimit: Int): SpillingCollectIterator[T] = {
    val nPartitions = rdd.partitions.length
    val x = new SpillingCollectIterator(localTmpdir, fs, nPartitions, sizeLimit)
    val ctc = classTag[T]
    SparkBackend.sparkContext("SpillingCollectIterator.apply").runJob(
      rdd,
      (_, it: Iterator[T]) => it.toArray(ctc),
      0 until nPartitions,
      x.append _)
    x
  }
}

class SpillingCollectIterator[T: ClassTag] private (localTmpdir: String, fs: FS, nPartitions: Int, sizeLimit: Int) extends Iterator[T] {
  private[this] val files: Array[(String, Long)] = new Array(nPartitions)
  private[this] val buf: Array[Array[T]] = new Array(nPartitions)
  private[this] var _size: Long = 0L
  private[this] var i: Int = 0
  private[this] var it: Iterator[T] = null

  private def append(partition: Int, a: Array[T]): Unit = synchronized {
    assert(buf(partition) == null)
    buf(partition) = a
    _size += a.length
    if (_size > sizeLimit) {
      val file = ExecuteContext.createTmpPathNoCleanup(localTmpdir, s"spilling-collect-iterator-$partition")
      using(fs.createNoCompression(file)) { os =>
        var k = 0
        while (k < buf.length) {
          val vals = buf(k)
          if (vals != null) {
            buf(k) = null
            val pos = os.getPosition
            val oos = new ObjectOutputStream(os)
            oos.writeInt(vals.length)
            var j = 0
            while (j < vals.length) {
              oos.writeObject(vals(j))
              j += 1
            }
            files(k) = (file, pos)
            oos.flush()
          }
          k += 1
        }
      }
      _size = 0
    }
  }

  def hasNext: Boolean = {
    if (it == null || !it.hasNext) {
      if (i >= files.length) {
        it = null
        return false
      } else if (files(i) == null) {
        assert(buf(i) != null)
        it = buf(i).iterator
        buf(i) = null
      } else {
        val (filename, pos) = files(i)
        using(fs.openNoCompression(filename)) { is =>
          is.seek(pos)
          using(new ObjectInputStream(is)) { ois =>
            val length = ois.readInt()
            val arr = new Array[T](length)
            var j = 0
            while (j < length) {
              arr(j) = ois.readObject().asInstanceOf[T]
              j += 1
            }
            it = arr.iterator
          }
        }
      }
      i += 1
    }
    it.hasNext
  }

  def next: T = {
    hasNext
    it.next
  }
} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{HashPartitioner, OneToOneDependency}
import org.apache.spark.graphx._
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package is.hail.expr.ir

import is.hail.types._
import is.hail.types.virtual._

import scala.reflect.{ClassTag, classTag}

object TypeToIRIntermediateClassTag {
  def apply(t: Type): ClassTag[_] = t.fundamentalType match {
    case TVoid => classTag[Unit]
    case TBoolean => classTag[Boolean]
    case TInt32 => classTag[Int]
    case TInt64 => classTag[Long]
    case TFloat32 => classTag[Float]
    case TFloat64 => classTag[Double]
    case _: TBaseStruct | _: TArray | TBinary => classTag[Long]
  }
} 

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import is.hail.types.physical.PSet
import is.hail.utils._
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TSet(elementType: Type) extends TContainer {
  override lazy val fundamentalType: TArray = TArray(elementType.fundamentalType)

  def _toPretty = s"Set[$elementType]"

  override def pyString(sb: StringBuilder): Unit = {
    sb.append("set<")
    elementType.pyString(sb)
    sb.append('>')
  }

  override def canCompare(other: Type): Boolean = other match {
    case TSet(otherType) => elementType.canCompare(otherType)
    case _ => false
  }

  override def unify(concrete: Type): Boolean = concrete match {
    case TSet(celementType) => elementType.unify(celementType)
    case _ => false
  }

  override def subst() = TSet(elementType.subst())

  def _typeCheck(a: Any): Boolean =
    a.isInstanceOf[Set[_]] && a.asInstanceOf[Set[_]].forall(elementType.typeCheck)

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Set[")
    elementType.pretty(sb, indent, compact)
    sb.append("]")
  }

  override lazy val ordering: ExtendedOrdering = mkOrdering()

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    ExtendedOrdering.setOrdering(elementType.ordering, missingEqual)

  override def _showStr(a: Annotation): String =
    a.asInstanceOf[Set[Annotation]]
      .map { case elt => elementType.showStr(elt) }
      .mkString("{", ",", "}")


  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[Annotation] = Gen.buildableOf[Set](elementType.genValue)

  override def scalaClassTag: ClassTag[Set[AnyRef]] = classTag[Set[AnyRef]]

  override def valueSubsetter(subtype: Type): Any => Any = {
    assert(elementType == subtype.asInstanceOf[TSet].elementType)
    identity
  }
} 

package is.hail.types.virtual

import is.hail.annotations._
import is.hail.backend.BroadcastValue
import is.hail.check._
import is.hail.types.physical.PLocus
import is.hail.utils._
import is.hail.variant._

import scala.reflect.{ClassTag, classTag}

object TLocus {
  def apply(rg: ReferenceGenome): TLocus = TLocus(rg.broadcastRG)

  val representation: TStruct = {
    TStruct(
      "contig" -> TString,
      "position" -> TInt32)
  }

  def schemaFromRG(rg: Option[ReferenceGenome], required: Boolean = false): Type = rg match {
    case Some(ref) => TLocus(ref)
    case None => TLocus.representation
  }
}

case class TLocus(rgBc: BroadcastRG) extends ComplexType {
  def rg: ReferenceGenome = rgBc.value

  def _toPretty = s"Locus($rg)"

  override def pyString(sb: StringBuilder): Unit = {
    sb.append("locus<")
    sb.append(prettyIdentifier(rg.name))
    sb.append('>')
  }
  def _typeCheck(a: Any): Boolean = a.isInstanceOf[Locus]

  override def genNonmissingValue: Gen[Annotation] = Locus.gen(rg)

  override def scalaClassTag: ClassTag[Locus] = classTag[Locus]

  override lazy val ordering: ExtendedOrdering = mkOrdering()

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    ExtendedOrdering.extendToNull(rg.locusOrdering, missingEqual)

  lazy val representation: TStruct = TLocus.representation

  def locusOrdering: Ordering[Locus] = rg.locusOrdering

  override def unify(concrete: Type): Boolean = concrete match {
    case TLocus(crgBc) => rg == crgBc.value
    case _ => false
  }
} 

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TStream(elementType: Type) extends TIterable {
  override def pyString(sb: StringBuilder): Unit = {
    sb.append("stream<")
    elementType.pyString(sb)
    sb.append('>')
  }
  override val fundamentalType: TStream = {
    if (elementType == elementType.fundamentalType)
      this
    else
      this.copy(elementType = elementType.fundamentalType)
  }

  def _toPretty = s"Stream[$elementType]"

  override def canCompare(other: Type): Boolean =
    throw new UnsupportedOperationException("Stream comparison is currently undefined.")

  override def unify(concrete: Type): Boolean = concrete match {
    case TStream(celementType) => elementType.unify(celementType)
    case _ => false
  }

  override def subst() = TStream(elementType.subst())

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Stream[")
    elementType.pretty(sb, indent, compact)
    sb.append("]")
  }

  def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] &&
    a.asInstanceOf[IndexedSeq[_]].forall(elementType.typeCheck)

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def isRealizable = false

  override def genNonmissingValue: Gen[Annotation] =
    throw new UnsupportedOperationException("Streams don't have associated annotations.")

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    throw new UnsupportedOperationException("Stream comparison is currently undefined.")

  override def scalaClassTag: ClassTag[Iterator[AnyRef]] = classTag[Iterator[AnyRef]]
} 

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import is.hail.types.physical.PDict
import is.hail.utils._
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TDict(keyType: Type, valueType: Type) extends TContainer {
  lazy val elementType: TBaseStruct = (TStruct("key" -> keyType, "value" -> valueType)).asInstanceOf[TBaseStruct]

  override val fundamentalType: TArray = TArray(elementType.fundamentalType)

  override def canCompare(other: Type): Boolean = other match {
    case TDict(okt, ovt) => keyType.canCompare(okt) && valueType.canCompare(ovt)
    case _ => false
  }

  override def children = FastSeq(keyType, valueType)

  override def unify(concrete: Type): Boolean = {
    concrete match {
      case TDict(kt, vt) => keyType.unify(kt) && valueType.unify(vt)
      case _ => false
    }
  }

  override def subst() = TDict(keyType.subst(), valueType.subst())

  def _toPretty = s"Dict[$keyType, $valueType]"

  override def pyString(sb: StringBuilder): Unit = {
    sb.append("dict<")
    keyType.pyString(sb)
    sb.append(", ")
    valueType.pyString(sb)
    sb.append('>')
  }

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Dict[")
    keyType.pretty(sb, indent, compact)
    if (compact)
      sb += ','
    else
      sb.append(", ")
    valueType.pretty(sb, indent, compact)
    sb.append("]")
  }

  def _typeCheck(a: Any): Boolean = a == null || (a.isInstanceOf[Map[_, _]] &&
    a.asInstanceOf[Map[_, _]].forall { case (k, v) => keyType.typeCheck(k) && valueType.typeCheck(v) })

  override def _showStr(a: Annotation): String =
    a.asInstanceOf[Map[Annotation, Annotation]]
      .map { case (k, v) => s"${keyType.showStr(k)}:${valueType.showStr(v)}}" }
      .mkString("{", ",", "}")

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[Annotation] =
    Gen.buildableOf2[Map](Gen.zip(keyType.genValue, valueType.genValue))

  override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean): Boolean =
    a1 == a2 || (a1 != null && a2 != null &&
      a1.asInstanceOf[Map[Any, _]].outerJoin(a2.asInstanceOf[Map[Any, _]])
        .forall { case (_, (o1, o2)) =>
          o1.liftedZip(o2).exists { case (v1, v2) => valueType.valuesSimilar(v1, v2, tolerance, absolute) }
        })

  override def scalaClassTag: ClassTag[Map[_, _]] = classTag[Map[_, _]]

  override lazy val ordering: ExtendedOrdering = mkOrdering()

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    ExtendedOrdering.mapOrdering(elementType.ordering, missingEqual)

  override def valueSubsetter(subtype: Type): Any => Any = {
    val subdict = subtype.asInstanceOf[TDict]
    assert(keyType == subdict.keyType)
    if (valueType == subdict.valueType)
      return identity

    val subsetValue = valueType.valueSubsetter(subdict.valueType)
    (a: Any) => a.asInstanceOf[Map[Any, Any]].mapValues(subsetValue)
  }
} 

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TArray(elementType: Type) extends TContainer {
  override def pyString(sb: StringBuilder): Unit = {
    sb.append("array<")
    elementType.pyString(sb)
    sb.append('>')
  }
  override val fundamentalType: TArray = {
    if (elementType == elementType.fundamentalType)
      this
    else
      this.copy(elementType = elementType.fundamentalType)
  }

  def _toPretty = s"Array[$elementType]"

  override def canCompare(other: Type): Boolean = other match {
    case TArray(otherType) => elementType.canCompare(otherType)
    case _ => false
  }

  override def unify(concrete: Type): Boolean = concrete match {
    case TArray(celementType) => elementType.unify(celementType)
    case _ => false
  }

  override def subst() = TArray(elementType.subst())

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Array[")
    elementType.pretty(sb, indent, compact)
    sb.append("]")
  }

  def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] &&
    a.asInstanceOf[IndexedSeq[_]].forall(elementType.typeCheck)

  override def _showStr(a: Annotation): String =
    a.asInstanceOf[IndexedSeq[Annotation]]
      .map(elt => elementType.showStr(elt))
      .mkString("[", ",", "]")

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[IndexedSeq[Annotation]] =
    Gen.buildableOf[Array](elementType.genValue).map(x => x: IndexedSeq[Annotation])

  override val ordering: ExtendedOrdering = mkOrdering()

  def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    ExtendedOrdering.iterableOrdering(elementType.ordering, missingEqual)

  override def scalaClassTag: ClassTag[IndexedSeq[AnyRef]] = classTag[IndexedSeq[AnyRef]]

  override def valueSubsetter(subtype: Type): Any => Any = {
    if (this == subtype)
      return identity

    val subsetElem = elementType.valueSubsetter(subtype.asInstanceOf[TArray].elementType)
    (a: Any) => a.asInstanceOf[IndexedSeq[Any]].map(subsetElem)
  }
} 

package is.hail.types.virtual

import is.hail.annotations._
import is.hail.check.Gen
import is.hail.types.physical.PBaseStruct
import is.hail.utils._
import org.apache.spark.sql.Row
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

object TBaseStruct {
  
  def getOrdering(types: Array[Type], missingEqual: Boolean = true): ExtendedOrdering =
    ExtendedOrdering.rowOrdering(types.map(_.ordering), missingEqual)

  def getJoinOrdering(types: Array[Type]): ExtendedOrdering =
    ExtendedOrdering.rowOrdering(types.map(_.mkOrdering(missingEqual = false)), _missingEqual = false)
}

abstract class TBaseStruct extends Type {
  def types: Array[Type]

  def fields: IndexedSeq[Field]

  override def children: Seq[Type] = types

  def size: Int

  def _toPretty: String = {
    val sb = new StringBuilder
    _pretty(sb, 0, compact = true)
    sb.result()
  }

  override def _typeCheck(a: Any): Boolean = a match {
    case row: Row =>
      row.length == types.length &&
        isComparableAt(a)
    case _ => false
  }

  def relaxedTypeCheck(a: Any): Boolean = a match {
    case row: Row =>
      row.length <= types.length &&
        isComparableAt(a)
    case _ => false
  }

  def isComparableAt(a: Annotation): Boolean = a match {
    case row: Row =>
      row.toSeq.zip(types).forall {
        case (v, t) => t.typeCheck(v)
      }
    case _ => false
  }

  def isIsomorphicTo(other: TBaseStruct): Boolean =
    size == other.size && isCompatibleWith(other)

  def isPrefixOf(other: TBaseStruct): Boolean =
    size <= other.size && isCompatibleWith(other)

  def isCompatibleWith(other: TBaseStruct): Boolean =
    fields.zip(other.fields).forall{ case (l, r) => l.typ == r.typ }

  def truncate(newSize: Int): TBaseStruct

  override def _showStr(a: Annotation): String = {
    if (types.isEmpty)
      "()"
    else {
      Array.tabulate(size)(i => types(i).showStr(a.asInstanceOf[Row].get(i)))
        .mkString("(", ",", ")")
    }
  }

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[Annotation] = {
    if (types.isEmpty) {
      Gen.const(Annotation.empty)
    } else
      Gen.size.flatMap(fuel =>
        if (types.length > fuel)
          Gen.uniformSequence(types.map(t => Gen.const(null))).map(a => Annotation(a: _*))
        else
          Gen.uniformSequence(types.map(t => t.genValue)).map(a => Annotation(a: _*)))
  }

  override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean): Boolean =
    a1 == a2 || (a1 != null && a2 != null
      && types.zip(a1.asInstanceOf[Row].toSeq).zip(a2.asInstanceOf[Row].toSeq)
      .forall {
        case ((t, x1), x2) =>
          t.valuesSimilar(x1, x2, tolerance, absolute)
      })

  override def scalaClassTag: ClassTag[Row] = classTag[Row]
} 

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import is.hail.types.physical.PInterval
import is.hail.utils.{FastSeq, Interval}

import scala.reflect.{ClassTag, classTag}

case class TInterval(pointType: Type) extends ComplexType {
  override def children = FastSeq(pointType)

  def _toPretty = s"""Interval[$pointType]"""

  override def pyString(sb: StringBuilder): Unit = {
    sb.append("interval<")
    pointType.pyString(sb)
    sb.append('>')
  }
  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Interval[")
    pointType.pretty(sb, indent, compact)
    sb.append("]")
  }

  def _typeCheck(a: Any): Boolean = a.isInstanceOf[Interval] && {
    val i = a.asInstanceOf[Interval]
    pointType.typeCheck(i.start) && pointType.typeCheck(i.end)
  }

  override def genNonmissingValue: Gen[Annotation] = Interval.gen(pointType.ordering, pointType.genValue)

  override def scalaClassTag: ClassTag[Interval] = classTag[Interval]

  override lazy val ordering: ExtendedOrdering = mkOrdering()

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    Interval.ordering(pointType.ordering, startPrimary=true, missingEqual)

  lazy val representation: TStruct = {
    TStruct(
      "start" -> pointType,
      "end" -> pointType,
      "includesStart" -> TBoolean,
      "includesEnd" -> TBoolean)
  }

  override def unify(concrete: Type): Boolean = concrete match {
    case TInterval(cpointType) => pointType.unify(cpointType)
    case _ => false
  }

  override def subst() = TInterval(pointType.subst())
} 

package org.argus.jawa.flow.summary

import com.google.common.base.Charsets
import com.google.common.io.Resources
import org.argus.jawa.core._
import org.argus.jawa.core.elements.Signature
import org.argus.jawa.core.util._
import org.argus.jawa.flow.summary.susaf.HeapSummaryProcessor
import org.argus.jawa.flow.summary.susaf.parser.SummaryParser
import org.argus.jawa.flow.summary.susaf.rule.HeapSummary

import scala.reflect.{ClassTag, classTag}


class SummaryManager(global: Global) {

  //  Map from signature to Summary
  private val summaries: MMap[Signature, MSet[Summary[_]]] = mmapEmpty
  private val heapSummariesMatchFileAndSubsig: MMap[String, IMap[String, HeapSummary]] = mmapEmpty

  def register(signature: Signature, summary: Summary[_]): Unit = summaries.getOrElseUpdate(signature, msetEmpty) += summary

  def register(name: String, suCode: String, fileAndSubsigMatch: Boolean): IMap[Signature, Summary[_]] = {
    val su = SummaryParser(suCode)
    su.defaultTypes.foreach { case (baseType, fields) =>
      HeapSummaryProcessor.addDefaultTypes(global, baseType, fields)
    }
    if(fileAndSubsigMatch) {
      val s = su.summaries.map{ case (k, v) => k.getSubSignature -> v}
      this.heapSummariesMatchFileAndSubsig(name) = s
    } else {
      su.summaries.foreach { case (signature, summary) =>
        register(signature, summary)
      }
    }
    su.summaries
  }

  def contains(sig: Signature): Boolean = summaries.contains(sig)
  def contains(file: String, subsig: String): Boolean = heapSummariesMatchFileAndSubsig.get(file) match {
    case Some(map) => map.contains(subsig)
    case None => false
  }

  def getSummaries(sig: Signature): ISet[Summary[_]] = summaries.getOrElse(sig, msetEmpty).toSet

  def getSummary[T <: Summary[_] : ClassTag](sig: Signature): Option[T] = {
    summaries.get(sig) match {
      case Some(sus) =>
        sus.foreach {
          case t if classTag[T].runtimeClass.isInstance(t) => return Some(t.asInstanceOf[T])
          case _ =>
        }
        None
      case None => None
    }
  }

  def getHeapSummaryPb(sig: Signature): Option[summary.HeapSummary] = {
    getSummary[HeapSummary](sig) match {
      case Some(su) =>
        Some(SummaryToProto.toProto(su))
      case None =>
        None
    }
  }

  def registerFile(safsuPath: String, name: String, fileAndSubsigMatch: Boolean): Unit = {
    val url = Resources.getResource(safsuPath)
    val code = Resources.toString(url, Charsets.UTF_8)
    register(name, code, fileAndSubsigMatch)
  }

  def registerExternalFile(safsuPath: FileResourceUri, name: String, fileAndSubsigMatch: Boolean): Unit = {
    val url = FileUtil.toFile(safsuPath).toURI.toURL
    val code = Resources.toString(url, Charsets.UTF_8)
    register(name, code, fileAndSubsigMatch)
  }

  def getSummariesByFile(name: String): IMap[String, HeapSummary] = {
    this.heapSummariesMatchFileAndSubsig.getOrElse(name, imapEmpty)
  }
} 

// Copyright (C) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in project root for information.

package com.microsoft.ml.spark.core.schema

import com.microsoft.ml.spark.core.test.base.TestBase
import org.apache.spark.sql.types._

import scala.reflect.{ClassTag, classTag}

class TestCategoricalMap extends TestBase {

  
  test("Test: Create basic CategoricalMap") {
    for (mmlStyle <- List(true, false)) {
      val isOrdinal = mmlStyle
      testMapBasic(Array("as", "", "efe"),
                   "wrong_level", StringType, isOrdinal, mmlStyle)
      testMapBasic(Array[Int](34, 54747, -346, 756, 0),
                   -45, IntegerType, isOrdinal, mmlStyle)
      testMapBasic(Array[Long](34, 54747, -346, 756, 0),
                   (-45: Long), LongType, isOrdinal, mmlStyle)
      testMapBasic(Array[Double](34.45, 54.747, -3.46, 7.56, 0),
                   (-45: Double), DoubleType, isOrdinal, mmlStyle)
    }
  }

} 

package com.github.j5ik2o.dddbase.example.model
import java.time.ZonedDateTime

import com.github.j5ik2o.dddbase.{ Aggregate, AggregateId }

import scala.reflect.{ classTag, ClassTag }

case class UserMessageId(userId: Long, messageId: Long) extends AggregateId {
  override type IdType = (Long, Long)
  override val value = (userId, messageId)
}

case class UserMessage(
    id: UserMessageId,
    status: Status,
    message: String,
    createdAt: ZonedDateTime,
    updatedAt: Option[ZonedDateTime]
) extends Aggregate {
  override type IdType        = UserMessageId
  override type AggregateType = UserMessage
  override protected val tag: ClassTag[UserMessage] = classTag[UserMessage]
} 

package scala.collection.concurrent.ctries_new

import scala.reflect.{ClassTag, classTag}

trait Spec {

  implicit def implicitously = scala.language.implicitConversions
  implicit def reflectively  = scala.language.reflectiveCalls

  implicit def str2ops(s: String) = new {
    def in[U](body: =>U): Unit = {
      // just execute body
      body
    }
  }

  implicit def any2ops(a: Any) = new {
    def shouldEqual(other: Any) = assert(a == other)
  }

  def evaluating[U](body: =>U) = new {
    def shouldProduce[T <: Throwable: ClassTag]() = {
      var produced = false
      try body
      catch {
        case e: Throwable => if (e.getClass == implicitly[ClassTag[T]].runtimeClass) produced = true
      } finally {
        assert(produced, "Did not produce exception of type: " + implicitly[ClassTag[T]])
      }
    }
  }

} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

package org.apache.spark.util

import java.util

import scala.reflect.{classTag, ClassTag}

private[spark] object CollectionsUtils {
  def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
    // For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
    classTag[K] match {
      case ClassTag.Float =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Float]], x.asInstanceOf[Float])
      case ClassTag.Double =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Double]], x.asInstanceOf[Double])
      case ClassTag.Byte =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Byte]], x.asInstanceOf[Byte])
      case ClassTag.Char =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Char]], x.asInstanceOf[Char])
      case ClassTag.Short =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Short]], x.asInstanceOf[Short])
      case ClassTag.Int =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Int]], x.asInstanceOf[Int])
      case ClassTag.Long =>
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[Long]], x.asInstanceOf[Long])
      case _ =>
        val comparator = implicitly[Ordering[K]].asInstanceOf[java.util.Comparator[Any]]
        (l, x) => util.Arrays.binarySearch(l.asInstanceOf[Array[AnyRef]], x, comparator)
    }
  }
} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{HashPartitioner, OneToOneDependency}
import org.apache.spark.graphx._
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

} 

package com.twilio.open.streaming.trend.discovery

import java.io.{ByteArrayInputStream, InputStream}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.google.protobuf.Message
import com.googlecode.protobuf.format.JsonFormat
import com.holdenkarau.spark.testing.{LocalSparkContext, SparkContextProvider}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers, Suite}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.Seq
import scala.io.Source
import scala.reflect.ClassTag
import scala.reflect.classTag

object TestHelper {
  val log: Logger = LoggerFactory.getLogger("com.twilio.open.streaming.trend.discovery.TestHelper")
  val mapper: ObjectMapper = {
    val m = new ObjectMapper()
    m.registerModule(DefaultScalaModule)
  }

  val jsonFormat: JsonFormat = new JsonFormat

  def loadScenario[T<: Message : ClassTag](file: String): Seq[T] = {
    val fileString = Source.fromFile(file).mkString
    val parsed = mapper.readValue(fileString, classOf[Sceanario])
    parsed.input.map { data =>
      val json = mapper.writeValueAsString(data)
      convert[T](json)
    }
  }

  def convert[T<: Message : ClassTag](json: String): T = {
    val clazz = classTag[T].runtimeClass
    val builder = clazz.getMethod("newBuilder").invoke(clazz).asInstanceOf[Message.Builder]
    try {
      val input: InputStream = new ByteArrayInputStream(json.getBytes())
      jsonFormat.merge(input, builder)
      builder.build().asInstanceOf[T]
    } catch {
      case e: Exception =>
        throw e
    }
  }

}

@SerialVersionUID(1L)
case class KafkaDataFrame(key: Array[Byte], topic: Array[Byte], value: Array[Byte]) extends Serializable

case class Sceanario(input: Seq[Any], expected: Option[Any] = None)

trait SparkSqlTest extends BeforeAndAfterAll with SparkContextProvider {
  self: Suite =>

  @transient var _sparkSql: SparkSession = _
  @transient private var _sc: SparkContext = _

  override def sc: SparkContext = _sc

  def conf: SparkConf

  def sparkSql: SparkSession = _sparkSql

  override def beforeAll() {
    _sparkSql = SparkSession.builder().config(conf).getOrCreate()

    _sc = _sparkSql.sparkContext
    setup(_sc)
    super.beforeAll()
  }

  override def afterAll() {
    try {
      _sparkSql.close()
      _sparkSql = null
      LocalSparkContext.stop(_sc)
      _sc = null
    } finally {
      super.afterAll()
    }
  }

} 

package org.apache.spark.graphx.impl

import scala.reflect.{classTag, ClassTag}

import org.apache.spark.{HashPartitioner, OneToOneDependency}
import org.apache.spark.graphx._
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
    @transient override val partitionsRDD: RDD[(PartitionID, EdgePartition[ED, VD])],
    val targetStorageLevel: StorageLevel = StorageLevel.MEMORY_ONLY)
  extends EdgeRDD[ED](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) {

  override def setName(_name: String): this.type = {
    if (partitionsRDD.name != null) {
      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
    } else {
      partitionsRDD.setName(_name)
    }
    this
  }
  setName("EdgeRDD")

  
  override def count(): Long = {
    partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
  }

  override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
    mapEdgePartitions((pid, part) => part.map(f))

  override def reverse: EdgeRDDImpl[ED, VD] = mapEdgePartitions((pid, part) => part.reverse)

  def filter(
      epred: EdgeTriplet[VD, ED] => Boolean,
      vpred: (VertexId, VD) => Boolean): EdgeRDDImpl[ED, VD] = {
    mapEdgePartitions((pid, part) => part.filter(epred, vpred))
  }

  override def innerJoin[ED2: ClassTag, ED3: ClassTag]
      (other: EdgeRDD[ED2])
      (f: (VertexId, VertexId, ED, ED2) => ED3): EdgeRDDImpl[ED3, VD] = {
    val ed2Tag = classTag[ED2]
    val ed3Tag = classTag[ED3]
    this.withPartitionsRDD[ED3, VD](partitionsRDD.zipPartitions(other.partitionsRDD, true) {
      (thisIter, otherIter) =>
        val (pid, thisEPart) = thisIter.next()
        val (_, otherEPart) = otherIter.next()
        Iterator(Tuple2(pid, thisEPart.innerJoin(otherEPart)(f)(ed2Tag, ed3Tag)))
    })
  }

  def mapEdgePartitions[ED2: ClassTag, VD2: ClassTag](
      f: (PartitionID, EdgePartition[ED, VD]) => EdgePartition[ED2, VD2]): EdgeRDDImpl[ED2, VD2] = {
    this.withPartitionsRDD[ED2, VD2](partitionsRDD.mapPartitions({ iter =>
      if (iter.hasNext) {
        val (pid, ep) = iter.next()
        Iterator(Tuple2(pid, f(pid, ep)))
      } else {
        Iterator.empty
      }
    }, preservesPartitioning = true))
  }

  private[graphx] def withPartitionsRDD[ED2: ClassTag, VD2: ClassTag](
      partitionsRDD: RDD[(PartitionID, EdgePartition[ED2, VD2])]): EdgeRDDImpl[ED2, VD2] = {
    new EdgeRDDImpl(partitionsRDD, this.targetStorageLevel)
  }

  override private[graphx] def withTargetStorageLevel(
      targetStorageLevel: StorageLevel): EdgeRDDImpl[ED, VD] = {
    new EdgeRDDImpl(this.partitionsRDD, targetStorageLevel)
  }

}