org.json4s.Extraction Scala Examples

package io.iohk.ethereum.keystore

import java.util.UUID

import akka.util.ByteString
import io.iohk.ethereum.domain.Address
import io.iohk.ethereum.keystore.EncryptedKey._
import org.json4s.JsonAST.{JObject, JString, JValue}
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.{CustomSerializer, DefaultFormats, Extraction, JField}
import org.spongycastle.util.encoders.Hex

import scala.util.Try

object EncryptedKeyJsonCodec {

  private val byteStringSerializer = new CustomSerializer[ByteString](_ => (
    { case JString(s) => ByteString(Hex.decode(s)) },
    { case bs: ByteString => JString(Hex.toHexString(bs.toArray)) }
  ))

  private implicit val formats = DefaultFormats + byteStringSerializer

  private def asHex(bs: ByteString): String =
    Hex.toHexString(bs.toArray)

  def toJson(encKey: EncryptedKey): String = {
    import encKey._
    import cryptoSpec._

    val json =
      ("id" -> id.toString) ~
      ("address" -> asHex(address.bytes)) ~
      ("version" -> version) ~
      ("crypto" -> (
        ("cipher" -> cipher) ~
        ("ciphertext" -> asHex(ciphertext)) ~
        ("cipherparams" -> ("iv" -> asHex(iv))) ~
        encodeKdf(kdfParams) ~
        ("mac" -> asHex(mac))
      ))

    pretty(render(json))
  }

  def fromJson(jsonStr: String): Either[String, EncryptedKey] = Try {
    val json = parse(jsonStr).transformField { case JField(k, v) => JField(k.toLowerCase, v) }

    val uuid = UUID.fromString((json \ "id").extract[String])
    val address = Address((json \ "address").extract[String])
    val version = (json \ "version").extract[Int]

    val crypto = json \ "crypto"
    val cipher = (crypto \ "cipher").extract[String]
    val ciphertext = (crypto \ "ciphertext").extract[ByteString]
    val iv = (crypto \ "cipherparams" \ "iv").extract[ByteString]
    val mac = (crypto \ "mac").extract[ByteString]

    val kdfParams = extractKdf(crypto)
    val cryptoSpec = CryptoSpec(cipher, ciphertext, iv, kdfParams, mac)
    EncryptedKey(uuid, address, cryptoSpec, version)

  }.fold(ex => Left(ex.toString), encKey => Right(encKey))

  private def encodeKdf(kdfParams: KdfParams): JObject =
    kdfParams match {
      case ScryptParams(salt, n, r, p, dklen) =>
        ("kdf" -> Scrypt) ~
        ("kdfparams" -> Extraction.decompose(kdfParams))

      case Pbkdf2Params(salt, prf, c, dklen) =>
        ("kdf" -> Pbkdf2) ~
        ("kdfparams" -> Extraction.decompose(kdfParams))
    }

  private def extractKdf(crypto: JValue): KdfParams = {
    val kdf = (crypto \ "kdf").extract[String]
    kdf.toLowerCase match {
      case Scrypt =>
        (crypto \ "kdfparams").extract[ScryptParams]

      case Pbkdf2 =>
        (crypto \ "kdfparams").extract[Pbkdf2Params]
    }
  }

} 

package com.webank.wedatasphere.linkis.resourcemanager

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.{CustomSerializer, Extraction}


trait ResultResource

case class NotEnoughResource(reason: String = null) extends ResultResource

case class AvailableResource(ticketId: String) extends ResultResource

case class UserResultResource(ticketId: String, user: String) extends ResultResource

object ResultResourceSerializer extends CustomSerializer[ResultResource](implicit formats => ( {
  case JObject(List(("NotEnoughResource", JObject(List(("reason", reason)))))) => NotEnoughResource(reason.extract[String])
  case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => AvailableResource(ticketId.extract[String])
  case JObject(List(("UserResultResource", JObject(List(("ticketId", ticketId), ("user", user)))))) =>
    UserResultResource(ticketId.extract[String], user.extract[String])
}, {
  case r: NotEnoughResource => ("NotEnoughResource", ("reason", Extraction.decompose(r.reason)))
  case r: AvailableResource => ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId)))
  case r: UserResultResource => ("UserResultResource", ("ticketId", r.ticketId) ~ ("user", r.user))
})) 

package com.webank.wedatasphere.linkis.resourcemanager.domain

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}


case class ModuleResourceInfo(moduleInstance: ServiceInstance, totalResource: Resource, usedResource: Resource)

object ModuleResourceInfoSerializer extends CustomSerializer[ModuleResourceInfo](implicit formats => ( {
  case JObject(List(("ModuleResourceInfo", JObject(List(("moduleInstance", moduleInstance), ("totalResource", totalResource), ("usedResource", usedResource)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ModuleResourceInfo(moduleInstance.extract[ServiceInstance], totalResource.extract[Resource], usedResource.extract[Resource])
}, {
  case m: ModuleResourceInfo =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("ModuleResourceInfo", new JObject(List(("moduleInstance", Extraction.decompose(m.moduleInstance)), ("totalResource", Extraction.decompose(m.totalResource)), ("usedResource", Extraction.decompose(m.usedResource)))))
}))

case class ModuleResourceRecord(moduleInfo: ModuleInfo, moduleUsedResources: Resource, moduleLeftResource: Resource, moduleLockedResource: Resource, registerTime: Long = System.currentTimeMillis()) 

package com.webank.wedatasphere.linkis.resourcemanager.domain

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.ResourceRequestPolicy.ResourceRequestPolicy
import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceRequestPolicy, ResourceSerializer}
import org.json4s.JsonAST.JObject
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}


case class ModuleInfo(moduleInstance: ServiceInstance,
                      totalResource: Resource,
                      protectedResource: Resource, //Enter the protection mode when the resource reaches(当资源达到多少时，进入保护模式)
                      resourceRequestPolicy: ResourceRequestPolicy
                     )

object ModuleInfoSerializer extends CustomSerializer[ModuleInfo](implicit formats => ( {
  case JObject(List(("moduleInstance", moduleInstance), ("totalResource", totalResource), ("protectedResource", protectedResource), ("resourceRequestPolicy", resourceRequestPolicy))) =>
    implicit val formats = DefaultFormats + ResourceSerializer + ModuleInstanceSerializer
    new ModuleInfo(moduleInstance.extract[ServiceInstance], totalResource.extract[Resource], protectedResource.extract[Resource], ResourceRequestPolicy.withName(resourceRequestPolicy.extract[String]))
}, {
  case i: ModuleInfo =>
    implicit val formats = DefaultFormats + ResourceSerializer + ModuleInstanceSerializer
    val policy = Extraction.decompose(i.resourceRequestPolicy.toString)
    new JObject(List(("moduleInstance", Extraction.decompose(i.moduleInstance)), ("totalResource", Extraction.decompose(i.totalResource)), ("protectedResource", Extraction.decompose(i.protectedResource)), ("resourceRequestPolicy", policy)))
})
) 

package com.webank.wedatasphere.linkis.resourcemanager.domain

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}



trait UserResourceInfo

case class UserPreUsedResource(ticketId: String, moduleInstance: ServiceInstance, resource: Resource) extends UserResourceInfo

case class UserUsedResource(ticketId: String, moduleInstance: ServiceInstance, resource: Resource, engineInstance: ServiceInstance = null) extends UserResourceInfo

case class UserReleasedResource(ticketId: String, moduleInstance: ServiceInstance) extends UserResourceInfo


object UserResourceInfoSerializer extends CustomSerializer[UserResourceInfo](implicit formats => ( {
  case JObject(List(("UserPreUsedResource", JObject(List(("ticketId", ticketId), ("moduleInstance", moduleInstance), ("resource", resource)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    new UserPreUsedResource(ticketId.extract[String], moduleInstance.extract[ServiceInstance], resource.extract[Resource])
  case JObject(List(("UserUsedResource", JObject(List(("ticketId", ticketId), ("moduleInstance", moduleInstance), ("resource", resource), ("engineInstance", engineInstance)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    new UserUsedResource(ticketId.extract[String], moduleInstance.extract[ServiceInstance], resource.extract[Resource], engineInstance.extract[ServiceInstance])
  case JObject(List(("UserReleasedResource", JObject(List(("ticketId", ticketId), ("moduleInstance", moduleInstance)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    new UserReleasedResource(ticketId.extract[String], moduleInstance.extract[ServiceInstance])
}, {
  case d: UserPreUsedResource =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("UserPreUsedResource", new JObject(List(("ticketId", Extraction.decompose(d.ticketId)), ("moduleInstance", Extraction.decompose(d.moduleInstance)), ("resource", Extraction.decompose(d.resource)))))
  case d: UserUsedResource =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("UserUsedResource", new JObject(List(("ticketId", Extraction.decompose(d.ticketId)), ("moduleInstance", Extraction.decompose(d.moduleInstance)), ("resource", Extraction.decompose(d.resource)), ("engineInstance", Extraction.decompose(d.engineInstance)))))
  case d: UserReleasedResource =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("UserReleasedResource", new JObject(List(("ticketId", Extraction.decompose(d.ticketId)), ("moduleInstance", Extraction.decompose(d.moduleInstance)))))
})
) 

package com.webank.wedatasphere.linkis.rpc

import java.lang.reflect.ParameterizedType
import java.util

import com.webank.wedatasphere.linkis.rpc.transform.{JavaCollectionSerializer, JavaMapSerializer}
import org.apache.commons.lang.ClassUtils
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.jackson.Serialization
import org.json4s.reflect.ManifestFactory
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}

object RPCFormatsTest {

  trait ResultResource
  class AvailableResource(val ticketId: String) extends ResultResource

  object ResultResourceSerializer extends CustomSerializer[ResultResource](implicit formats => ({
    case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => new AvailableResource(ticketId.extract[String])
  },{
    case r: AvailableResource => ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId)))
  }))

  def testRPC1(args: Array[String]): Unit = {
    implicit val formats = DefaultFormats + ResultResourceSerializer
    val serializerClasses = formats.customSerializers.map(_.getClass.getGenericSuperclass match {
      case p: ParameterizedType =>
        val params = p.getActualTypeArguments
        if(params == null || params.isEmpty) null
        else params(0).asInstanceOf[Class[_]]
    }).filter(_ != null)
    val a = new AvailableResource("aaa")
    val str = Serialization.write(a)
    println(str)
    val clazz = classOf[AvailableResource]
    println(serializerClasses)
    val realClass1 = serializerClasses.find(ClassUtils.isAssignable(clazz, _))
    println(realClass1)
    val realClass = realClass1.getOrElse(clazz)
    val obj = Serialization.read(str)(formats, ManifestFactory.manifestOf(realClass))
    println(obj)
    println(classOf[Array[_]].getClass.getName)
  }

  case class TestCollection1(a: String, list: java.util.List[String])
  case class TestCollection2(a: String, list: java.util.Map[String, Integer])

  def testRPC2(args: Array[String]): Unit = {
    implicit val formats = DefaultFormats + JavaCollectionSerializer + JavaMapSerializer
    //    val a = TestCollection1("1", new util.ArrayList[String]())
    val a = TestCollection2("1", new util.HashMap[String, Integer]())
    //    a.list.add("1111")
    a.list.put("1111", 2)
    val str = Serialization.write(a)
    println(str)
    val realClass = classOf[TestCollection2]
    val obj = Serialization.read(str)(formats, ManifestFactory.manifestOf(realClass))
    println(obj)
  }

  def main(args: Array[String]): Unit = {
    testRPC2(args)
  }
} 

package rhttpc.transport.json4s

import java.lang.reflect.Constructor

import org.json4s.{CustomSerializer, Extraction, Formats, Serializer, TypeInfo}
import org.json4s.JsonAST._

import scala.util.Try

object ExceptionSerializer extends Serializer[Throwable] {
  override def deserialize(implicit format: Formats): PartialFunction[(TypeInfo, JValue), Throwable] = {
    case (
      TypeInfo(clazz, _),
      JObject(("jsonClass", JString(ExceptionClassHavingConstructorWithMessageAndCause(constructor))) ::
        ("message", JString(message)) ::
        ("cause", cause) ::
        Nil)) if classOf[Throwable].isAssignableFrom(clazz) =>
      constructor.newInstance(message, Extraction.extract[Throwable](cause))
    case (
      TypeInfo(clazz, _),
      JObject(("jsonClass", JString(ExceptionClassHavingConstructorWithMessageAndCause(constructor))) ::
        ("message", JNull) ::
        ("cause", cause) ::
        Nil)) if classOf[Throwable].isAssignableFrom(clazz) =>
      constructor.newInstance(null, Extraction.extract[Throwable](cause))
    case (
      TypeInfo(clazz, _),
      JObject(("jsonClass", JString(ExceptionClassHavingConstructorWithMessageOnly(constructor))) ::
        ("message", JString(message)) ::
        Nil)) if classOf[Throwable].isAssignableFrom(clazz) =>
      constructor.newInstance(message)
    case (
      TypeInfo(clazz, _),
      JObject(("jsonClass", JString(ExceptionClassHavingConstructorWithMessageOnly(constructor))) ::
        ("message", JNull) ::
        Nil)) if classOf[Throwable].isAssignableFrom(clazz) =>
      constructor.newInstance(null)
  }

  override def serialize(implicit formats: Formats): PartialFunction[Any, JValue] = {
    case ExceptionInstanceHavingConstructorWithMessageAndCause(ex) =>
      JObject(
        formats.typeHintFieldName -> JString(ex.getClass.getName),
        "message" -> Option(ex.getMessage).map(JString).getOrElse(JNull),
        "cause" -> Extraction.decompose(ex.getCause)
      )
    case ExceptionInstanceHavingConstructorWithMessageOnly(ex) =>
      JObject(
        formats.typeHintFieldName -> JString(ex.getClass.getName),
        "message" -> Option(ex.getMessage).map(JString).getOrElse(JNull)
      )
  }
}


object ExceptionClassHavingConstructorWithMessageAndCause {
  def unapply(className: String): Option[Constructor[Throwable]] = {
    (for {
      clazz <- Try(Class.forName(className))
      if classOf[Throwable].isAssignableFrom(clazz)
      constructor <- constructorWithMessageAndCause(clazz)
    } yield constructor).toOption
  }

  def constructorWithMessageAndCause(clazz: Class[_]): Try[Constructor[Throwable]] =
    Try(clazz.getDeclaredConstructor(classOf[String], classOf[Throwable]).asInstanceOf[Constructor[Throwable]])
}

object ExceptionInstanceHavingConstructorWithMessageAndCause {

  def unapply(instance: Throwable): Option[Throwable] = {
    ExceptionClassHavingConstructorWithMessageAndCause.constructorWithMessageAndCause(instance.getClass).map(_ => instance).toOption
  }

}

object ExceptionClassHavingConstructorWithMessageOnly {
  def unapply(className: String): Option[Constructor[Throwable]] = {
    (for {
      clazz <- Try(Class.forName(className))
      if classOf[Throwable].isAssignableFrom(clazz)
      constructor <- constructorWithMessageOnly(clazz)
    } yield constructor).toOption
  }

  def constructorWithMessageOnly(clazz: Class[_]): Try[Constructor[Throwable]] =
    Try(clazz.getDeclaredConstructor(classOf[String]).asInstanceOf[Constructor[Throwable]])
}

object ExceptionInstanceHavingConstructorWithMessageOnly {

  def unapply(instance: Throwable): Option[Throwable] = {
    ExceptionClassHavingConstructorWithMessageOnly.constructorWithMessageOnly(instance.getClass).map(_ => instance).toOption
  }

} 

package jp.co.bizreach.kinesis.spark

import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.regions.Regions
import jp.co.bizreach.kinesis._
import org.apache.commons.codec.digest.DigestUtils
import org.apache.spark.TaskContext
import org.json4s.jackson.JsonMethods
import org.json4s.{DefaultFormats, Extraction, Formats}
import org.slf4j.LoggerFactory

class KinesisRDDWriter[A <: AnyRef](streamName: String, region: Regions,
                                    credentials: SparkAWSCredentials,
                                    chunk: Int, endpoint: Option[String]) extends Serializable {
  private val logger = LoggerFactory.getLogger(getClass)

  def write(task: TaskContext, data: Iterator[A]): Unit = {
    // send data, including retry
    def put(a: Seq[PutRecordsEntry]) = endpoint.map(e => KinesisRDDWriter.endpointClient(credentials)(e)(region))
      .getOrElse(KinesisRDDWriter.client(credentials)(region))
      .putRecordsWithRetry(PutRecordsRequest(streamName, a))
      .zipWithIndex.collect { case (Left(e), i) => a(i) -> s"${e.errorCode}: ${e.errorMessage}" }

    val errors = data.foldLeft(
      (Nil: Seq[PutRecordsEntry], Nil: Seq[(PutRecordsEntry, String)])
    ){ (z, x) =>
      val (records, failed) = z
      val payload = serialize(x)
      val entry   = PutRecordsEntry(DigestUtils.sha256Hex(payload), payload)

      // record exceeds max size
      if (entry.recordSize > recordMaxDataSize)
        records -> ((entry -> "per-record size limit") +: failed)

      // execute
      else if (records.size >= chunk || (records.map(_.recordSize).sum + entry.recordSize) >= recordsMaxDataSize)
        (entry +: Nil) -> (put(records) ++ failed)

      // buffering
      else
        (entry +: records) -> failed
    } match {
      case (Nil, e)  => e
      case (rest, e) => put(rest) ++ e
    }

    // failed records
    if (errors.nonEmpty) dump(errors)
  }

  protected def dump(errors: Seq[(PutRecordsEntry, String)]): Unit =
    logger.error(
      s"""Could not put record, count: ${errors.size}, following details:
         |${errors map { case (entry, message) => message + "\n" + new String(entry.data, "UTF-8") } mkString "\n"}
       """.stripMargin)

  protected def serialize(a: A)(implicit formats: Formats = DefaultFormats): Array[Byte] =
    JsonMethods.mapper.writeValueAsBytes(Extraction.decompose(a)(formats))

}

object KinesisRDDWriter {
  private val cache = collection.concurrent.TrieMap.empty[Regions, AmazonKinesis]


  private val client: SparkAWSCredentials => Regions => AmazonKinesis = {
    credentials => implicit region =>
      cache.getOrElseUpdate(region, AmazonKinesis(credentials.provider))
  }

  private val endpointClient: SparkAWSCredentials => String => Regions => AmazonKinesis = {
    credentials => endpoint => implicit region =>
      cache.getOrElseUpdate(region, AmazonKinesis(credentials.provider, new EndpointConfiguration(endpoint, region.getName)))
  }

} 

package org.sparklinedata.druid.client.test

import org.apache.commons.io.IOUtils
import org.apache.spark.sql.sources.druid.DruidQueryResultIterator
import org.json4s.Extraction
import org.json4s.jackson.JsonMethods._
import org.scalatest.{BeforeAndAfterAll, FunSuite}
import org.sparklinedata.druid.Utils
import org.sparklinedata.druid.client.{DruidCoordinatorClient, DruidQueryServerClient}
import org.sparklinedata.druid.testenv.DruidCluster

class DruidClientTest extends FunSuite with BeforeAndAfterAll with TestUtils {

  import org.sparklinedata.druid.client.test.TPCHQueries._

  var brokerClient : DruidQueryServerClient = _
  var coordClient : DruidCoordinatorClient = _

  import Utils._

  override def beforeAll() = {
    brokerClient = new DruidQueryServerClient("localhost", DruidCluster.instance.brokerPort)
    coordClient = new DruidCoordinatorClient("localhost", DruidCluster.instance.coordinatorPort)
  }

  test("timeBoundary") {
    println(brokerClient.timeBoundary("tpch"))
  }

  test("coordTimeBoundary") {
    println(coordClient.timeBoundary("tpch"))
  }

  test("metaData") {
    println(brokerClient.metadata("tpch", false))
  }

  test("tpchQ1") {
    println(pretty(render(Extraction.decompose(q1))))

    val r = brokerClient.executeQuery(q1)
    r.foreach(println)

  }

  test("tpchQ3") {
    println(pretty(render(Extraction.decompose(q3))))

    val r = brokerClient.executeQuery(q3)
    r.foreach(println)

  }

  test("tpchQ1MonthGrain") {
    println(pretty(render(Extraction.decompose(q1))))

    val r = brokerClient.executeQuery(q1)
    r.foreach(println)
  }

  test("streamQueryResult") {

    def qRis = getClass.getClassLoader.getResourceAsStream("sampleQueryResult.json")

    for(i <- 0 to 5) {
      recordTime("streamed read") {

        val is = IOUtils.toBufferedInputStream(qRis)
        val it = DruidQueryResultIterator(false, is)
        while (it.hasNext) {
          it.next
        }
      }
      recordTime("list read") {
        val is = IOUtils.toBufferedInputStream(qRis)
        val it = DruidQueryResultIterator(false, is, (), true)
        while (it.hasNext) {
          it.next
        }
      }

    }

  }

  test("serversInfo") {
    val sInfo = coordClient.serversInfo
    println(pretty(render(Extraction.decompose(sInfo))))
  }

  test("dataSourceInfo") {
    val dInfo = coordClient.dataSourceInfo("tpch")
    println(pretty(render(Extraction.decompose(dInfo))))
  }

} 

package org.sparklinedata.druid.client.test

import org.apache.spark.sql.hive.test.sparklinedata.TestHive
import org.apache.spark.sql.hive.test.sparklinedata.TestHive._
import org.apache.spark.sql.util.PlanUtil
import org.json4s.Extraction
import org.json4s.jackson.JsonMethods._
import org.sparklinedata.druid.DruidQuery
import org.sparklinedata.druid.metadata.{FunctionalDependency, FunctionalDependencyType}

class DataSourceTest extends BaseTest {

  import org.sparklinedata.druid.Utils._

  test("baseTable") { td =>
    sql("select * from orderLineItemPartSupplierBase").show(10)
  }

  test("noQuery") { td =>
    val df = sql("select * from orderLineItemPartSupplier")
    df.explain(true)
    df.show(10)
  }

  test("tpchQ1") { td =>

    val dq =
      compact(render(Extraction.decompose(new DruidQuery(TPCHQueries.q1)))).replace('\n', ' ')

    val q = s"""CREATE TEMPORARY TABLE orderLineItemPartSupplier2
      USING org.sparklinedata.druid
      OPTIONS (sourceDataframe "orderLineItemPartSupplierBase",
      timeDimensionColumn "l_shipdate",
      druidDatasource "tpch",
      druidHost '$zkConnectString',
      columnMapping '$colMapping',
      numProcessingThreadsPerHistorical '1',
      functionalDependencies '$functionalDependencies',
      starSchema '$flatStarSchema',
      druidQuery '$dq')""".stripMargin

    println(q)

    sql(q)

    sql("select * from orderLineItemPartSupplier2").show(10)
  }

  test("tpchQ1MonthGrain") { td =>

    val dq =
      compact(render(Extraction.decompose(new DruidQuery(TPCHQueries.q1MonthGrain)))
      ).replace('\n', ' ')

    sql(

      s"""CREATE TEMPORARY TABLE orderLineItemPartSupplier2
      USING org.sparklinedata.druid
      OPTIONS (sourceDataframe "orderLineItemPartSupplierBase",
      timeDimensionColumn "l_shipdate",
      druidDatasource "tpch",
      druidHost '$zkConnectString',
      columnMapping '$colMapping',
      numProcessingThreadsPerHistorical '1',
      functionalDependencies '$functionalDependencies',
      starSchema '$flatStarSchema',
      druidQuery '$dq')""".stripMargin
    )

    sql("select * from orderLineItemPartSupplier2").show(10)
  }

  test("direct") { td =>
    val dq = new DruidQuery(TPCHQueries.q1MonthGrain)
    val drInfo = PlanUtil.druidRelationInfo("orderLineItemPartSupplier")(TestHive)
    val df = PlanUtil.dataFrame(drInfo.get, dq)(TestHive)

    println(df.queryExecution.optimizedPlan)
    df.show(10)
  }

  test("t2") { td =>
    import org.json4s._
    import org.json4s.jackson.JsonMethods._

    val fd = FunctionalDependency("a", "b", FunctionalDependencyType.OneToOne)
    println(pretty(render(Extraction.decompose(fd))))
  }


} 

package org.argus.amandroid.serialization

import org.argus.jawa.core.util._
import org.argus.amandroid.core.model.ApkModel
import org.argus.amandroid.core.appInfo.ApkCertificate
import org.argus.amandroid.core.decompile.DecompileLayout
import org.argus.amandroid.core.parser.{ComponentInfo, ComponentType, IntentFilterDataBase, LayoutControl}
import org.argus.jawa.core.elements.{JawaType, Signature}
import org.json4s.{CustomSerializer, Extraction, Formats, JValue}
import org.json4s.JsonDSL._

object ApkModelSerializer extends CustomSerializer[ApkModel](format => (
  {
    case jv: JValue =>
      implicit val formats: Formats = format + JawaTypeSerializer + JawaTypeKeySerializer + SignatureSerializer + SignatureKeySerializer + IntentFilterDataBaseSerializer + DecompileLayoutSerializer + new org.json4s.ext.EnumNameSerializer(ComponentType)
      val nameUri  = (jv \ "nameUri").extract[FileResourceUri]
      val layout = (jv \ "layout").extract[DecompileLayout]
      val certificates = (jv \ "certificates").extract[ISet[ApkCertificate]]
      val activities = (jv \ "activities").extract[ISet[JawaType]]
      val services = (jv \ "services").extract[ISet[JawaType]]
      val receivers = (jv \ "receivers").extract[ISet[JawaType]]
      val providers = (jv \ "provider").extract[ISet[JawaType]]
      val drReceivers = (jv \ "drReceivers").extract[ISet[JawaType]]
      val rpcMethods = (jv \ "rpcMethods").extract[IMap[JawaType, IMap[Signature, Boolean]]]
      val uses_permissions = (jv \ "uses_permissions").extract[ISet[String]]
      val callbackMethods = (jv \ "callbackMethods").extract[IMap[JawaType, ISet[Signature]]]
      val componentInfos = (jv \ "componentInfos").extract[ISet[ComponentInfo]]
      val layoutControls = (jv \ "layoutControls").extract[IMap[Int, LayoutControl]]
      val appPackageName = (jv \ "appPackageName").extract[Option[String]]
      val intentFdb = (jv \ "intentFdb").extract[IntentFilterDataBase]
      val codeLineCounter = (jv \ "codeLineCounter").extract[Int]
      val envMap = (jv \ "envMap").extract[IMap[JawaType, (Signature, String)]]
      val apk = ApkModel(nameUri, layout)
      apk.addCertificates(certificates)
      apk.addActivities(activities)
      apk.addServices(services)
      apk.addReceivers(receivers)
      apk.addProviders(providers)
      apk.addDynamicRegisteredReceivers(drReceivers)
      apk.addRpcMethods(rpcMethods)
      apk.addUsesPermissions(uses_permissions)
      apk.addCallbackMethods(callbackMethods)
      apk.addComponentInfos(componentInfos)
      apk.addLayoutControls(layoutControls)
      apk.setPackageName(appPackageName.getOrElse(""))
      apk.setIntentFilterDB(intentFdb)
      apk.setCodeLineCounter(codeLineCounter)
      apk.addEnvMap(envMap)
      apk
  },
  {
    case model: ApkModel =>
      implicit val formats: Formats = format + JawaTypeSerializer + JawaTypeKeySerializer + SignatureSerializer + SignatureKeySerializer + IntentFilterDataBaseSerializer + DecompileLayoutSerializer + new org.json4s.ext.EnumNameSerializer(ComponentType)
      val nameUri: FileResourceUri = model.nameUri
      val layout: DecompileLayout = model.layout
      val certificates: ISet[ApkCertificate] = model.getCertificates
      val activities: ISet[JawaType] = model.getActivities
      val services: ISet[JawaType] = model.getServices
      val receivers: ISet[JawaType] = model.getReceivers
      val providers: ISet[JawaType] = model.getProviders
      val drReceivers: ISet[JawaType] = model.getDynamicRegisteredReceivers
      val rpcMethods: IMap[JawaType, IMap[Signature, Boolean]] = model.getRpcMethodMapping
      val uses_permissions: ISet[String] = model.getUsesPermissions
      val callbackMethods: IMap[JawaType, ISet[Signature]] = model.getCallbackMethodMapping
      val componentInfos: ISet[ComponentInfo] = model.getComponentInfos
      val layoutControls: IMap[Int, LayoutControl] = model.getLayoutControls
      val appPackageName: String = model.getPackageName
      val intentFdb: IntentFilterDataBase = model.getIntentFilterDB
      val codeLineCounter: Int = model.getCodeLineCounter
      val envMap: IMap[JawaType, (Signature, String)] = model.getEnvMap
      ("nameUri" -> nameUri) ~
      ("layout" -> Extraction.decompose(layout)) ~
      ("certificates" -> Extraction.decompose(certificates)) ~
      ("activities" -> Extraction.decompose(activities)) ~
      ("services" -> Extraction.decompose(services)) ~
      ("receivers" -> Extraction.decompose(receivers)) ~
      ("providers" -> Extraction.decompose(providers)) ~
      ("drReceivers" -> Extraction.decompose(drReceivers)) ~
      ("rpcMethods" -> Extraction.decompose(rpcMethods)) ~
      ("uses_permissions" -> Extraction.decompose(uses_permissions)) ~
      ("callbackMethods" -> Extraction.decompose(callbackMethods)) ~
      ("componentInfos" -> Extraction.decompose(componentInfos)) ~
      ("layoutControls" -> Extraction.decompose(layoutControls)) ~
      ("appPackageName" -> Option(appPackageName)) ~
      ("intentFdb" -> Extraction.decompose(intentFdb)) ~
      ("codeLineCounter" -> codeLineCounter) ~
      ("envMap" -> Extraction.decompose(envMap))
  }
)) 

package org.argus.amandroid.serialization

import org.argus.amandroid.core.decompile.DecompileLayout
import org.argus.jawa.core.util.{FileResourceUri, ISet}
import org.json4s.{CustomSerializer, Extraction, Formats, JValue}
import org.json4s.JsonDSL._


object DecompileLayoutSerializer extends CustomSerializer[DecompileLayout](format => (
  {
    case jv: JValue =>
      implicit val formats: Formats = format
      val outputUri  = (jv \ "outputUri").extract[FileResourceUri]
      val createFolder = (jv \ "createFolder").extract[Boolean]
      val srcFolder = (jv \ "srcFolder").extract[String]
      val libFolder = (jv \ "libFolder").extract[String]
      val createSeparateFolderForDexes = (jv \ "createSeparateFolderForDexes").extract[Boolean]
      val pkg = (jv \ "pkg").extract[String]
      val outputSrcUri = (jv \ "outputSrcUri").extract[FileResourceUri]
      val sourceFolders = (jv \ "sourceFolders").extract[ISet[String]]
      val libFolders = (jv \ "libFolders").extract[ISet[String]]
      val dependencies = (jv \ "dependencies").extract[ISet[String]]
      val thirdPartyLibraries = (jv \ "thirdPartyLibraries").extract[ISet[String]]
      val layout = DecompileLayout(outputUri, createFolder, srcFolder, libFolder, createSeparateFolderForDexes)
      layout.pkg = pkg
      layout.outputSrcUri = outputSrcUri
      layout.sourceFolders = sourceFolders
      layout.libFolders = libFolders
      layout.dependencies = dependencies
      layout.thirdPartyLibraries = thirdPartyLibraries
      layout
  },
  {
    case layout: DecompileLayout =>
      implicit val formats: Formats = format
      val outputUri: FileResourceUri = layout.outputSrcUri
      val createFolder: Boolean = layout.createFolder
      val srcFolder: String = layout.srcFolder
      val libFolder: String = layout.libFolder
      val createSeparateFolderForDexes: Boolean = layout.createSeparateFolderForDexes
      val pkg: String = layout.pkg
      val outputSrcUri: FileResourceUri = layout.outputSrcUri
      val sourceFolders: ISet[String] = layout.sourceFolders
      val libFolders: ISet[String] = layout.libFolders
      val dependencies: ISet[String] = layout.dependencies
      val thirdPartyLibraries: ISet[String] = layout.thirdPartyLibraries
      ("outputUri" -> outputUri) ~
      ("createFolder" -> createFolder) ~
      ("srcFolder" -> srcFolder) ~
      ("libFolder" -> libFolder) ~
      ("createSeparateFolderForDexes" -> createSeparateFolderForDexes) ~
      ("pkg" -> pkg) ~
      ("outputSrcUri" -> outputSrcUri) ~
      ("sourceFolders" -> Extraction.decompose(sourceFolders)) ~
      ("libFolders" -> Extraction.decompose(libFolders)) ~
      ("dependencies" -> Extraction.decompose(dependencies)) ~
      ("thirdPartyLibraries" -> Extraction.decompose(thirdPartyLibraries))
  }
)) 

package is.hail.expr.ir

import java.io.OutputStreamWriter

import is.hail.utils._
import is.hail.types._
import is.hail.io.fs.FS
import is.hail.rvd._
import org.json4s.jackson.JsonMethods
import org.json4s.{DefaultFormats, Extraction, Formats, JValue, ShortTypeHints}

import scala.language.implicitConversions

object SortOrder {
  def deserialize(b: Byte): SortOrder =
    if (b == 0.toByte) Ascending
    else if (b == 1.toByte) Descending
    else throw new RuntimeException(s"invalid sort order: $b")
}

sealed abstract class SortOrder {
  def serialize: Byte
}

case object Ascending extends SortOrder {
  def serialize: Byte = 0.toByte
}

case object Descending extends SortOrder {
  def serialize: Byte = 1.toByte
}

case class SortField(field: String, sortOrder: SortOrder)

abstract class AbstractTableSpec extends RelationalSpec {
  def references_rel_path: String

  def table_type: TableType

  def rowsComponent: RVDComponentSpec = getComponent[RVDComponentSpec]("rows")

  def rowsSpec: AbstractRVDSpec

  def globalsSpec: AbstractRVDSpec

  def indexed: Boolean = rowsSpec.indexed
}

object TableSpec {
  def apply(fs: FS, path: String, params: TableSpecParameters): TableSpec = {
    val globalsComponent = params.components("globals").asInstanceOf[RVDComponentSpec]
    val globalsSpec = globalsComponent.rvdSpec(fs, path)

    val rowsComponent = params.components("rows").asInstanceOf[RVDComponentSpec]
    val rowsSpec = rowsComponent.rvdSpec(fs, path)

    new TableSpec(params, globalsSpec, rowsSpec)
  }

  def fromJValue(fs: FS, path: String, jv: JValue): TableSpec = {
    implicit val formats: Formats = RelationalSpec.formats
    val params = jv.extract[TableSpecParameters]
    TableSpec(fs, path, params)
  }
}

case class TableSpecParameters(
  file_version: Int,
  hail_version: String,
  references_rel_path: String,
  table_type: TableType,
  components: Map[String, ComponentSpec]) {

  def write(fs: FS, path: String) {
    using(new OutputStreamWriter(fs.create(path + "/metadata.json.gz"))) { out =>
      out.write(JsonMethods.compact(decomposeWithName(this, "TableSpec")(RelationalSpec.formats)))
    }
  }
}

class TableSpec(
  val params: TableSpecParameters,
  val globalsSpec: AbstractRVDSpec,
  val rowsSpec: AbstractRVDSpec) extends AbstractTableSpec {
  def file_version: Int = params.file_version

  def hail_version: String = params.hail_version

  def components: Map[String, ComponentSpec] = params.components

  def references_rel_path: String = params.references_rel_path

  def table_type: TableType = params.table_type

  def toJValue: JValue = {
    decomposeWithName(params, "TableSpec")(RelationalSpec.formats)
  }
} 

package com.salesforce.op.utils.json

import com.salesforce.op.test.TestCommon
import org.json4s.jackson.JsonMethods._
import org.json4s.{DefaultFormats, Extraction, Formats}
import org.junit.runner.RunWith
import org.scalatest.FlatSpec
import org.scalatest.junit.JUnitRunner


@RunWith(classOf[JUnitRunner])
class SpecialDoubleSerializerTest extends FlatSpec with TestCommon {

  val data = Map(
    "normal" -> Seq(-1.1, 0.0, 2.3),
    "infs" -> Seq(Double.NegativeInfinity, Double.PositiveInfinity),
    "minMax" -> Seq(Double.MinValue, Double.MaxValue),
    "nan" -> Seq(Double.NaN)
  )

  Spec[SpecialDoubleSerializer] should behave like
    readWriteDoubleValues(data)(
      json = """{"normal":[-1.1,0.0,2.3],"infs":["-Infinity","Infinity"],"minMax":[-1.7976931348623157E308,1.7976931348623157E308],"nan":["NaN"]}""" // scalastyle:off
    )(DefaultFormats + new SpecialDoubleSerializer)

  Spec[SpecialDoubleSerializer] + " (with big decimal)" should behave like
    readWriteDoubleValues(data)(
      json = """{"normal":[-1.1,0.0,2.3],"infs":["-Infinity","Infinity"],"minMax":[-1.7976931348623157E+308,1.7976931348623157E+308],"nan":["NaN"]}""" // scalastyle:off
    )(DefaultFormats.withBigDecimal + new SpecialDoubleSerializer)


  def readWriteDoubleValues(input: Map[String, Seq[Double]])(json: String)(implicit formats: Formats): Unit = {
    it should "write double entries" in {
      compact(Extraction.decompose(input)) shouldBe json
    }
    it should "read double entries" in {
      val parsed = parse(json).extract[Map[String, Seq[Double]]]
      parsed.keys shouldBe input.keys
      parsed zip input foreach {
        case (("nan", a), ("nan", b)) => a.foreach(_.isNaN shouldBe true)
        case ((_, a), (_, b)) => a should contain theSameElementsAs b
      }
    }
  }
} 

package com.salesforce.op.utils.json

import com.salesforce.op.test.TestCommon
import enumeratum.{Enum, EnumEntry}
import org.json4s.jackson.JsonMethods._
import org.json4s.{DefaultFormats, Extraction}
import org.junit.runner.RunWith
import org.scalatest.FunSpec
import org.scalatest.junit.JUnitRunner

import scala.util.Success


@RunWith(classOf[JUnitRunner])
class EnumEntrySerializerTest extends FunSpec with TestCommon {

  implicit val formats = DefaultFormats + EnumEntrySerializer.json4s[TestEnumType](TestEnumType)
  val serdes = Seq(EnumEntrySerializer.jackson[TestEnumType](TestEnumType))

  val data = TestData(a = TestEnumType.One, b = Seq(TestEnumType.Two, TestEnumType.Three))
  val dataJson = """{"a":"One","b":["Two","Three"]}"""

  describe("EnumEntrySerializer") {
    describe("(json4s)") {
      it("write enum entries") {
        compact(Extraction.decompose(data)) shouldBe dataJson
      }
      it("read enum entries") {
        parse(dataJson).extract[TestData] shouldBe data
      }
      it("read enum entries ignoring case") {
        parse(dataJson.toLowerCase).extract[TestData] shouldBe data
      }
    }
    describe("(jackson)") {
      it("write enum entries") {
        JsonUtils.toJsonString(data, pretty = false, serdes = serdes) shouldBe dataJson
      }
      it("read enum entries") {
        JsonUtils.fromString[TestData](dataJson, serdes = serdes) shouldBe Success(data)
      }
      it("read enum entries ignoring case") {
        JsonUtils.fromString[TestData](dataJson.toLowerCase, serdes = serdes) shouldBe Success(data)
      }
    }
  }

}

private[json] case class TestData(a: TestEnumType, b: Seq[TestEnumType])

sealed trait TestEnumType extends EnumEntry with Serializable
object TestEnumType extends Enum[TestEnumType] {
  val values = findValues
  case object One extends TestEnumType
  case object Two extends TestEnumType
  case object Three extends TestEnumType
} 

package io.vamp.common.util

import org.json4s.{ Extraction, Formats }

import scala.collection.JavaConverters._
import scala.reflect.runtime.currentMirror
import scala.reflect.runtime.universe._

object ObjectUtil {

  def isPrimitive(any: Any) = any match {
    case _: Boolean ⇒ true
    case _: Byte    ⇒ true
    case _: Char    ⇒ true
    case _: Short   ⇒ true
    case _: Int     ⇒ true
    case _: Long    ⇒ true
    case _: Float   ⇒ true
    case _: Double  ⇒ true
    case _: String  ⇒ true
    case _          ⇒ false
  }

  def unwrap: Any ⇒ Any = {
    case p if isPrimitive(p)  ⇒ p
    case e: Enumeration#Value ⇒ e
    case l: List[_]           ⇒ l.map(unwrap)
    case m: Map[_, _]         ⇒ m.map { case (k, v) ⇒ k → unwrap(v) }
    case null                 ⇒ None
    case Some(s)              ⇒ Option(unwrap(s))
    case None                 ⇒ None
    case any ⇒
      val reflection = currentMirror.reflect(any)
      currentMirror.reflect(any).symbol.typeSignature.members.toList
        .collect { case s: TermSymbol if !s.isMethod ⇒ reflection.reflectField(s) }
        .map(r ⇒ r.symbol.name.toString.trim → unwrap(r.get))
        .toMap
  }

  def asJava: Any ⇒ Any = {
    case l: List[_]   ⇒ l.map(asJava).asJava
    case m: Map[_, _] ⇒ m.map({ case (k, v) ⇒ k → asJava(v) }).asJava
    case Some(s)      ⇒ Option(asJava(s))
    case any          ⇒ any
  }

  def asScala: Any ⇒ AnyRef = {
    case value: java.util.Map[_, _]   ⇒ value.asScala.map({ case (k, v) ⇒ k → asScala(v) }).toMap
    case value: java.util.List[_]     ⇒ value.asScala.map(asScala).toList
    case value: java.lang.Iterable[_] ⇒ value.asScala.map(asScala).toList
    case value: java.util.Optional[_] ⇒ if (value.isPresent) Option(asScala(value.get)) else None
    case value                        ⇒ value.asInstanceOf[AnyRef]
  }

  def merge(maps: Map[String, Any]*)(implicit formats: Formats): Map[String, AnyRef] = {
    maps.tail.foldLeft(Extraction.decompose(maps.head)) {
      (op1, op2) ⇒ op1 merge Extraction.decompose(op2)
    }.extract[Map[String, AnyRef]]
  }
}