org.json4s.JsonAST.JObject Scala Examples

package org.apache.spark.deploy

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._

import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner

private[deploy] object JsonProtocol {
 def writeWorkerInfo(obj: WorkerInfo): JObject = {
   ("id" -> obj.id) ~
   ("host" -> obj.host) ~
   ("port" -> obj.port) ~
   ("webuiaddress" -> obj.webUiAddress) ~
   ("cores" -> obj.cores) ~
   ("coresused" -> obj.coresUsed) ~
   ("coresfree" -> obj.coresFree) ~
   ("memory" -> obj.memory) ~
   ("memoryused" -> obj.memoryUsed) ~
   ("memoryfree" -> obj.memoryFree) ~
   ("state" -> obj.state.toString) ~
   ("lastheartbeat" -> obj.lastHeartbeat)
 }

  def writeApplicationInfo(obj: ApplicationInfo): JObject = {
    ("starttime" -> obj.startTime) ~
    ("id" -> obj.id) ~
    ("name" -> obj.desc.name) ~
    ("cores" -> obj.desc.maxCores) ~
    ("user" ->  obj.desc.user) ~
    ("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
    ("submitdate" -> obj.submitDate.toString) ~
    ("state" -> obj.state.toString) ~
    ("duration" -> obj.duration)
  }

  def writeApplicationDescription(obj: ApplicationDescription): JObject = {
    ("name" -> obj.name) ~
    ("cores" -> obj.maxCores) ~
    ("memoryperslave" -> obj.memoryPerExecutorMB) ~
    ("user" -> obj.user) ~
    ("command" -> obj.command.toString)
  }

  def writeExecutorRunner(obj: ExecutorRunner): JObject = {
    ("id" -> obj.execId) ~
    ("memory" -> obj.memory) ~
    ("appid" -> obj.appId) ~
    ("appdesc" -> writeApplicationDescription(obj.appDesc))
  }

  def writeDriverInfo(obj: DriverInfo): JObject = {
    ("id" -> obj.id) ~
    ("starttime" -> obj.startTime.toString) ~
    ("state" -> obj.state.toString) ~
    ("cores" -> obj.desc.cores) ~
    ("memory" -> obj.desc.mem)
  }

  def writeMasterState(obj: MasterStateResponse): JObject = {
    val aliveWorkers = obj.workers.filter(_.isAlive())
    ("url" -> obj.uri) ~
    ("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
    ("cores" -> aliveWorkers.map(_.cores).sum) ~
    ("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~
    ("memory" -> aliveWorkers.map(_.memory).sum) ~
    ("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~
    ("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
    ("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
    ("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
    ("status" -> obj.status.toString)
  }

  def writeWorkerState(obj: WorkerStateResponse): JObject = {
    ("id" -> obj.workerId) ~
    ("masterurl" -> obj.masterUrl) ~
    ("masterwebuiurl" -> obj.masterWebUiUrl) ~
    ("cores" -> obj.cores) ~
    ("coresused" -> obj.coresUsed) ~
    ("memory" -> obj.memory) ~
    ("memoryused" -> obj.memoryUsed) ~
    ("executors" -> obj.executors.toList.map(writeExecutorRunner)) ~
    ("finishedexecutors" -> obj.finishedExecutors.toList.map(writeExecutorRunner))
  }
} 

package stellar.sdk.model

import org.json4s.JsonAST.JObject
import org.json4s.{DefaultFormats, Formats, JArray, JValue}
import stellar.sdk.KeyPair
import stellar.sdk.model.AmountParser.{AssetDeserializer, parseAsset}
import stellar.sdk.model.response.ResponseParser

case class PaymentPath(source: Amount, destination: Amount, path: Seq[Asset])

object PaymentPathDeserializer extends ResponseParser[PaymentPath]({
  o: JObject =>
    implicit val formats = DefaultFormats
    implicit val assetDeserializer = AssetDeserializer
    
    PaymentPath(
      source = AmountParser.amount("source_", o),
      destination = AmountParser.amount("destination_", o),
      path = {
        val JArray(values) = (o \ "path").extract[JArray]
        values.map { jv => parseAsset("", jv) }
      }
    )
})

object AmountParser {

  implicit val formats = DefaultFormats

  def parseAsset(prefix: String, o: JValue)(implicit formats: Formats): Asset = {
    val assetType = (o \ s"${prefix}asset_type").extract[String]
    def code = (o \ s"${prefix}asset_code").extract[String]
    def issuer = KeyPair.fromAccountId((o \ s"${prefix}asset_issuer").extract[String])
    assetType match {
      case "native" => NativeAsset
      case "credit_alphanum4" => IssuedAsset4(code, issuer)
      case "credit_alphanum12" => IssuedAsset12(code, issuer)
      case t => throw new RuntimeException(s"Unrecognised ${prefix}asset type: $t")
    }
  }

  def amount(prefix: String, o: JObject)(implicit formats: Formats): Amount = {
    val asset = parseAsset(prefix, o)
    val units = Amount.toBaseUnits((o \ s"${prefix}amount").extract[String]).get
    Amount(units, asset)
  }

  object AssetDeserializer extends ResponseParser[Asset](parseAsset("", _))
} 

package stellar.sdk.model

import java.time.ZonedDateTime

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import stellar.sdk.model.response.ResponseParser
import stellar.sdk.{KeyPair, PublicKeyOps}

case class Trade(id: String, ledgerCloseTime: ZonedDateTime, offerId: Long,
                 baseOfferId: Long, counterOfferId: Long,
                 baseAccount: PublicKeyOps, baseAmount: Amount,
                 counterAccount: PublicKeyOps, counterAmount: Amount,
                 baseIsSeller: Boolean)


object TradeDeserializer extends ResponseParser[Trade]({
  o: JObject =>
    implicit val formats = DefaultFormats

    def account(accountKey: String = "account") = KeyPair.fromAccountId((o \ accountKey).extract[String])

    def date(key: String) = ZonedDateTime.parse((o \ key).extract[String])

    def doubleFromString(key: String) = (o \ key).extract[String].toDouble

    def asset(prefix: String = "", issuerKey: String = "asset_issuer") = {
      def assetCode = (o \ s"${prefix}asset_code").extract[String]

      def assetIssuer = KeyPair.fromAccountId((o \ s"$prefix$issuerKey").extract[String])

      (o \ s"${prefix}asset_type").extract[String] match {
        case "native" => NativeAsset
        case "credit_alphanum4" => IssuedAsset4(assetCode, assetIssuer)
        case "credit_alphanum12" => IssuedAsset12(assetCode, assetIssuer)
        case t => throw new RuntimeException(s"Unrecognised asset type '$t'")
      }
    }

    def amount(prefix: String = "") = {
      val units = Amount.toBaseUnits(doubleFromString(s"${prefix}amount")).get
      asset(prefix) match {
        case nna: NonNativeAsset => IssuedAmount(units, nna)
        case NativeAsset => NativeAmount(units)
      }
    }

    Trade(
      id = (o \ "id").extract[String],
      ledgerCloseTime = date("ledger_close_time"),
      offerId = (o \ "offer_id").extract[String].toLong,
      baseOfferId = (o \ "base_offer_id").extract[String].toLong,
      counterOfferId = (o \ "counter_offer_id").extract[String].toLong,
      baseAccount = account("base_account"),
      baseAmount = amount("base_"),
      counterAccount = account("counter_account"),
      counterAmount = amount("counter_"),
      baseIsSeller = (o \ "base_is_seller").extract[Boolean]
    )
}) 

package stellar.sdk.model

import java.time.Instant
import java.util.concurrent.TimeUnit

import org.json4s.JsonAST.JObject
import org.json4s.{DefaultFormats, JValue}
import stellar.sdk.model.response.ResponseParser

import scala.concurrent.duration.Duration

case class TradeAggregation(instant: Instant, tradeCount: Int, baseVolume: Double, counterVolume: Double,
                            average: Double, open: Price, high: Price, low: Price, close: Price)

object TradeAggregationDeserializer extends ResponseParser[TradeAggregation]({ o: JObject =>
  implicit val formats = DefaultFormats

  def price(p: JValue): Price = Price((p \ "N").extract[Int], (p \ "D").extract[Int])

  TradeAggregation(
    instant = Instant.ofEpochMilli((o \ "timestamp").extract[String].toLong),
    tradeCount = (o \ "trade_count").extract[String].toInt,
    baseVolume = (o \ "base_volume").extract[String].toDouble,
    counterVolume = (o \ "counter_volume").extract[String].toDouble,
    average = (o \ "avg").extract[String].toDouble,
    open = price(o \ "open_r"),
    high = price(o \ "high_r"),
    low = price(o \ "low_r"),
    close = price(o \ "close_r"))
})

object TradeAggregation {

  sealed class Resolution(val duration: Duration)

  val OneMinute = new Resolution(Duration.create(1, TimeUnit.MINUTES))
  val FiveMinutes = new Resolution(OneMinute.duration * 5.0)
  val FifteenMinutes = new Resolution(FiveMinutes.duration * 3.0)
  val OneHour = new Resolution(FifteenMinutes.duration * 4.0)
  val OneDay = new Resolution(OneHour.duration * 24.0)
  val OneWeek = new Resolution(OneDay.duration * 7.0)

} 

package stellar.sdk.model.result

import java.time.ZonedDateTime

import org.json4s.{DefaultFormats, Formats}
import org.json4s.JsonAST.JObject
import stellar.sdk.model._
import stellar.sdk.model.ledger.TransactionLedgerEntries.arr
import stellar.sdk.model.ledger.{LedgerEntryChange, LedgerEntryChanges, TransactionLedgerEntries}
import stellar.sdk.model.response.ResponseParser
import stellar.sdk.util.ByteArrays.base64
import stellar.sdk.{KeyPair, PublicKey}

import scala.util.Try


case class TransactionHistory(hash: String, ledgerId: Long, createdAt: ZonedDateTime, account: PublicKey,
                              sequence: Long, maxFee: NativeAmount, feeCharged: NativeAmount, operationCount: Int,
                              memo: Memo, signatures: Seq[String], envelopeXDR: String, resultXDR: String,
                              resultMetaXDR: String, feeMetaXDR: String, validAfter: Option[ZonedDateTime],
                              validBefore: Option[ZonedDateTime], feeBump: Option[FeeBumpHistory]) {

  lazy val result: TransactionResult = TransactionResult.decodeXDR(resultXDR)

  def ledgerEntries: TransactionLedgerEntries = TransactionLedgerEntries.decodeXDR(resultMetaXDR)
  def feeLedgerEntries: Seq[LedgerEntryChange] = LedgerEntryChanges.decodeXDR(feeMetaXDR)

  @deprecated("Replaced by `feeCharged`", "v0.7.2")
  val feePaid: NativeAmount = feeCharged

}


object TransactionHistoryDeserializer extends {
} with ResponseParser[TransactionHistory]({
  o: JObject =>
    implicit val formats: Formats = DefaultFormats

    val maxFee = NativeAmount((o \ "max_fee").extract[String].toInt)
    val signatures = (o \ "signatures").extract[List[String]]
    val hash = (o \ "hash").extract[String]

    val inner = for {
      hash <- (o \ "inner_transaction" \ "hash").extractOpt[String]
      maxFee <- (o \ "inner_transaction" \ "max_fee").extractOpt[Int].map(NativeAmount(_))
      signatures <- (o \ "inner_transaction" \ "signatures").extractOpt[List[String]]
    } yield (hash, maxFee, signatures)

    TransactionHistory(
      hash = inner.map(_._1).getOrElse(hash),
      ledgerId = (o \ "ledger").extract[Long],
      createdAt = ZonedDateTime.parse((o \ "created_at").extract[String]),
      account = KeyPair.fromAccountId((o \ "source_account").extract[String]),
      sequence = (o \ "source_account_sequence").extract[String].toLong,
      maxFee = inner.map(_._2).getOrElse(maxFee),
      feeCharged = NativeAmount((o \ "fee_charged").extract[String].toInt),
      operationCount = (o \ "operation_count").extract[Int],
      memo = (o \ "memo_type").extract[String] match {
        case "none" => NoMemo
        case "id" => MemoId(BigInt((o \ "memo").extract[String]).toLong)
        case "text" => MemoText((o \ "memo").extractOpt[String].getOrElse(""))
        case "hash" => MemoHash(base64((o \ "memo").extract[String]).toIndexedSeq)
        case "return" => MemoReturnHash(base64((o \ "memo").extract[String]).toIndexedSeq)
      },
      signatures = inner.map(_._3).getOrElse(signatures),
      envelopeXDR = (o \ "envelope_xdr").extract[String],
      resultXDR = (o \ "result_xdr").extract[String],
      resultMetaXDR = (o \ "result_meta_xdr").extract[String],
      feeMetaXDR = (o \ "fee_meta_xdr").extract[String],
      // TODO (jem) - Remove the Try wrappers when https://github.com/stellar/go/issues/1381 is fixed.
      validBefore = Try((o \ "valid_before").extractOpt[String].map(ZonedDateTime.parse)).getOrElse(None),
      validAfter = Try((o \ "valid_after").extractOpt[String].map(ZonedDateTime.parse)).getOrElse(None),
      feeBump = inner.map { _ => FeeBumpHistory(maxFee, hash, signatures) }
    )
}) 

package stellar.sdk.inet

import java.net.HttpURLConnection.{HTTP_BAD_REQUEST, HTTP_NOT_FOUND}

import okhttp3.HttpUrl
import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import org.json4s.native.JsonMethods
import org.specs2.mutable.Specification
import stellar.sdk.model.response.ResponseParser

class PageSpec extends Specification {

  implicit val formats = DefaultFormats + RawPageDeserializer + HelloDeserializer

  "page parsing" should {
    "return an empty page if no results were found" >> {
      val page = PageParser.parse[String](HttpUrl.parse("http://localhost/"), HTTP_NOT_FOUND, "")
      page.xs must beEmpty
    }

    "throw a bad request exception with the reasons when provided" >> {
      val url = HttpUrl.parse("http://localhost/")
      PageParser.parse[String](url, HTTP_BAD_REQUEST,
        """{
          |  "type": "https://stellar.org/horizon-errors/bad_request",
          |  "title": "Bad Request",
          |  "status": 400,
          |  "detail": "The request you sent was invalid in some way.",
          |  "extras": {
          |    "invalid_field": "cursor",
          |    "reason": "cursor must contain exactly one colon"
          |  }
          |}""".stripMargin) must throwA[HorizonBadRequest].like { e =>
        e.getMessage mustEqual "Bad request. http://localhost/ -> cursor must contain exactly one colon"
      }
    }

    "throw a bad request exception with the full document when the reason is not provided" >> {
      val url = HttpUrl.parse("http://localhost/")
      PageParser.parse[String](url, HTTP_BAD_REQUEST, "random text") must throwA[HorizonBadRequest].like { e =>
        e.getMessage mustEqual "Bad request. http://localhost/ -> random text"
      }
    }

    "parse the member values and provide a link to the next page" >> {
      val doc =
        """
          |{
          |  "_links": {
          |    "self": {
          |      "href": "https://horizon-testnet.stellar.org/hello?cursor=\u0026limit=10\u0026order=asc"
          |    },
          |    "next": {
          |      "href": "https://horizon-testnet.stellar.org/hello?cursor=2045052972961793-0\u0026limit=10\u0026order=asc"
          |    },
          |    "prev": {
          |      "href": "https://horizon-testnet.stellar.org/hello?cursor=940258535411713-0\u0026limit=10\u0026order=desc"
          |    }
          |  },
          |  "_embedded": {
          |    "records": [
          |      {"hello":"world"},
          |      {"hello":"whirled"}
          |    ]
          |  }
          |}
        """.stripMargin

      JsonMethods.parse(doc).extract[RawPage].parse[String](HttpUrl.parse("http://localhost/")) mustEqual Page(
        List("world", "whirled"),
        nextLink = Some(HttpUrl.parse("https://horizon-testnet.stellar.org/hello?cursor=2045052972961793-0&limit=10&order=asc"))
      )
    }
  }

  object HelloDeserializer extends ResponseParser[String]({ o: JObject =>
    implicit val formats = DefaultFormats
    (o \ "hello").extract[String]
  })

} 

package org.apache.livy.thriftserver.types

import org.json4s.{DefaultFormats, JValue}
import org.json4s.JsonAST.{JObject, JString}
import org.json4s.jackson.JsonMethods.parse


  def schemaFromSparkJson(sparkJson: String): Schema = {
    val schema = parse(sparkJson) \ "fields"
    val fields = schema.children.map { field =>
      val name = (field \ "name").extract[String]
      val hiveType = toFieldType(field \ "type")
      // TODO: retrieve comment from metadata
      Field(name, hiveType, "")
    }
    Schema(fields.toArray)
  }
} 

package org.apache.spark.deploy

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._

import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner

private[deploy] object JsonProtocol {
 def writeWorkerInfo(obj: WorkerInfo): JObject = {
   ("id" -> obj.id) ~
   ("host" -> obj.host) ~
   ("port" -> obj.port) ~
   ("webuiaddress" -> obj.webUiAddress) ~
   ("cores" -> obj.cores) ~
   ("coresused" -> obj.coresUsed) ~
   ("coresfree" -> obj.coresFree) ~
   ("memory" -> obj.memory) ~
   ("memoryused" -> obj.memoryUsed) ~
   ("memoryfree" -> obj.memoryFree) ~
   ("state" -> obj.state.toString) ~
   ("lastheartbeat" -> obj.lastHeartbeat)
 }

  def writeApplicationInfo(obj: ApplicationInfo): JObject = {
    ("starttime" -> obj.startTime) ~
    ("id" -> obj.id) ~
    ("name" -> obj.desc.name) ~
    ("cores" -> obj.desc.maxCores) ~
    ("user" -> obj.desc.user) ~
    ("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
    ("submitdate" -> obj.submitDate.toString) ~
    ("state" -> obj.state.toString) ~
    ("duration" -> obj.duration)
  }

  def writeApplicationDescription(obj: ApplicationDescription): JObject = {
    ("name" -> obj.name) ~
    ("cores" -> obj.maxCores) ~
    ("memoryperslave" -> obj.memoryPerExecutorMB) ~
    ("user" -> obj.user) ~
    ("command" -> obj.command.toString)
  }

  def writeExecutorRunner(obj: ExecutorRunner): JObject = {
    ("id" -> obj.execId) ~
    ("memory" -> obj.memory) ~
    ("appid" -> obj.appId) ~
    ("appdesc" -> writeApplicationDescription(obj.appDesc))
  }

  def writeDriverInfo(obj: DriverInfo): JObject = {
    ("id" -> obj.id) ~
    ("starttime" -> obj.startTime.toString) ~
    ("state" -> obj.state.toString) ~
    ("cores" -> obj.desc.cores) ~
    ("memory" -> obj.desc.mem)
  }

  def writeMasterState(obj: MasterStateResponse): JObject = {
    val aliveWorkers = obj.workers.filter(_.isAlive())
    ("url" -> obj.uri) ~
    ("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
    ("cores" -> aliveWorkers.map(_.cores).sum) ~
    ("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~
    ("memory" -> aliveWorkers.map(_.memory).sum) ~
    ("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~
    ("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
    ("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
    ("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
    ("status" -> obj.status.toString)
  }

  def writeWorkerState(obj: WorkerStateResponse): JObject = {
    ("id" -> obj.workerId) ~
    ("masterurl" -> obj.masterUrl) ~
    ("masterwebuiurl" -> obj.masterWebUiUrl) ~
    ("cores" -> obj.cores) ~
    ("coresused" -> obj.coresUsed) ~
    ("memory" -> obj.memory) ~
    ("memoryused" -> obj.memoryUsed) ~
    ("executors" -> obj.executors.toList.map(writeExecutorRunner)) ~
    ("finishedexecutors" -> obj.finishedExecutors.toList.map(writeExecutorRunner))
  }
} 

package org.clulab.wm.eidos.apps.batch

import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.meta.CluText
import org.json4s.JsonAST.JField
import org.json4s.JsonAST.JObject
import org.json4s.JsonAST.JString

import scala.collection.mutable

object FilterCluKeysFromDirectory extends App {
  val metaDir = args(0)
  val files = FileUtils.findFiles(metaDir, "json")
  val keys = mutable.Set.empty[String]

  files.foreach { file =>
    try {
      println(s"Extracting from ${file.getName}")
      val jValue = CluText.getJValue(file)

      val newKeys = for {
        JObject(mt) <- jValue
        JField("N", JString(key)) <- mt
      } yield
        key.toString

      keys ++= newKeys
      println(keys)
    }
    catch {
      case exception: Exception =>
        println(s"Exception for file $file")
        exception.printStackTrace()
    }
  }
  println(keys)
} 

package is.hail.services.batch_client

import is.hail.utils._

import org.json4s.JsonAST.{JArray, JBool, JInt, JObject, JString}
import org.json4s.{DefaultFormats, Formats}
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test

class BatchClientSuite extends TestNGSuite {
  @Test def testBasic(): Unit = {
    val client = new BatchClient()
    val token = tokenUrlSafe(32)
    val batch = client.run(
      JObject(
        "billing_project" -> JString("test"),
        "n_jobs" -> JInt(1),
        "token" -> JString(token)),
      FastIndexedSeq(
        JObject(
          "always_run" -> JBool(false),
          "image" -> JString("ubuntu:18.04"),
          "mount_docker_socket" -> JBool(false),
          "command" -> JArray(List(
            JString("/bin/bash"),
            JString("-c"),
            JString("echo 'Hello, world!'"))),
          "job_id" -> JInt(0),
          "parent_ids" -> JArray(List()))))
    implicit val formats: Formats = DefaultFormats
    assert((batch \ "state").extract[String] == "success")
  }
} 

package org.apache.spark.deploy

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._

import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner

private[deploy] object JsonProtocol {
 def writeWorkerInfo(obj: WorkerInfo): JObject = {
   ("id" -> obj.id) ~
   ("host" -> obj.host) ~
   ("port" -> obj.port) ~
   ("webuiaddress" -> obj.webUiAddress) ~
   ("cores" -> obj.cores) ~
   ("coresused" -> obj.coresUsed) ~
   ("coresfree" -> obj.coresFree) ~
   ("memory" -> obj.memory) ~
   ("memoryused" -> obj.memoryUsed) ~
   ("memoryfree" -> obj.memoryFree) ~
   ("state" -> obj.state.toString) ~
   ("lastheartbeat" -> obj.lastHeartbeat)
 }

  def writeApplicationInfo(obj: ApplicationInfo): JObject = {
    ("starttime" -> obj.startTime) ~
    ("id" -> obj.id) ~
    ("name" -> obj.desc.name) ~
    ("cores" -> obj.desc.maxCores) ~
    ("user" -> obj.desc.user) ~
    ("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
    ("submitdate" -> obj.submitDate.toString) ~
    ("state" -> obj.state.toString) ~
    ("duration" -> obj.duration)
  }

  def writeApplicationDescription(obj: ApplicationDescription): JObject = {
    ("name" -> obj.name) ~
    ("cores" -> obj.maxCores) ~
    ("memoryperslave" -> obj.memoryPerExecutorMB) ~
    ("user" -> obj.user) ~
    ("command" -> obj.command.toString)
  }

  def writeExecutorRunner(obj: ExecutorRunner): JObject = {
    ("id" -> obj.execId) ~
    ("memory" -> obj.memory) ~
    ("appid" -> obj.appId) ~
    ("appdesc" -> writeApplicationDescription(obj.appDesc))
  }

  def writeDriverInfo(obj: DriverInfo): JObject = {
    ("id" -> obj.id) ~
    ("starttime" -> obj.startTime.toString) ~
    ("state" -> obj.state.toString) ~
    ("cores" -> obj.desc.cores) ~
    ("memory" -> obj.desc.mem)
  }

  def writeMasterState(obj: MasterStateResponse): JObject = {
    val aliveWorkers = obj.workers.filter(_.isAlive())
    ("url" -> obj.uri) ~
    ("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
    ("cores" -> aliveWorkers.map(_.cores).sum) ~
    ("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~
    ("memory" -> aliveWorkers.map(_.memory).sum) ~
    ("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~
    ("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
    ("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
    ("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
    ("status" -> obj.status.toString)
  }

  def writeWorkerState(obj: WorkerStateResponse): JObject = {
    ("id" -> obj.workerId) ~
    ("masterurl" -> obj.masterUrl) ~
    ("masterwebuiurl" -> obj.masterWebUiUrl) ~
    ("cores" -> obj.cores) ~
    ("coresused" -> obj.coresUsed) ~
    ("memory" -> obj.memory) ~
    ("memoryused" -> obj.memoryUsed) ~
    ("executors" -> obj.executors.toList.map(writeExecutorRunner)) ~
    ("finishedexecutors" -> obj.finishedExecutors.toList.map(writeExecutorRunner))
  }
} 

package org.apache.spark.deploy

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._

import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner

private[deploy] object JsonProtocol {
 def writeWorkerInfo(obj: WorkerInfo): JObject = {
   ("id" -> obj.id) ~
   ("host" -> obj.host) ~
   ("port" -> obj.port) ~
   ("webuiaddress" -> obj.webUiAddress) ~
   ("cores" -> obj.cores) ~
   ("coresused" -> obj.coresUsed) ~
   ("coresfree" -> obj.coresFree) ~
   ("memory" -> obj.memory) ~
   ("memoryused" -> obj.memoryUsed) ~
   ("memoryfree" -> obj.memoryFree) ~
   ("state" -> obj.state.toString) ~
   ("lastheartbeat" -> obj.lastHeartbeat)
 }

  def writeApplicationInfo(obj: ApplicationInfo): JObject = {
    ("starttime" -> obj.startTime) ~
    ("id" -> obj.id) ~
    ("name" -> obj.desc.name) ~
    ("cores" -> obj.desc.maxCores) ~
    ("user" ->  obj.desc.user) ~
    ("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
    ("submitdate" -> obj.submitDate.toString) ~
    ("state" -> obj.state.toString) ~
    ("duration" -> obj.duration)
  }

  def writeApplicationDescription(obj: ApplicationDescription): JObject = {
    ("name" -> obj.name) ~
    ("cores" -> obj.maxCores) ~
    ("memoryperslave" -> obj.memoryPerExecutorMB) ~
    ("user" -> obj.user) ~
    ("command" -> obj.command.toString)
  }

  def writeExecutorRunner(obj: ExecutorRunner): JObject = {
    ("id" -> obj.execId) ~
    ("memory" -> obj.memory) ~
    ("appid" -> obj.appId) ~
    ("appdesc" -> writeApplicationDescription(obj.appDesc))
  }

  def writeDriverInfo(obj: DriverInfo): JObject = {
    ("id" -> obj.id) ~
    ("starttime" -> obj.startTime.toString) ~
    ("state" -> obj.state.toString) ~
    ("cores" -> obj.desc.cores) ~
    ("memory" -> obj.desc.mem)
  }

  def writeMasterState(obj: MasterStateResponse): JObject = {
    ("url" -> obj.uri) ~
    ("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
    ("cores" -> obj.workers.map(_.cores).sum) ~
    ("coresused" -> obj.workers.map(_.coresUsed).sum) ~
    ("memory" -> obj.workers.map(_.memory).sum) ~
    ("memoryused" -> obj.workers.map(_.memoryUsed).sum) ~
    ("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
    ("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
    ("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
    ("status" -> obj.status.toString)
  }

  def writeWorkerState(obj: WorkerStateResponse): JObject = {
    ("id" -> obj.workerId) ~
    ("masterurl" -> obj.masterUrl) ~
    ("masterwebuiurl" -> obj.masterWebUiUrl) ~
    ("cores" -> obj.cores) ~
    ("coresused" -> obj.coresUsed) ~
    ("memory" -> obj.memory) ~
    ("memoryused" -> obj.memoryUsed) ~
    ("executors" -> obj.executors.toList.map(writeExecutorRunner)) ~
    ("finishedexecutors" -> obj.finishedExecutors.toList.map(writeExecutorRunner))
  }
} 

package com.salesforce.op.stages

import com.salesforce.op.stages.OpPipelineStageReaderWriter._
import com.salesforce.op.stages.sparkwrappers.generic.SparkWrapperParams
import org.apache.hadoop.fs.Path
import org.apache.spark.ml.util.MLWriter
import org.apache.spark.ml.{Estimator, SparkDefaultParamsReadWrite}
import org.json4s.JsonAST.{JObject, JValue}
import org.json4s.jackson.JsonMethods.{compact, render}

import scala.util.{Failure, Success}


  def writeToJson(path: String): JObject = {
    stage match {
      case _: Estimator[_] => return JObject() // no need to serialize estimators
      case s: SparkWrapperParams[_] =>
        // Set save path for all Spark wrapped stages of type [[SparkWrapperParams]] so they can save
        s.setStageSavePath(path)
      case _ =>
    }
    // We produce stage metadata for all the Spark params
    val metadata = SparkDefaultParamsReadWrite.getMetadataToSave(stage)

    // Write out the stage using the specified writer instance
    val writer = readerWriterFor[OpPipelineStageBase](stage.getClass.asInstanceOf[Class[OpPipelineStageBase]])
    val stageJson: JValue = writer.write(stage) match {
      case Failure(err) => throw new RuntimeException(s"Failed to write out stage '${stage.uid}'", err)
      case Success(json) => json
    }

    // Join metadata & with stage ctor args
    val j = metadata.merge(JObject(FieldNames.CtorArgs.entryName -> stageJson))
    render(j).asInstanceOf[JObject]
  }

} 

package com.salesforce.op.stages

import com.salesforce.op.stages.sparkwrappers.generic.SparkWrapperParams
import org.apache.hadoop.fs.Path
import org.apache.spark.ml.PipelineStage
import org.apache.spark.ml.param.{Param, ParamPair, Params}
import org.apache.spark.ml.util.{Identifiable, MLReader, MLWritable}
import org.apache.spark.util.SparkUtils
import org.json4s.JsonAST.{JObject, JValue}
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods.{compact, parse, render}
import org.json4s.{DefaultFormats, Formats, JString}

class SparkStageParam[S <: PipelineStage with Params]
(
  parent: String,
  name: String,
  doc: String,
  isValid: Option[S] => Boolean
) extends Param[Option[S]](parent, name, doc, isValid) {

  import SparkStageParam._

  
  override def jsonDecode(jsonStr: String): Option[S] = {
    val json = parse(jsonStr)
    val uid = (json \ "uid").extractOpt[String]
    val path = (json \ "path").extractOpt[String]

    path -> uid match {
      case (None, _) | (_, None) | (_, Some(NoUID)) =>
        savePath = None
        None
      case (Some(p), Some(stageUid)) =>
        savePath = Option(p)
        val stagePath = new Path(p, stageUid).toString
        val className = (json \ "className").extract[String]
        val cls = SparkUtils.classForName(className)
        val stage = cls.getMethod("read").invoke(null).asInstanceOf[MLReader[PipelineStage]].load(stagePath)
        Option(stage).map(_.asInstanceOf[S])
    }
  }
}

object SparkStageParam {
  implicit val formats: Formats = DefaultFormats
  val NoClass = ""
  val NoUID = ""

  def updateParamsMetadataWithPath(jValue: JValue, path: String): JValue = jValue match {
    case JObject(pairs) => JObject(
      pairs.map {
        case (SparkWrapperParams.SparkStageParamName, j) =>
          SparkWrapperParams.SparkStageParamName -> j.merge(JObject("path" -> JString(path)))
        case param => param
      }
    )
    case j => throw new IllegalArgumentException(s"Cannot recognize JSON Spark params metadata: $j")
  }

} 

package com.salesforce.op.features

import com.salesforce.op.features.types._
import com.salesforce.op.stages.{OPStage, OpPipelineStage}
import org.json4s.JsonAST.{JObject, JValue}
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods
import org.json4s.jackson.JsonMethods._
import org.json4s.{DefaultFormats, Formats}

import scala.reflect.runtime.universe.WeakTypeTag
import scala.util.Try



  def fromJson(
    json: JValue,
    stages: Map[String, OPStage],
    features: Map[String, OPFeature]
  ): Try[OPFeature] = Try {
    val typeName = (json \ "typeName").extract[String]
    val uid = (json \ "uid").extract[String]
    val name = (json \ "name").extract[String]
    val isResponse = (json \ "isResponse").extract[Boolean]
    val originStageUid = (json \ "originStage").extract[String]
    val parentUids = (json \ "parents").extract[Array[String]]

    val originStage: Option[OPStage] = stages.get(originStageUid)
    if (originStage.isEmpty) {
      throw new RuntimeException(s"Origin stage $originStageUid not found for feature $name ($uid)")
    }

    // Order is important and so are duplicates, eg f = f1 + f1 has 2 parents but both the same feature
    val parents: Seq[OPFeature] = parentUids.flatMap(id => features.get(id))
    if (parents.length != parentUids.length) {
      throw new RuntimeException(s"Not all the parent features were found for feature $name ($uid)")
    }

    val wtt = FeatureType.featureTypeTag(typeName).asInstanceOf[WeakTypeTag[FeatureType]]
    Feature[FeatureType](
      uid = uid,
      name = name,
      isResponse = isResponse,
      parents = parents,
      originStage = originStage.get.asInstanceOf[OpPipelineStage[FeatureType]]
    )(wtt = wtt)

  }

} 

package org.apache.spark.deploy

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._

import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner

private[deploy] object JsonProtocol {
 def writeWorkerInfo(obj: WorkerInfo): JObject = {
   ("id" -> obj.id) ~
   ("host" -> obj.host) ~
   ("port" -> obj.port) ~
   ("webuiaddress" -> obj.webUiAddress) ~
   ("cores" -> obj.cores) ~
   ("coresused" -> obj.coresUsed) ~
   ("coresfree" -> obj.coresFree) ~
   ("memory" -> obj.memory) ~
   ("memoryused" -> obj.memoryUsed) ~
   ("memoryfree" -> obj.memoryFree) ~
   ("state" -> obj.state.toString) ~
   ("lastheartbeat" -> obj.lastHeartbeat)
 }

  def writeApplicationInfo(obj: ApplicationInfo): JObject = {
    ("starttime" -> obj.startTime) ~
    ("id" -> obj.id) ~
    ("name" -> obj.desc.name) ~
    ("cores" -> obj.desc.maxCores) ~
    ("user" ->  obj.desc.user) ~
    ("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
    ("submitdate" -> obj.submitDate.toString) ~
    ("state" -> obj.state.toString) ~
    ("duration" -> obj.duration)
  }

  def writeApplicationDescription(obj: ApplicationDescription): JObject = {
    ("name" -> obj.name) ~
    ("cores" -> obj.maxCores) ~
    ("memoryperslave" -> obj.memoryPerExecutorMB) ~
    ("user" -> obj.user) ~
    ("command" -> obj.command.toString)
  }

  def writeExecutorRunner(obj: ExecutorRunner): JObject = {
    ("id" -> obj.execId) ~
    ("memory" -> obj.memory) ~
    ("appid" -> obj.appId) ~
    ("appdesc" -> writeApplicationDescription(obj.appDesc))
  }

  def writeDriverInfo(obj: DriverInfo): JObject = {
    ("id" -> obj.id) ~
    ("starttime" -> obj.startTime.toString) ~
    ("state" -> obj.state.toString) ~
    ("cores" -> obj.desc.cores) ~
    ("memory" -> obj.desc.mem)
  }

  def writeMasterState(obj: MasterStateResponse): JObject = {
    val aliveWorkers = obj.workers.filter(_.isAlive())
    ("url" -> obj.uri) ~
    ("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
    ("cores" -> aliveWorkers.map(_.cores).sum) ~
    ("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~
    ("memory" -> aliveWorkers.map(_.memory).sum) ~
    ("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~
    ("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
    ("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
    ("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
    ("status" -> obj.status.toString)
  }

  def writeWorkerState(obj: WorkerStateResponse): JObject = {
    ("id" -> obj.workerId) ~
    ("masterurl" -> obj.masterUrl) ~
    ("masterwebuiurl" -> obj.masterWebUiUrl) ~
    ("cores" -> obj.cores) ~
    ("coresused" -> obj.coresUsed) ~
    ("memory" -> obj.memory) ~
    ("memoryused" -> obj.memoryUsed) ~
    ("executors" -> obj.executors.toList.map(writeExecutorRunner)) ~
    ("finishedexecutors" -> obj.finishedExecutors.toList.map(writeExecutorRunner))
  }
} 

package stellar.sdk.model.response

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import stellar.sdk._
import stellar.sdk.model.{Amount, IssuedAsset12, IssuedAsset4, NonNativeAsset}

case class AssetResponse(asset: NonNativeAsset, amount: Long, numAccounts: Int, authRequired: Boolean, authRevocable: Boolean)

object AssetRespDeserializer extends ResponseParser[AssetResponse]({ o: JObject =>
  implicit val formats = DefaultFormats
  val asset = {
    val code = (o \ "asset_code").extract[String]
    val issuer = KeyPair.fromAccountId((o \ "asset_issuer").extract[String])
    (o \ "asset_type").extract[String] match {
      case "credit_alphanum4" => IssuedAsset4(code, issuer)
      case "credit_alphanum12" => IssuedAsset12(code, issuer)
      case t => throw new RuntimeException(s"Unrecognised asset type: $t")
    }
  }
  val amount = Amount.toBaseUnits((o \ "amount").extract[String].toDouble).getOrElse(
    throw new RuntimeException(s"Invalid asset amount: ${(o \ "amount").extract[Double]}"))
  val numAccounts = (o \ "num_accounts").extract[Int]
  val authRequired = (o \ "flags" \ "auth_required").extract[Boolean]
  val authRevocable = (o \ "flags" \ "auth_revocable").extract[Boolean]
  AssetResponse(asset, amount, numAccounts, authRequired, authRevocable)
}) 

package org.squbs.admin

import java.net.URLEncoder

import akka.http.scaladsl.model.Uri.Path
import akka.http.scaladsl.model._
import org.json4s.JsonAST.{JObject, JString}
import org.json4s.jackson.JsonMethods._
import org.squbs.unicomplex.{RouteDefinition, WebContext}
import org.squbs.util.ConfigUtil._

class AdminSvc extends RouteDefinition with WebContext {

  val prefix = if (webContext == "") "/bean" else s"/$webContext/bean"

  val exclusions = context.system.settings.config.get[Seq[String]]("squbs.admin.exclusions", Seq.empty[String]).toSet
  val (exBeans, exFieldSet) = exclusions partition { !_.contains("::") }

  val exFields = exFieldSet map { fieldSpec =>
    val fields = fieldSpec split "::"
    fields(0) -> fields(1)
  } groupBy (_._1) mapValues { _.map(_._2) }


  val route =
    get {
      pathEndOrSingleSlash {
        extractUri { uri =>
          complete {
            val kv = MBeanUtil.allObjectNames collect {
              case name if !(exBeans contains name) =>
                val resource = Path(s"$prefix/${URLEncoder.encode(name.replace('=', '~'), "UTF-8")}")
                name -> JString(uri.withPath(resource).toString())
            }
            HttpResponse(entity = HttpEntity(ContentTypes.`application/json`, pretty(render(JObject(kv)))))
          }
        }
      } ~
      path("bean" / Segment) { encName =>
        complete {
          val name = encName.replace('~', '=').replace('%', '/')
          val response: HttpResponse =
            if (exBeans contains name) HttpResponse(StatusCodes.NotFound, entity = StatusCodes.NotFound.defaultMessage)
            else MBeanUtil.asJSON(name, exFields getOrElse (name, Set.empty))
              .map { json => HttpResponse(entity = json) }
              .getOrElse (HttpResponse(StatusCodes.NotFound, entity = StatusCodes.NotFound.defaultMessage))
          response
        }
      }
    }
} 

package io.vamp.common.json

import org.json4s.JsonAST.{ JObject, JString }
import org.json4s._

object ThrowableSerializer {
  def apply(message: Option[String]): SerializationFormat = new SerializationFormat {
    override def customSerializers = super.customSerializers :+ new ThrowableSerializer(message)
  }
}

class ThrowableSerializer(message: Option[String] = None) extends Serializer[Throwable] {

  override def serialize(implicit format: Formats): PartialFunction[Any, JValue] = {
    case t: Throwable ⇒ new JObject(List(JField("message", JString(message.getOrElse(t.getMessage)))))
  }

  override def deserialize(implicit format: Formats): PartialFunction[(TypeInfo, JValue), Throwable] = SerializationFormat.unsupported
} 

package io.vamp.common.json

import io.vamp.common.util.TextUtil
import org.json4s.JsonAST.JObject
import org.json4s._

object MapSerializer extends SerializationFormat {
  override def customSerializers: List[Serializer[_]] = super.customSerializers :+ new MapSerializer()
}

class MapSerializer extends Serializer[Map[_, _]] {
  override def serialize(implicit format: Formats): PartialFunction[Any, JValue] = {
    case map: Map[_, _] ⇒
      new JObject(map.map {
        case (name, value) ⇒
          val newName = if (name.toString.contains("-") || name.toString.contains("_") || name.toString.contains("/")) name.toString else TextUtil.toSnakeCase(name.toString, dash = false)
          JField(newName, Extraction.decompose(value))
      }.toList)
  }

  def deserialize(implicit format: Formats): PartialFunction[(TypeInfo, JValue), Map[_, _]] = SerializationFormat.unsupported
} 

package com.gu.teamcity

import java.io.ByteArrayInputStream
import java.util.Date

import jetbrains.buildServer.messages.{BuildMessage1, DefaultMessagesInfo, Status}
import jetbrains.buildServer.serverSide.{BuildServerAdapter, SRunningBuild}
import org.joda.time.{DateTime, DateTimeZone}
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._

import scala.util.{Failure, Success}

class ManifestUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapter {

  override def beforeBuildFinish(runningBuild: SRunningBuild) {
    import scala.collection.convert.wrapAsScala._

    if (!runningBuild.isHasInternalArtifactsOnly) {
      val properties = Seq(
        "projectName" -> S3Plugin.cleanFullName(runningBuild),
        "buildNumber" -> runningBuild.getBuildNumber,
        "startTime" -> new DateTime(runningBuild.getStartDate).withZone(DateTimeZone.UTC).toString //Joda default is ISO8601
      ) ++ runningBuild.getRevisions.flatMap(revision => Seq(
        "revision" -> revision.getRevision,
        "vcsURL" -> revision.getRoot.getProperties.get("url")
      )) ++ Option(runningBuild.getBranch).map(b =>
        "branch" -> b.getDisplayName
      ).orElse(runningBuild.getVcsRootEntries.headOption.map(r =>
        "branch" -> r.getProperties.get("branch")
      ))

      val propertiesJSON = pretty(render(properties.foldLeft(JObject())(_ ~ _)))
      val jsBytes = propertiesJSON.getBytes("UTF-8")

      config.buildManifestBucket.map { bucket =>
        s3.upload(bucket, runningBuild, "build.json", new ByteArrayInputStream(jsBytes), jsBytes.length) match {			
          case Failure(e) => runningBuild.getBuildLog().message(s"Error uploading manifest: ${e.getMessage}",
              Status.ERROR,new Date,DefaultMessagesInfo.MSG_BUILD_FAILURE,DefaultMessagesInfo.SOURCE_ID,null)
          case Success(_) => runningBuild.getBuildLog().message("Manifest S3 upload complete",
              Status.NORMAL,new Date,DefaultMessagesInfo.MSG_TEXT,DefaultMessagesInfo.SOURCE_ID,null) 
        }
      }
    }
  }

  private def normalMessage(text: String) =
    new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_TEXT, Status.NORMAL, new Date, text)
} 

package com.tencent.angel.sona.ml.param
import com.tencent.angel.sona.ml.util.Identifiable
import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import org.json4s.jackson.JsonMethods.{compact, parse, render}

class JObjectParam(parent: String, name: String, doc: String, isValid: JObject => Boolean)
  extends Param[JObject](parent, name, doc, isValid) {
  def this(parent: String, name: String, doc: String) =
    this(parent, name, doc, (value: JObject) => value != null)

  def this(parent: Identifiable, name: String, doc: String, isValid: JObject => Boolean) =
    this(parent.uid, name, doc, isValid)

  def this(parent: Identifiable, name: String, doc: String) = this(parent.uid, name, doc)

  override def w(value: JObject): ParamPair[JObject] = super.w(value)

  override def jsonEncode(value: JObject): String = {
    compact(render(value))
  }

  override def jsonDecode(json: String): JObject = {
    implicit val formats: DefaultFormats = DefaultFormats
    parse(json).asInstanceOf[JObject]
  }
} 

package io.coral.actors.transform

import akka.actor.Props
import io.coral.actors.{SimpleEmitTrigger, CoralActor}
import io.coral.lib.JsonTemplate
import org.json4s.JsonAST.{JObject, JValue}
import org.json4s._

object JsonActor {
	implicit val formats = org.json4s.DefaultFormats

	def getParams(json: JValue) = {
		for {
			template <- (json \ "params" \ "template").extractOpt[JObject]
			if (JsonTemplate.validate(template))
		} yield {
			template
		}
	}

	def apply(json: JValue): Option[Props] = {
		getParams(json).map(_ => Props(classOf[JsonActor], json))
	}

}

class JsonActor(json: JObject)
	extends CoralActor(json)
	with SimpleEmitTrigger {

	val template = JsonTemplate(JsonActor.getParams(json).get)

	override def simpleEmitTrigger(json: JObject): Option[JValue] = {
		Some(template.interpret(json))
	}

} 

package io.coral.actors.connector

import java.io.FileWriter
import akka.actor.{ActorLogging, Props}
import io.coral.actors.CoralActor
import org.json4s.JsonAST.JObject
import org.json4s._
import org.json4s.jackson.JsonMethods._
import scala.concurrent.Future


object LogActor {
	implicit val formats = org.json4s.DefaultFormats

	def getParams(json: JValue) = {
		val file = (json \ "params" \ "file").extractOpt[String]
		val append = (json \ "params" \ "append").extractOpt[Boolean]
		Some((file, append getOrElse false))
	}

	def apply(json: JValue): Option[Props] = {
		getParams(json).map(_ => Props(classOf[LogActor], json))
	}
}

class LogActor(json: JObject) extends CoralActor(json) with ActorLogging {
	val (file, append) = LogActor.getParams(json).get
	var fileWriter: Option[FileWriter] = None

	override def preStart() = {
		if (file.isDefined) {
			fileWriter = Some(new FileWriter(file.get, append))
		}
	}

	override def postStop() = {
		fileWriter match {
			case None =>
			case Some(f) => f.close()

		}
	}

	override def trigger = {
		json => Future {
			fileWriter match {
				case None =>
					log.info(compact(json))
				case Some(f) =>
					f.write(compact(json) + "\n")
					f.flush()
			}

			Some(JNothing)
		}
	}
} 

package io.coral.actors.connector

import java.util.Properties

import akka.actor.{Props, ActorLogging}
import io.coral.actors.{NoEmitTrigger, CoralActor}
import io.coral.lib.KafkaJsonProducer.KafkaEncoder
import io.coral.lib.{KafkaJsonProducer, ConfigurationBuilder}
import org.json4s.JsonAST.{JObject, JValue}
import kafka.serializer.Encoder

object KafkaProducerActor {
	implicit val formats = org.json4s.DefaultFormats
	val builder = new ConfigurationBuilder("kafka.producer")

	def getParams(json: JValue) = {
		for {
			kafka <- (json \ "params" \ "kafka").extractOpt[JObject]
			topic <- (json \ "params" \ "topic").extractOpt[String]
		} yield {
			val properties = producerProperties(kafka)
			(properties, topic)
		}
	}

	private def producerProperties(json: JObject): Properties = {
		val properties = builder.properties
		json.values.foreach { case (k: String, v: String) => properties.setProperty(k, v) }
		properties
	}

	def apply(json: JValue): Option[Props] = {
		getParams(json).map(_ => Props(classOf[KafkaProducerActor[KafkaEncoder]], json, KafkaJsonProducer()))
	}

	def apply[T <: KafkaEncoder](json: JValue, encoder: Class[T]): Option[Props] = {
		getParams(json).map(_ => Props(classOf[KafkaProducerActor[T]], json, KafkaJsonProducer(encoder)))
	}
}

class KafkaProducerActor[T <: Encoder[JValue]](json: JObject, connection: KafkaJsonProducer[T])
	extends CoralActor(json)
	with NoEmitTrigger
	with ActorLogging {
	val (properties, topic) = KafkaProducerActor.getParams(json).get
	lazy val kafkaSender = connection.createSender(topic, properties)

	override def noEmitTrigger(json: JObject) = {
		val key = (json \ "key").extractOpt[String]
		val message = (json \ "message").extract[JObject]
		send(key, message)
	}

	private def send(key: Option[String], message: JObject) = {
		try {
			kafkaSender.send(key, message)
		} catch {
			case e: Exception => log.error(e, "failed to send message to Kafka")
		}
	}
} 

package io.coral.actors.connector

import java.util.Properties

import akka.actor.Props
import io.coral.actors.CoralActor
import io.coral.actors.connector.KafkaConsumerActor.{StopReadingMessageQueue, ReadMessageQueue}
import io.coral.lib.{ConfigurationBuilder, KafkaJsonConsumer}
import kafka.serializer.Decoder
import kafka.tools.MessageFormatter
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.json4s.JsonAST.{JNothing, JObject, JValue}

object KafkaConsumerActor {
	case class ReadMessageQueue()
	case class StopReadingMessageQueue()

	implicit val formats = org.json4s.DefaultFormats
	val builder = new ConfigurationBuilder("kafka.consumer")

	def getParams(json: JValue) = {
		for {
			kafka <- (json \ "params" \ "kafka").extractOpt[JObject]
			topic <- (json \ "params" \ "topic").extractOpt[String]
		} yield {
			val properties = consumerProperties(kafka)
			(properties, topic)
		}
	}

	def consumerProperties(json: JObject): Properties = {
		val properties = builder.properties

		json.values.foreach {
			case (k: String, v: String) =>
				properties.setProperty(k, v)
		}

		properties
	}

	def apply(json: JValue): Option[Props] = {
		getParams(json).map(_ => Props(classOf[KafkaConsumerActor], json, KafkaJsonConsumer()))
	}

	def apply(json: JValue, decoder: Decoder[JValue]): Option[Props] = {
		getParams(json).map(_ => Props(classOf[KafkaConsumerActor], json, KafkaJsonConsumer(decoder)))
	}
}

class KafkaConsumerActor(json: JObject, connection: KafkaJsonConsumer) extends CoralActor(json) {
	val (properties, topic) = KafkaConsumerActor.getParams(json).get
	lazy val stream = connection.stream(topic, properties)
	var shouldStop = false

	override def preStart(): Unit = {
		super.preStart()
	}

	override def receiveExtra: Receive = {
		case ReadMessageQueue() if stream.hasNextInTime =>
			val message: JValue = stream.next
			stream.commitOffsets

			if (message != JNothing) {
				emit(message)
			}

			if (!shouldStop) {
				self ! ReadMessageQueue()
			}
		case ReadMessageQueue() =>
			self ! ReadMessageQueue()
		case StopReadingMessageQueue() =>
			shouldStop = true
	}

	
} 

package io.coral.actors

import org.json4s._

import scala.concurrent.Future
import org.json4s.JsonAST.{JObject, JValue}

trait Trigger {
	type TriggerType = JObject => Future[Option[JValue]]

	def trigger: TriggerType
}

trait NoEmitTrigger extends Trigger {
	override def trigger: TriggerType =
		json => {
			noEmitTrigger(json)
			Future.successful(Some(JNothing))
		}

	def noEmitTrigger(json: JObject): Unit
}

trait NoTrigger extends Trigger {
	override def trigger: TriggerType = json => Future.successful(Some(JNothing))
}

trait SimpleEmitTrigger extends Trigger {
	override def trigger: TriggerType = {
		json =>
			Future.successful(simpleEmitTrigger(json))
	}

	def simpleEmitTrigger(json: JObject): Option[JValue]
} 

package io.coral.lib

import java.util.Properties

import io.coral.lib.KafkaJsonProducer.KafkaEncoder
import kafka.producer.{KeyedMessage, ProducerConfig, Producer}
import kafka.serializer.Encoder
import kafka.utils.VerifiableProperties
import org.json4s.JsonAST.{JObject, JValue}
import org.json4s.jackson.JsonMethods._

object KafkaJsonProducer {
	type KafkaEncoder = Encoder[JValue]
	def apply() = new KafkaJsonProducer(classOf[JsonEncoder])
	def apply[T <: KafkaEncoder](encoder: Class[T]) = new KafkaJsonProducer(encoder)
}

class KafkaJsonProducer[T <: KafkaEncoder](encoderClass: Class[T]) {
	def createSender(topic: String, properties: Properties): KafkaSender = {
		val props = properties.clone.asInstanceOf[Properties]
		props.put("serializer.class", encoderClass.getName)
		val producer = createProducer(props)
		new KafkaSender(topic, producer)
	}

	def createProducer(props: Properties): Producer[String, JValue] = {
		new Producer[String, JValue](new ProducerConfig(props))
	}
}

class KafkaSender(topic: String, producer: Producer[String, JValue]) {
	def send(key: Option[String], message: JObject) = {
		val keyedMessage: KeyedMessage[String, JValue] = key match {
			case Some(key) => new KeyedMessage(topic, key, message)
			case None => new KeyedMessage(topic, message)
		}

		producer.send(keyedMessage)
	}
}

class JsonEncoder(verifiableProperties: VerifiableProperties) extends KafkaEncoder {
	override def toBytes(value: JValue): Array[Byte] = {
		compact(value).getBytes("UTF-8")
	}
} 

package io.coral.lib

import java.util.Properties

import io.coral.lib.KafkaJsonProducer.KafkaEncoder
import kafka.utils.VerifiableProperties
import org.json4s.JsonAST.{JObject, JValue}
import org.scalatest.{Matchers, WordSpec}
import org.json4s.jackson.JsonMethods._
import kafka.producer.{ProducerConfig, KeyedMessage, Producer}
import org.mockito.{Mockito, ArgumentCaptor}
import org.mockito.Mockito._
import scala.collection.mutable

class KafkaJsonProducerSpec extends WordSpec with Matchers {
	"A KafkaJsonProducer" should {
		"create a KafkaJsonProducer with the JsonEncoder" in {
			val producer = KafkaJsonProducer()
			assert(producer.getClass == classOf[KafkaJsonProducer[JsonEncoder]])
		}

		"create a KafkaJsonProducer with the specified Encoder" in {
			val producer = KafkaJsonProducer(classOf[MyEncoder])
			assert(producer.getClass == classOf[KafkaJsonProducer[MyEncoder]])
		}

		"create a sender" in {
			val producer = new MyKafkaJsonProducer
			producer.createSender("topic", new Properties)
			val serializer = producer.receivedProperties.get("serializer.class")
			assert(serializer == classOf[MyEncoder].getName)
		}
	}

	"A KafkaSender" should {
		"send the JSON provided without a key to Kafka" in {
			val messageJson = """{"key1": "value1", "key2": "value2"}"""

			val keyedMessage = sendMessage(None, messageJson)

			assert(keyedMessage.topic == "test")
			assert(keyedMessage.hasKey == false)
			assert(keyedMessage.message == parse(messageJson))
		}

		"send the JSON provided with a key to Kafka" in {
			val messageJson = """{"key3": "value3", "key4": "value4"}"""

			val keyedMessage = sendMessage(Some("key"), messageJson)

			assert(keyedMessage.key == "key")
			assert(keyedMessage.topic == "test")
			assert(keyedMessage.message == parse(messageJson))
		}
	}

	"A JsonEncoder" should {
		"encode the provided json" in {
			val json = """{"key1": "value1"}"""
			val encoder = new JsonEncoder(new VerifiableProperties)
			val result = encoder.toBytes(parse(json))
			assert(parse(new String(result, "UTF-8")) == parse(json))
		}
	}

	private def sendMessage(key: Option[String], messageJson: String): KeyedMessage[String, JValue] = {
		val producer = Mockito.mock(classOf[Producer[String, JValue]])
		val sender = new KafkaSender("test", producer)
		sender.send(key, parse(messageJson).asInstanceOf[JObject])

		val argumentCaptor = ArgumentCaptor.forClass(classOf[KeyedMessage[String, JValue]])
		verify(producer).send(argumentCaptor.capture())

		val keyedMessages = argumentCaptor.getAllValues
		assert(keyedMessages.size == 1)

		// The following construction is necessary because capturing of parameters
		// with Mockito, Scala type interference, and multiple arguments
		// don't work together without explicit casts.
		keyedMessages.get(0).asInstanceOf[mutable.WrappedArray.ofRef[KeyedMessage[String, JValue]]](0)
	}
}

class MyEncoder(verifiableProperties: VerifiableProperties) extends KafkaEncoder {
	override def toBytes(value: JValue): Array[Byte] = {
		Array()
	}
}

class MyKafkaJsonProducer extends KafkaJsonProducer(classOf[MyEncoder]) {
	var receivedProperties: Properties = _

	override def createProducer(props: Properties): Producer[String, JValue] = {
		receivedProperties = props
		Mockito.mock(classOf[Producer[String, JValue]])
	}
} 

package com.softwaremill.clippy

import org.json4s.JsonAST.{JField, JObject, JString, JValue}

case class Library(groupId: String, artifactId: String, version: String) {
  def toJson: JValue = JObject(
    "groupId"    -> JString(groupId),
    "artifactId" -> JString(artifactId),
    "version"    -> JString(version)
  )

  override def toString = s"$groupId:$artifactId:$version"
}

object Library {
  def fromJson(jvalue: JValue): Option[Library] =
    (for {
      JObject(fields)                           <- jvalue
      JField("groupId", JString(groupId))       <- fields
      JField("artifactId", JString(artifactId)) <- fields
      JField("version", JString(version))       <- fields
    } yield Library(groupId, artifactId, version)).headOption
} 

package scalapb.json4s

import com.google.protobuf.any.{Any => PBAny}
import org.json4s.JsonAST.{JNothing, JObject, JString, JValue}

import scala.language.existentials

object AnyFormat {
  val anyWriter: (Printer, PBAny) => JValue = {
    case (printer, any) =>
      // Find the companion so it can be used to JSON-serialize the message. Perhaps this can be circumvented by
      // including the original GeneratedMessage with the Any (at least in memory).
      val cmp = printer.typeRegistry
        .findType(any.typeUrl)
        .getOrElse(
          throw new IllegalStateException(
            s"Unknown type ${any.typeUrl} in Any.  Add a TypeRegistry that supports this type to the Printer."
          )
        )

      // Unpack the message...
      val message = any.unpack(cmp)

      // ... and add the @type marker to the resulting JSON
      printer.toJson(message) match {
        case JObject(fields) =>
          JObject(("@type" -> JString(any.typeUrl)) +: fields)
        case value =>
          // Safety net, this shouldn't happen
          throw new IllegalStateException(
            s"Message of type ${any.typeUrl} emitted non-object JSON: $value"
          )
      }
  }

  val anyParser: (Parser, JValue) => PBAny = {
    case (parser, obj @ JObject(fields)) =>
      obj \ "@type" match {
        case JString(typeUrl) =>
          val cmp = parser.typeRegistry
            .findType(typeUrl)
            .getOrElse(
              throw new JsonFormatException(
                s"Unknown type ${typeUrl} in Any.  Add a TypeRegistry that supports this type to the Parser."
              )
            )
          val message = parser.fromJson(obj, true)(cmp)
          PBAny(typeUrl = typeUrl, value = message.toByteString)

        case JNothing =>
          throw new JsonFormatException(s"Missing type url when parsing $obj")

        case unknown =>
          throw new JsonFormatException(
            s"Expected string @type field, got $unknown"
          )
      }

    case (_, unknown) =>
      throw new JsonFormatException(s"Expected an object, got $unknown")
  }
} 

package org.apress.prospark

import java.nio.charset.StandardCharsets

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.eclipse.paho.client.mqttv3.MqttClient
import org.eclipse.paho.client.mqttv3.MqttMessage
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence
import org.json4s.DefaultFormats
import org.json4s.JField
import org.json4s.JsonAST.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.parse
import org.json4s.string2JsonInput

object MqttSinkAppF {

  def main(args: Array[String]) {
    if (args.length != 3) {
      System.err.println(
        "Usage: MqttSinkApp <appname> <outputBrokerUrl> <topic>")
      System.exit(1)
    }

    val Seq(appName, outputBrokerUrl, topic) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val batchInterval = 10

    val ssc = new StreamingContext(conf, Seconds(batchInterval))

    val mqttSink = ssc.sparkContext.broadcast(MqttSinkLazy(outputBrokerUrl))

    HttpUtils.createStream(ssc, url = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22IBM,GOOG,MSFT,AAPL,FB,ORCL,YHOO,TWTR,LNKD,INTC%22)%0A%09%09&format=json&diagnostics=true&env=http%3A%2F%2Fdatatables.org%2Falltables.env",
      interval = batchInterval)
      .flatMap(rec => {
        val query = parse(rec) \ "query"
        ((query \ "results" \ "quote").children).map(rec => JObject(JField("Timestamp", query \ "created")).merge(rec))
      })
      .map(rec => {
        implicit val formats = DefaultFormats
        rec.children.map(f => f.extract[String]) mkString ","
      })
      .foreachRDD { rdd =>
        rdd.foreachPartition { par =>
          par.foreach(message => mqttSink.value.client.publish(topic, new MqttMessage(message.getBytes(StandardCharsets.UTF_8))))
        }
      }

    ssc.start()
    ssc.awaitTermination()
  }

}

class MqttSinkLazy(brokerUrl: String) extends Serializable {
  lazy val client = {
    val client = new MqttClient(brokerUrl, MqttClient.generateClientId(), new MemoryPersistence())
    client.connect()
    sys.addShutdownHook {
      client.disconnect()
      client.close()
    }
    client
  }
}

object MqttSinkLazy {
  val brokerUrl = "tcp://localhost:1883"
  val client = new MqttSinkLazy(brokerUrl)

  def apply(brokerUrl: String): MqttSinkLazy = {
    client
  }
} 

package io.iohk.ethereum.keystore

import java.util.UUID

import akka.util.ByteString
import io.iohk.ethereum.domain.Address
import io.iohk.ethereum.keystore.EncryptedKey._
import org.json4s.JsonAST.{JObject, JString, JValue}
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.{CustomSerializer, DefaultFormats, Extraction, JField}
import org.spongycastle.util.encoders.Hex

import scala.util.Try

object EncryptedKeyJsonCodec {

  private val byteStringSerializer = new CustomSerializer[ByteString](_ => (
    { case JString(s) => ByteString(Hex.decode(s)) },
    { case bs: ByteString => JString(Hex.toHexString(bs.toArray)) }
  ))

  private implicit val formats = DefaultFormats + byteStringSerializer

  private def asHex(bs: ByteString): String =
    Hex.toHexString(bs.toArray)

  def toJson(encKey: EncryptedKey): String = {
    import encKey._
    import cryptoSpec._

    val json =
      ("id" -> id.toString) ~
      ("address" -> asHex(address.bytes)) ~
      ("version" -> version) ~
      ("crypto" -> (
        ("cipher" -> cipher) ~
        ("ciphertext" -> asHex(ciphertext)) ~
        ("cipherparams" -> ("iv" -> asHex(iv))) ~
        encodeKdf(kdfParams) ~
        ("mac" -> asHex(mac))
      ))

    pretty(render(json))
  }

  def fromJson(jsonStr: String): Either[String, EncryptedKey] = Try {
    val json = parse(jsonStr).transformField { case JField(k, v) => JField(k.toLowerCase, v) }

    val uuid = UUID.fromString((json \ "id").extract[String])
    val address = Address((json \ "address").extract[String])
    val version = (json \ "version").extract[Int]

    val crypto = json \ "crypto"
    val cipher = (crypto \ "cipher").extract[String]
    val ciphertext = (crypto \ "ciphertext").extract[ByteString]
    val iv = (crypto \ "cipherparams" \ "iv").extract[ByteString]
    val mac = (crypto \ "mac").extract[ByteString]

    val kdfParams = extractKdf(crypto)
    val cryptoSpec = CryptoSpec(cipher, ciphertext, iv, kdfParams, mac)
    EncryptedKey(uuid, address, cryptoSpec, version)

  }.fold(ex => Left(ex.toString), encKey => Right(encKey))

  private def encodeKdf(kdfParams: KdfParams): JObject =
    kdfParams match {
      case ScryptParams(salt, n, r, p, dklen) =>
        ("kdf" -> Scrypt) ~
        ("kdfparams" -> Extraction.decompose(kdfParams))

      case Pbkdf2Params(salt, prf, c, dklen) =>
        ("kdf" -> Pbkdf2) ~
        ("kdfparams" -> Extraction.decompose(kdfParams))
    }

  private def extractKdf(crypto: JValue): KdfParams = {
    val kdf = (crypto \ "kdf").extract[String]
    kdf.toLowerCase match {
      case Scrypt =>
        (crypto \ "kdfparams").extract[ScryptParams]

      case Pbkdf2 =>
        (crypto \ "kdfparams").extract[Pbkdf2Params]
    }
  }

} 

package com.webank.wedatasphere.linkis.resourcemanager

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.{CustomSerializer, Extraction}


trait ResultResource

case class NotEnoughResource(reason: String = null) extends ResultResource

case class AvailableResource(ticketId: String) extends ResultResource

case class UserResultResource(ticketId: String, user: String) extends ResultResource

object ResultResourceSerializer extends CustomSerializer[ResultResource](implicit formats => ( {
  case JObject(List(("NotEnoughResource", JObject(List(("reason", reason)))))) => NotEnoughResource(reason.extract[String])
  case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => AvailableResource(ticketId.extract[String])
  case JObject(List(("UserResultResource", JObject(List(("ticketId", ticketId), ("user", user)))))) =>
    UserResultResource(ticketId.extract[String], user.extract[String])
}, {
  case r: NotEnoughResource => ("NotEnoughResource", ("reason", Extraction.decompose(r.reason)))
  case r: AvailableResource => ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId)))
  case r: UserResultResource => ("UserResultResource", ("ticketId", r.ticketId) ~ ("user", r.user))
})) 

package com.webank.wedatasphere.linkis.resourcemanager.domain

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}


case class ModuleResourceInfo(moduleInstance: ServiceInstance, totalResource: Resource, usedResource: Resource)

object ModuleResourceInfoSerializer extends CustomSerializer[ModuleResourceInfo](implicit formats => ( {
  case JObject(List(("ModuleResourceInfo", JObject(List(("moduleInstance", moduleInstance), ("totalResource", totalResource), ("usedResource", usedResource)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ModuleResourceInfo(moduleInstance.extract[ServiceInstance], totalResource.extract[Resource], usedResource.extract[Resource])
}, {
  case m: ModuleResourceInfo =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("ModuleResourceInfo", new JObject(List(("moduleInstance", Extraction.decompose(m.moduleInstance)), ("totalResource", Extraction.decompose(m.totalResource)), ("usedResource", Extraction.decompose(m.usedResource)))))
}))

case class ModuleResourceRecord(moduleInfo: ModuleInfo, moduleUsedResources: Resource, moduleLeftResource: Resource, moduleLockedResource: Resource, registerTime: Long = System.currentTimeMillis()) 

package com.webank.wedatasphere.linkis.resourcemanager.domain

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.ResourceRequestPolicy.ResourceRequestPolicy
import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceRequestPolicy, ResourceSerializer}
import org.json4s.JsonAST.JObject
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}


case class ModuleInfo(moduleInstance: ServiceInstance,
                      totalResource: Resource,
                      protectedResource: Resource, //Enter the protection mode when the resource reaches(当资源达到多少时，进入保护模式)
                      resourceRequestPolicy: ResourceRequestPolicy
                     )

object ModuleInfoSerializer extends CustomSerializer[ModuleInfo](implicit formats => ( {
  case JObject(List(("moduleInstance", moduleInstance), ("totalResource", totalResource), ("protectedResource", protectedResource), ("resourceRequestPolicy", resourceRequestPolicy))) =>
    implicit val formats = DefaultFormats + ResourceSerializer + ModuleInstanceSerializer
    new ModuleInfo(moduleInstance.extract[ServiceInstance], totalResource.extract[Resource], protectedResource.extract[Resource], ResourceRequestPolicy.withName(resourceRequestPolicy.extract[String]))
}, {
  case i: ModuleInfo =>
    implicit val formats = DefaultFormats + ResourceSerializer + ModuleInstanceSerializer
    val policy = Extraction.decompose(i.resourceRequestPolicy.toString)
    new JObject(List(("moduleInstance", Extraction.decompose(i.moduleInstance)), ("totalResource", Extraction.decompose(i.totalResource)), ("protectedResource", Extraction.decompose(i.protectedResource)), ("resourceRequestPolicy", policy)))
})
) 

package com.webank.wedatasphere.linkis.resourcemanager.domain

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}



trait UserResourceInfo

case class UserPreUsedResource(ticketId: String, moduleInstance: ServiceInstance, resource: Resource) extends UserResourceInfo

case class UserUsedResource(ticketId: String, moduleInstance: ServiceInstance, resource: Resource, engineInstance: ServiceInstance = null) extends UserResourceInfo

case class UserReleasedResource(ticketId: String, moduleInstance: ServiceInstance) extends UserResourceInfo


object UserResourceInfoSerializer extends CustomSerializer[UserResourceInfo](implicit formats => ( {
  case JObject(List(("UserPreUsedResource", JObject(List(("ticketId", ticketId), ("moduleInstance", moduleInstance), ("resource", resource)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    new UserPreUsedResource(ticketId.extract[String], moduleInstance.extract[ServiceInstance], resource.extract[Resource])
  case JObject(List(("UserUsedResource", JObject(List(("ticketId", ticketId), ("moduleInstance", moduleInstance), ("resource", resource), ("engineInstance", engineInstance)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    new UserUsedResource(ticketId.extract[String], moduleInstance.extract[ServiceInstance], resource.extract[Resource], engineInstance.extract[ServiceInstance])
  case JObject(List(("UserReleasedResource", JObject(List(("ticketId", ticketId), ("moduleInstance", moduleInstance)))))) =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    new UserReleasedResource(ticketId.extract[String], moduleInstance.extract[ServiceInstance])
}, {
  case d: UserPreUsedResource =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("UserPreUsedResource", new JObject(List(("ticketId", Extraction.decompose(d.ticketId)), ("moduleInstance", Extraction.decompose(d.moduleInstance)), ("resource", Extraction.decompose(d.resource)))))
  case d: UserUsedResource =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("UserUsedResource", new JObject(List(("ticketId", Extraction.decompose(d.ticketId)), ("moduleInstance", Extraction.decompose(d.moduleInstance)), ("resource", Extraction.decompose(d.resource)), ("engineInstance", Extraction.decompose(d.engineInstance)))))
  case d: UserReleasedResource =>
    implicit val formats = DefaultFormats + ModuleInstanceSerializer + ResourceSerializer
    ("UserReleasedResource", new JObject(List(("ticketId", Extraction.decompose(d.ticketId)), ("moduleInstance", Extraction.decompose(d.moduleInstance)))))
})
) 

package com.webank.wedatasphere.linkis.rpc

import java.lang.reflect.ParameterizedType
import java.util

import com.webank.wedatasphere.linkis.rpc.transform.{JavaCollectionSerializer, JavaMapSerializer}
import org.apache.commons.lang.ClassUtils
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.json4s.jackson.Serialization
import org.json4s.reflect.ManifestFactory
import org.json4s.{CustomSerializer, DefaultFormats, Extraction}

object RPCFormatsTest {

  trait ResultResource
  class AvailableResource(val ticketId: String) extends ResultResource

  object ResultResourceSerializer extends CustomSerializer[ResultResource](implicit formats => ({
    case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => new AvailableResource(ticketId.extract[String])
  },{
    case r: AvailableResource => ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId)))
  }))

  def testRPC1(args: Array[String]): Unit = {
    implicit val formats = DefaultFormats + ResultResourceSerializer
    val serializerClasses = formats.customSerializers.map(_.getClass.getGenericSuperclass match {
      case p: ParameterizedType =>
        val params = p.getActualTypeArguments
        if(params == null || params.isEmpty) null
        else params(0).asInstanceOf[Class[_]]
    }).filter(_ != null)
    val a = new AvailableResource("aaa")
    val str = Serialization.write(a)
    println(str)
    val clazz = classOf[AvailableResource]
    println(serializerClasses)
    val realClass1 = serializerClasses.find(ClassUtils.isAssignable(clazz, _))
    println(realClass1)
    val realClass = realClass1.getOrElse(clazz)
    val obj = Serialization.read(str)(formats, ManifestFactory.manifestOf(realClass))
    println(obj)
    println(classOf[Array[_]].getClass.getName)
  }

  case class TestCollection1(a: String, list: java.util.List[String])
  case class TestCollection2(a: String, list: java.util.Map[String, Integer])

  def testRPC2(args: Array[String]): Unit = {
    implicit val formats = DefaultFormats + JavaCollectionSerializer + JavaMapSerializer
    //    val a = TestCollection1("1", new util.ArrayList[String]())
    val a = TestCollection2("1", new util.HashMap[String, Integer]())
    //    a.list.add("1111")
    a.list.put("1111", 2)
    val str = Serialization.write(a)
    println(str)
    val realClass = classOf[TestCollection2]
    val obj = Serialization.read(str)(formats, ManifestFactory.manifestOf(realClass))
    println(obj)
  }

  def main(args: Array[String]): Unit = {
    testRPC2(args)
  }
} 

package rhttpc.transport.json4s

import org.json4s.JsonAST.{JObject, JString}
import org.json4s._

import scala.reflect.ClassTag

class CustomSerializerWithTypeHints[T: Manifest, JV <: JValue: ClassTag](
  ser: Formats => (PartialFunction[JV, T], PartialFunction[T, JV]))
  extends CustomSubTypesSerializer[T, JObject](implicit formats => {
  val (deserialize, serialize) = ser(formats)
  (
    {
      case JObject(_ :: ("value", jValue: JV) :: Nil) if deserialize.isDefinedAt(jValue) =>
        deserialize(jValue)
    },
    {
      case obj: T if serialize.isDefinedAt(obj) =>
        JObject(
          formats.typeHintFieldName -> JString(obj.getClass.getName),
          "value" -> serialize(obj)
        )
    }
  )
}) 

package org.apache.spark.sql.execution.datasources.druid

import org.json4s.JsonAST.{JArray, JObject, JValue}
import org.json4s.JsonDSL._
import scala.collection.{immutable, mutable}

case class DescTableRequest(dataSource: String) {

  def toJson: JValue = {
    JObject("queryType" -> "segmentMetadata", "dataSource" -> dataSource)
  }
}

case class DescTableResponse(data: Seq[(String, Any)])

object DescTableResponse {
  def parse(js: JValue): DescTableResponse = {
    var arr = new mutable.HashMap[String, Any]
    js match {
      case JArray(results) =>
        val columns = (results.last \ "columns").asInstanceOf[JObject].values
        columns.foreach { col =>
          arr += (col._1 -> col._2
            .asInstanceOf[immutable.HashMap[String, String]]
            .get("type")
            .get)
        }
        DescTableResponse(arr.toSeq.sortBy(_._1))
      case err @ _ =>
        throw new IllegalArgumentException("Invalid time series response: " + err)
    }
  }
} 

package org.apache.spark.sql.execution.datasources.druid

import org.json4s.JsonAST.{JObject, JValue}
import org.json4s.JsonDSL._

trait PostAggregationFieldSpec extends Expression {
  private def arith(rhs: PostAggregationFieldSpec, fn: String): PostAggregation =
    ArithmeticPostAggregation("n/a", fn, Seq(this, rhs))

  def *(rhs: PostAggregationFieldSpec): PostAggregation = arith(rhs, "*")

  def /(rhs: PostAggregationFieldSpec): PostAggregation = arith(rhs, "/")

  def +(rhs: PostAggregationFieldSpec): PostAggregation = arith(rhs, "+")

  def -(rhs: PostAggregationFieldSpec): PostAggregation = arith(rhs, "-")
}

trait PostAggregation extends PostAggregationFieldSpec {
  def as(outputName: String): PostAggregation
}

object PostAggregation {
  def constant(value: Double): ConstantPostAggregation =
    ConstantPostAggregation("constant", value)

  case class FieldAccess(fieldName: String) extends PostAggregationFieldSpec {
    def toJson: JValue = JObject("type" -> "fieldAccess", "fieldName" -> fieldName)
  }

}

case class ConstantPostAggregation(outputName: String, value: Double) extends PostAggregation {
  def toJson: JValue = JObject("type" -> "constant", "name" -> outputName, "value" -> value)

  def as(outputName: String): PostAggregation = copy(outputName = outputName)
}

case class ArithmeticPostAggregation(
    outputName: String,
    fn: String,
    fields: Seq[PostAggregationFieldSpec])
  extends PostAggregation {
  def toJson: JValue =
    JObject(
      "type" -> "arithmetic",
      "name" -> outputName,
      "fn" -> fn,
      "fields" -> fields.map(_.toJson))

  def as(outputName: String): ArithmeticPostAggregation = copy(outputName = outputName)
} 

package org.apache.spark.sql.execution.datasources.druid

import org.json4s.JsonAST.{JArray, JNull, JObject, JValue}
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._

sealed trait QueryFilter extends Expression {
  def and(other: QueryFilter): QueryFilter = And(Seq(this, other))

  def or(other: QueryFilter): QueryFilter = Or(Seq(this, other))
}

case class And(filters: Seq[Expression]) extends QueryFilter {

  override def and(other: QueryFilter): QueryFilter = copy(other +: filters)

  def toJson: JValue = JObject("type" -> "and", "fields" -> JArray(filters.toList.map(_.toJson)))
}

case class Or(filters: Seq[Expression]) extends QueryFilter {

  override def or(other: QueryFilter): QueryFilter = copy(other +: filters)

  def toJson: JValue = JObject("type" -> "or", "fields" -> JArray(filters.toList.map(_.toJson)))
}

case class Not(filter: Expression) extends QueryFilter {

  def toJson: JValue = JObject("type" -> "not", "field" -> filter.toJson)
}

case class IsNotNull(attributeNotNull: String) extends QueryFilter {

//  {
//    "field": {
//      "type": "selector",
//      "dimension": "added",
//      "value": ""
//    },
//    "type": "not"
//  }
  def toJson: JValue =
    JObject(
      "field" -> JObject("type" -> "selector", "dimension" -> attributeNotNull, "value" -> ""),
      "type" -> "not")
}

case class ExprQueryFilter(typeName: String, dimension: String, value: String)
  extends QueryFilter {
  def toJson: JValue = JObject("type" -> typeName, "dimension" -> dimension, "value" -> value)
}

case class SelectorQueryFilter(dimension: String, value: String) extends QueryFilter {
  def toJson: JValue = JObject("type" -> "selector", "dimension" -> dimension, "value" -> value)
}

case class RegexQueryFilter(dimension: String, pattern: String) extends QueryFilter {
  def toJson: JValue = JObject("type" -> "regex", "dimension" -> dimension, "pattern" -> pattern)
}

case class AllQueryFilter(condition: java.util.HashMap[String, Any]) extends QueryFilter {
  //  val json = JSONObject.fromObject(condition.get("filter")).toString
  //  def toJson: JValue = parse(json)
  def toJson: JValue = fromJsonNode(mapper.valueToTree(condition.get("filter")))
}

object QueryFilter {

  def custom(typeName: String, dimension: String, value: String): ExprQueryFilter =
    ExprQueryFilter(typeName, dimension, value)

  def where(dimension: String, value: String): SelectorQueryFilter =
    SelectorQueryFilter(dimension, value)

  def regex(dimension: String, pattern: String): RegexQueryFilter =
    RegexQueryFilter(dimension, pattern)

  val All = new QueryFilter {
    def toJson: JValue = JNull
  }
} 

// Copyright 2017, Yahoo Holdings Inc.
// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
package com.yahoo.maha.core

import org.json4s.JsonAST.JObject
import org.json4s.scalaz.JsonScalaz._


sealed trait PartitionLevel {
  def level: Int

  def asJSON: JObject = makeObj(List("level" -> toJSON(level)))
}

case object NoPartitionLevel extends PartitionLevel {
  val level: Int = 0
}
case object FirstPartitionLevel extends PartitionLevel {
  val level: Int = 1
}
case object SecondPartitionLevel extends PartitionLevel {
  val level: Int = 2
}
case object ThirdPartitionLevel extends PartitionLevel {
  val level: Int = 3
} 

// Copyright 2017, Yahoo Holdings Inc.
// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
package com.yahoo.maha.core

import org.json4s.JsonAST.{JNull, JObject, JValue}
import org.json4s.scalaz.JsonScalaz._


sealed trait ColumnAnnotation {
  def asJSON: JObject = makeObj(
    List(
      ("annotation" -> toJSON(this.getClass.getSimpleName))
    )
  )
}

sealed trait SingletonColumn

trait ClassNameHashCode {
  override final val hashCode: Int = this.getClass.toString.hashCode
  override def equals(other: Any) : Boolean =  this.hashCode == other.hashCode()
}

sealed trait ColumnAnnotationInstance extends ColumnAnnotation with ClassNameHashCode {
  def instance: ColumnAnnotation
}

case class HiveShardingExpression(expression: HiveDerivedExpression) extends ColumnAnnotationInstance with WithHiveEngine {
  def instance: ColumnAnnotation = HiveShardingExpression.instance

  val jUtils = JsonUtils

  override def asJSON(): JObject =
    makeObj(
      List(
        ("annotation" -> toJSON(this.getClass.getSimpleName))
        ,("expression" -> jUtils.asJSON(expression))
      )
    )
}
case object HiveShardingExpression {
  val instance: ColumnAnnotation = HiveShardingExpression(null)
}

case class PrestoShardingExpression(expression: PrestoDerivedExpression) extends ColumnAnnotationInstance with WithPrestoEngine {
  def instance: ColumnAnnotation = PrestoShardingExpression.instance

  val jUtils = JsonUtils

  override def asJSON(): JObject =
    makeObj(
      List(
        ("annotation" -> toJSON(this.getClass.getSimpleName))
        ,("expression" -> jUtils.asJSON(expression))
      )
    )
}
case object PrestoShardingExpression {
  val instance: ColumnAnnotation = PrestoShardingExpression(null)
}

case object PrimaryKey extends ColumnAnnotation
case object EscapingRequired extends ColumnAnnotation
case object HiveSnapshotTimestamp extends ColumnAnnotation with SingletonColumn with WithHiveEngine
case object OracleSnapshotTimestamp extends ColumnAnnotation with SingletonColumn with WithOracleEngine
case object PostgresSnapshotTimestamp extends ColumnAnnotation with SingletonColumn with WithPostgresEngine
case object IsAggregation extends ColumnAnnotation
case object CaseInsensitive extends ColumnAnnotation
case class ForeignKey(publicDimName: String) extends ColumnAnnotationInstance {
  def instance: ColumnAnnotation = ForeignKey.instance

  override def asJSON(): JObject =
    makeObj(
      List(
        ("annotation" -> toJSON(this.getClass.getSimpleName))
        ,("publicDimName" -> toJSON(publicDimName))
      )
    )
}
case object ForeignKey {
  val instance: ColumnAnnotation = ForeignKey("instance")
}
case class DayColumn(fmt: String) extends ColumnAnnotationInstance {
  def instance: ColumnAnnotation = DayColumn.instance

  override def asJSON(): JObject =
    makeObj(
      List(
        ("annotation" -> toJSON(this.getClass.getSimpleName))
        ,("fmt" -> toJSON(fmt))
      )
    )
}
case object DayColumn {
  val instance: ColumnAnnotation = DayColumn("instance")
} 

// Copyright 2017, Yahoo Holdings Inc.
// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
package com.yahoo.maha.core.fact

import com.yahoo.maha.core._
import org.json4s.JsonAST.{JArray, JObject}
import org.json4s.scalaz.JsonScalaz._


case class HiveCustomRollup(expression: HiveDerivedExpression) extends CustomRollup with WithHiveEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = expression.sourceColumns
  override lazy val sourcePrimitiveColumns: Set[String] = expression.sourcePrimitiveColumns
}
case class PrestoCustomRollup(expression: PrestoDerivedExpression) extends CustomRollup with WithPrestoEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = expression.sourceColumns
  override lazy val sourcePrimitiveColumns: Set[String] = expression.sourcePrimitiveColumns
}
case class OracleCustomRollup(expression: OracleDerivedExpression) extends CustomRollup with WithOracleEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = expression.sourceColumns
  override lazy val sourcePrimitiveColumns: Set[String] = expression.sourcePrimitiveColumns
}
case class PostgresCustomRollup(expression: PostgresDerivedExpression) extends CustomRollup with WithPostgresEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = expression.sourceColumns
  override lazy val sourcePrimitiveColumns: Set[String] = expression.sourcePrimitiveColumns
}
case class DruidCustomRollup(expression: DruidDerivedExpression) extends CustomRollup with WithDruidEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = expression.sourceColumns
  override lazy val sourcePrimitiveColumns: Set[String] = expression.sourcePrimitiveColumns
}
case class DruidFilteredRollup(filter: Filter, factCol: DruidExpression.FieldAccess,
                               delegateAggregatorRollupExpression: RollupExpression) extends CustomRollup with WithDruidEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = Set(filter.field, factCol.name) ++ delegateAggregatorRollupExpression.sourceColumns
}
case class DruidFilteredListRollup(filter: List[Filter], factCol: DruidExpression.FieldAccess,
                               delegateAggregatorRollupExpression: RollupExpression) extends CustomRollup with WithDruidEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = filter.map(fil => fil.field).toSet ++ delegateAggregatorRollupExpression.sourceColumns ++ Set(factCol.name)
}

case class DruidHyperUniqueRollup(column: String) extends CustomRollup with WithDruidEngine {
  override val hasDerivedExpression: Boolean = true
  override lazy val sourceColumns: Set[String] = Set(column)
}
case object DruidThetaSketchRollup extends CustomRollup with WithDruidEngine 

package org.apress.prospark

import java.nio.charset.StandardCharsets

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.eclipse.paho.client.mqttv3.MqttClient
import org.eclipse.paho.client.mqttv3.MqttMessage
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence
import org.json4s.DefaultFormats
import org.json4s.JField
import org.json4s.JsonAST.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.parse
import org.json4s.string2JsonInput

object MqttSinkAppB {

  def main(args: Array[String]) {
    if (args.length != 3) {
      System.err.println(
        "Usage: MqttSinkApp <appname> <outputBrokerUrl> <topic>")
      System.exit(1)
    }

    val Seq(appName, outputBrokerUrl, topic) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val batchInterval = 10

    val ssc = new StreamingContext(conf, Seconds(batchInterval))

    HttpUtils.createStream(ssc, url = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22IBM,GOOG,MSFT,AAPL,FB,ORCL,YHOO,TWTR,LNKD,INTC%22)%0A%09%09&format=json&diagnostics=true&env=http%3A%2F%2Fdatatables.org%2Falltables.env",
      interval = batchInterval)
      .flatMap(rec => {
        val query = parse(rec) \ "query"
        ((query \ "results" \ "quote").children).map(rec => JObject(JField("Timestamp", query \ "created")).merge(rec))
      })
      .map(rec => {
        implicit val formats = DefaultFormats
        rec.children.map(f => f.extract[String]) mkString ","
      })
      .foreachRDD { rdd =>
        rdd.foreach { rec =>
          {
            val client = new MqttClient(outputBrokerUrl, MqttClient.generateClientId(), new MemoryPersistence())
            client.connect()
            client.publish(topic, new MqttMessage(rec.getBytes(StandardCharsets.UTF_8)))
            client.disconnect()
            client.close()
          }
        }
      }

    ssc.start()
    ssc.awaitTermination()
  }

} 

package org.apache.spark.deploy

import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._

import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner

private[deploy] object JsonProtocol {
 def writeWorkerInfo(obj: WorkerInfo): JObject = {
   ("id" -> obj.id) ~
   ("host" -> obj.host) ~
   ("port" -> obj.port) ~
   ("webuiaddress" -> obj.webUiAddress) ~
   ("cores" -> obj.cores) ~
   ("coresused" -> obj.coresUsed) ~
   ("coresfree" -> obj.coresFree) ~
   ("memory" -> obj.memory) ~
   ("memoryused" -> obj.memoryUsed) ~
   ("memoryfree" -> obj.memoryFree) ~
   ("state" -> obj.state.toString) ~
   ("lastheartbeat" -> obj.lastHeartbeat)
 }

  def writeApplicationInfo(obj: ApplicationInfo): JObject = {
    ("starttime" -> obj.startTime) ~
    ("id" -> obj.id) ~
    ("name" -> obj.desc.name) ~
    ("cores" -> obj.desc.maxCores) ~
    ("user" -> obj.desc.user) ~
    ("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
    ("submitdate" -> obj.submitDate.toString) ~
    ("state" -> obj.state.toString) ~
    ("duration" -> obj.duration)
  }

  def writeApplicationDescription(obj: ApplicationDescription): JObject = {
    ("name" -> obj.name) ~
    ("cores" -> obj.maxCores) ~
    ("memoryperslave" -> obj.memoryPerExecutorMB) ~
    ("user" -> obj.user) ~
    ("command" -> obj.command.toString)
  }

  def writeExecutorRunner(obj: ExecutorRunner): JObject = {
    ("id" -> obj.execId) ~
    ("memory" -> obj.memory) ~
    ("appid" -> obj.appId) ~
    ("appdesc" -> writeApplicationDescription(obj.appDesc))
  }

  def writeDriverInfo(obj: DriverInfo): JObject = {
    ("id" -> obj.id) ~
    ("starttime" -> obj.startTime.toString) ~
    ("state" -> obj.state.toString) ~
    ("cores" -> obj.desc.cores) ~
    ("memory" -> obj.desc.mem)
  }

  def writeMasterState(obj: MasterStateResponse): JObject = {
    val aliveWorkers = obj.workers.filter(_.isAlive())
    ("url" -> obj.uri) ~
    ("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
    ("cores" -> aliveWorkers.map(_.cores).sum) ~
    ("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~
    ("memory" -> aliveWorkers.map(_.memory).sum) ~
    ("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~
    ("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
    ("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
    ("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
    ("status" -> obj.status.toString)
  }

  def writeWorkerState(obj: WorkerStateResponse): JObject = {
    ("id" -> obj.workerId) ~
    ("masterurl" -> obj.masterUrl) ~
    ("masterwebuiurl" -> obj.masterWebUiUrl) ~
    ("cores" -> obj.cores) ~
    ("coresused" -> obj.coresUsed) ~
    ("memory" -> obj.memory) ~
    ("memoryused" -> obj.memoryUsed) ~
    ("executors" -> obj.executors.toList.map(writeExecutorRunner)) ~
    ("finishedexecutors" -> obj.finishedExecutors.toList.map(writeExecutorRunner))
  }
} 

package org.apress.prospark

import java.nio.charset.StandardCharsets

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.eclipse.paho.client.mqttv3.MqttClient
import org.eclipse.paho.client.mqttv3.MqttMessage
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence
import org.json4s.DefaultFormats
import org.json4s.JField
import org.json4s.JsonAST.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.parse
import org.json4s.string2JsonInput

object MqttSinkAppD {

  def main(args: Array[String]) {
    if (args.length != 3) {
      System.err.println(
        "Usage: MqttSinkApp <appname> <outputBrokerUrl> <topic>")
      System.exit(1)
    }

    val Seq(appName, outputBrokerUrl, topic) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val batchInterval = 10

    val ssc = new StreamingContext(conf, Seconds(batchInterval))

    HttpUtils.createStream(ssc, url = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22IBM,GOOG,MSFT,AAPL,FB,ORCL,YHOO,TWTR,LNKD,INTC%22)%0A%09%09&format=json&diagnostics=true&env=http%3A%2F%2Fdatatables.org%2Falltables.env",
      interval = batchInterval)
      .flatMap(rec => {
        val query = parse(rec) \ "query"
        ((query \ "results" \ "quote").children).map(rec => JObject(JField("Timestamp", query \ "created")).merge(rec))
      })
      .map(rec => {
        implicit val formats = DefaultFormats
        rec.children.map(f => f.extract[String]) mkString ","
      })
      .foreachRDD { rdd =>
        rdd.foreachPartition { par =>
          par.foreach(message => MqttSink().publish(topic, new MqttMessage(message.getBytes(StandardCharsets.UTF_8))))
        }
      }

    ssc.start()
    ssc.awaitTermination()
  }
}

object MqttSink {
  val brokerUrl = "tcp://localhost:1883"
  val client = new MqttClient(brokerUrl, MqttClient.generateClientId(), new MemoryPersistence())
  client.connect()
  sys.addShutdownHook {
    client.disconnect()
    client.close()
  }

  def apply(): MqttClient = {
    client
  }
} 

package org.apress.prospark

import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.json4s.DefaultFormats
import org.json4s.JField
import org.json4s.JsonAST.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.parse
import org.json4s.string2JsonInput
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.Text
import java.nio.ByteBuffer
import org.apache.cassandra.hadoop.ColumnFamilyOutputFormat
import org.apache.cassandra.hadoop.ConfigHelper
import org.apache.cassandra.thrift.ColumnOrSuperColumn
import org.apache.cassandra.thrift.Column
import org.apache.cassandra.utils.ByteBufferUtil
import org.apache.cassandra.thrift.Mutation
import java.util.Arrays

object CassandraSinkApp {

  def main(args: Array[String]) {
    if (args.length != 6) {
      System.err.println(
        "Usage: CassandraSinkApp <appname> <cassandraHost> <cassandraPort> <keyspace> <columnFamilyName> <columnName>")
      System.exit(1)
    }

    val Seq(appName, cassandraHost, cassandraPort, keyspace, columnFamilyName, columnName) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val batchInterval = 10
    val windowSize = 20
    val slideInterval = 10

    val ssc = new StreamingContext(conf, Seconds(batchInterval))

    HttpUtils.createStream(ssc, url = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22IBM,GOOG,MSFT,AAPL,FB,ORCL,YHOO,TWTR,LNKD,INTC%22)%0A%09%09&format=json&diagnostics=true&env=http%3A%2F%2Fdatatables.org%2Falltables.env",
      interval = batchInterval)
      .flatMap(rec => {
        implicit val formats = DefaultFormats
        val query = parse(rec) \ "query"
        ((query \ "results" \ "quote").children)
          .map(rec => ((rec \ "symbol").extract[String], (rec \ "LastTradePriceOnly").extract[String].toFloat))
      })
      .reduceByKeyAndWindow((x: Float, y: Float) => (x + y), Seconds(windowSize), Seconds(slideInterval))
      .foreachRDD(rdd => {
        val jobConf = new Configuration()
        ConfigHelper.setOutputRpcPort(jobConf, cassandraPort)
        ConfigHelper.setOutputInitialAddress(jobConf, cassandraHost)
        ConfigHelper.setOutputColumnFamily(jobConf, keyspace, columnFamilyName)
        ConfigHelper.setOutputPartitioner(jobConf, "Murmur3Partitioner")
        rdd.map(rec => {
          val c = new Column()
          c.setName(ByteBufferUtil.bytes(columnName))
          c.setValue(ByteBufferUtil.bytes(rec._2 / (windowSize / batchInterval)))
          c.setTimestamp(System.currentTimeMillis)
          val m = new Mutation()
          m.setColumn_or_supercolumn(new ColumnOrSuperColumn())
          m.column_or_supercolumn.setColumn(c)
          (ByteBufferUtil.bytes(rec._1), Arrays.asList(m))
        }).saveAsNewAPIHadoopFile(keyspace, classOf[ByteBuffer], classOf[List[Mutation]], classOf[ColumnFamilyOutputFormat], jobConf)
      })

    ssc.start()
    ssc.awaitTermination()
  }
} 

package org.apress.prospark

import java.nio.charset.StandardCharsets

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.eclipse.paho.client.mqttv3.MqttClient
import org.eclipse.paho.client.mqttv3.MqttMessage
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence
import org.json4s.DefaultFormats
import org.json4s.JField
import org.json4s.JsonAST.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.parse
import org.json4s.string2JsonInput

object MqttSinkAppA {

  def main(args: Array[String]) {
    if (args.length != 3) {
      System.err.println(
        "Usage: MqttSinkApp <appname> <outputBrokerUrl> <topic>")
      System.exit(1)
    }

    val Seq(appName, outputBrokerUrl, topic) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val batchInterval = 10

    val ssc = new StreamingContext(conf, Seconds(batchInterval))

    HttpUtils.createStream(ssc, url = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22IBM,GOOG,MSFT,AAPL,FB,ORCL,YHOO,TWTR,LNKD,INTC%22)%0A%09%09&format=json&diagnostics=true&env=http%3A%2F%2Fdatatables.org%2Falltables.env",
      interval = batchInterval)
      .flatMap(rec => {
        val query = parse(rec) \ "query"
        ((query \ "results" \ "quote").children).map(rec => JObject(JField("Timestamp", query \ "created")).merge(rec))
      })
      .map(rec => {
        implicit val formats = DefaultFormats
        rec.children.map(f => f.extract[String]) mkString ","
      })
      .foreachRDD { rdd =>
        val client = new MqttClient(outputBrokerUrl, MqttClient.generateClientId(), new MemoryPersistence())
        client.connect()
        rdd.foreach(rec => client.publish(topic, new MqttMessage(rec.getBytes(StandardCharsets.UTF_8))))
        client.disconnect()
        client.close()
      }

    ssc.start()
    ssc.awaitTermination()
  }

} 

package org.apress.prospark

import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.eclipse.paho.client.mqttv3.MqttClient
import org.eclipse.paho.client.mqttv3.MqttMessage
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence
import org.json4s.DefaultFormats
import org.json4s.JField
import org.json4s.JsonAST.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.parse
import org.json4s.string2JsonInput
import org.apache.commons.pool2.PooledObject
import org.apache.commons.pool2.BasePooledObjectFactory
import org.apache.commons.pool2.impl.DefaultPooledObject
import org.apache.commons.pool2.impl.GenericObjectPool
import org.apache.commons.pool2.ObjectPool

object MqttSinkAppE {

  def main(args: Array[String]) {
    if (args.length != 3) {
      System.err.println(
        "Usage: MqttSinkApp <appname> <outputBrokerUrl> <topic>")
      System.exit(1)
    }

    val Seq(appName, outputBrokerUrl, topic) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val batchInterval = 10

    val ssc = new StreamingContext(conf, Seconds(batchInterval))

    HttpUtils.createStream(ssc, url = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22IBM,GOOG,MSFT,AAPL,FB,ORCL,YHOO,TWTR,LNKD,INTC%22)%0A%09%09&format=json&diagnostics=true&env=http%3A%2F%2Fdatatables.org%2Falltables.env",
      interval = batchInterval)
      .flatMap(rec => {
        val query = parse(rec) \ "query"
        ((query \ "results" \ "quote").children).map(rec => JObject(JField("Timestamp", query \ "created")).merge(rec))
      })
      .map(rec => {
        implicit val formats = DefaultFormats
        rec.children.map(f => f.extract[String]) mkString ","
      })
      .foreachRDD { rdd =>
        rdd.foreachPartition { par =>
          val mqttSink = MqttSinkPool().borrowObject()
          par.foreach(message => mqttSink.publish(topic, new MqttMessage(message.getBytes(StandardCharsets.UTF_8))))
          MqttSinkPool().returnObject(mqttSink)
        }
      }

    ssc.start()
    ssc.awaitTermination()
  }
}

object MqttSinkPool {
  val poolSize = 8
  val brokerUrl = "tcp://localhost:1883"
  val mqttPool = new GenericObjectPool[MqttClient](new MqttClientFactory(brokerUrl))
  mqttPool.setMaxTotal(poolSize)
  sys.addShutdownHook {
    mqttPool.close()
  }
  
  def apply(): GenericObjectPool[MqttClient] = {
    mqttPool
  }
}

class MqttClientFactory(brokerUrl: String) extends BasePooledObjectFactory[MqttClient] {
  override def create() = {
    val client = new MqttClient(brokerUrl, MqttClient.generateClientId(), new MemoryPersistence())
    client.connect()
    client
  }
  override def wrap(client: MqttClient) = new DefaultPooledObject[MqttClient](client)
  override def validateObject(pObj: PooledObject[MqttClient]) = pObj.getObject.isConnected()
  override def destroyObject(pObj: PooledObject[MqttClient]) = {
    pObj.getObject.disconnect()
    pObj.getObject.close()
  }
  override def passivateObject(pObj: PooledObject[MqttClient]) = {}
} 

package org.apress.prospark

import java.nio.charset.StandardCharsets

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.eclipse.paho.client.mqttv3.MqttClient
import org.eclipse.paho.client.mqttv3.MqttMessage
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence
import org.json4s.DefaultFormats
import org.json4s.JField
import org.json4s.JsonAST.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.parse
import org.json4s.string2JsonInput

object MqttSinkAppC {

  def main(args: Array[String]) {
    if (args.length != 3) {
      System.err.println(
        "Usage: MqttSinkApp <appname> <outputBrokerUrl> <topic>")
      System.exit(1)
    }

    val Seq(appName, outputBrokerUrl, topic) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val batchInterval = 10

    val ssc = new StreamingContext(conf, Seconds(batchInterval))

    HttpUtils.createStream(ssc, url = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22IBM,GOOG,MSFT,AAPL,FB,ORCL,YHOO,TWTR,LNKD,INTC%22)%0A%09%09&format=json&diagnostics=true&env=http%3A%2F%2Fdatatables.org%2Falltables.env",
      interval = batchInterval)
      .flatMap(rec => {
        val query = parse(rec) \ "query"
        ((query \ "results" \ "quote").children).map(rec => JObject(JField("Timestamp", query \ "created")).merge(rec))
      })
      .map(rec => {
        implicit val formats = DefaultFormats
        rec.children.map(f => f.extract[String]) mkString ","
      })
      .foreachRDD { rdd =>
        rdd.foreachPartition { par =>
          val client = new MqttClient(outputBrokerUrl, MqttClient.generateClientId(), new MemoryPersistence())
          client.connect()
          par.foreach(rec => client.publish(topic, new MqttMessage(rec.getBytes(StandardCharsets.UTF_8))))
          client.disconnect()
          client.close()
        }
      }

    ssc.start()
    ssc.awaitTermination()
  }
} 

package com.krux.hyperion.contrib.activity.notification

import java.net.{ HttpURLConnection, URL }

import org.json4s.JsonAST.{ JString, JObject }
import org.json4s.jackson.JsonMethods._
import scopt.OptionParser

object SendSlackMessage {
  case class Options(
    failOnError: Boolean = false,
    webhookUrl: String = "",
    user: Option[String] = None,
    message: Seq[String] = Seq.empty,
    iconEmoji: Option[String] = None,
    channel: Option[String] = None
  )

  def apply(options: Options): Boolean = try {
    // Setup the connection
    val connection = new URL(options.webhookUrl).openConnection().asInstanceOf[HttpURLConnection]
    connection.setDoOutput(true)
    connection.setRequestProperty("Content-Type", "application/json")
    connection.setRequestProperty("Accept", "application/json")

    // Write the message
    val output = connection.getOutputStream
    try {
      val message = Seq(
        "icon_emoji" -> options.iconEmoji,
        "channel" -> options.channel,
        "username" -> options.user,
        "text" -> Option(options.message.mkString("\n"))
      ).flatMap {
        case (k, None) => None
        case (k, Some(v)) => Option(k -> JString(v))
      }

      output.write(compact(render(JObject(message: _*))).getBytes)
    } finally {
      output.close()
    }

    // Check the response code
    connection.getResponseCode == 200 || !options.failOnError
  } catch {
    case e: Throwable =>
      System.err.println(e.toString)
      !options.failOnError
  }

  def main(args: Array[String]): Unit = {
    val parser = new OptionParser[Options](s"hyperion-notification-slack-activity") {
      override def showUsageOnError = true

      note("Sends a notification message to a Slack incoming webhook.")
      help("help").text("prints this usage text")
      opt[Unit]("fail-on-error").optional().action((_, c) => c.copy(failOnError = true))
        .text("Causes the activity to fail if any error received from the webhook")
      opt[String]("webhook-url").valueName("WEBHOOK").required().action((x, c) => c.copy(webhookUrl = x))
        .text("Sends the message to the given WEBHOOK url")
      opt[String]("user").valueName("NAME").optional().action((x, c) => c.copy(user = Option(x)))
        .text("Sends the message as the user with NAME")
      opt[String]("emoji").valueName("EMOJI").optional().action((x, c) => c.copy(iconEmoji = Option(x)))
        .text("Use EMOJI for the icon")
      opt[String]("to").valueName("CHANNEL or USERNAME").optional().action((x, c) => c.copy(channel = Option(x)))
        .text("Sends the message to #CHANNEL or @USERNAME")
      arg[String]("MESSAGE").required().unbounded().action((x, c) => c.copy(message = c.message :+ x))
        .text("Sends the given MESSAGE")
    }

    if (!parser.parse(args, Options()).exists(apply)) {
      System.exit(3)
    }
  }
} 

package stellar.sdk.model

import org.json4s.JsonAST.JObject
import org.json4s.native.JsonMethods._
import org.json4s.{DefaultFormats, JValue}
import stellar.sdk.KeyPair
import stellar.sdk.model.response.ResponseParser

case class OrderBook(selling: Asset, buying: Asset, bids: Seq[Order], asks: Seq[Order])

case class Order(price: Price, quantity: Long)

object OrderBookDeserializer extends ResponseParser[OrderBook]({ o: JObject =>
  implicit val formats = DefaultFormats

  def asset(obj: JValue) = {
    def assetCode = (obj \ s"asset_code").extract[String]

    def assetIssuer = KeyPair.fromAccountId((obj \ s"asset_issuer").extract[String])

    (obj \ s"asset_type").extract[String] match {
      case "native" => NativeAsset
      case "credit_alphanum4" => IssuedAsset4(assetCode, assetIssuer)
      case "credit_alphanum12" => IssuedAsset12(assetCode, assetIssuer)
      case t => throw new RuntimeException(s"Unrecognised asset type '$t'")
    }
  }

  def orders(obj: JValue) = {
    obj.children.map(c =>
      Order(
        price = Price(
          n = (c \ "price_r" \ "n").extract[Int],
          d = (c \ "price_r" \ "d").extract[Int]
        ),
        quantity = Amount.toBaseUnits((c \ "amount").extract[String]).get
      ))
  }

  try {
    OrderBook(
      selling = asset(o \ "base"),
      buying = asset(o \ "counter"),
      bids = orders(o \ "bids"),
      asks = orders(o \ "asks")
    )
  } catch {
    case t: Throwable => throw new RuntimeException(pretty(render(o)), t)
  }
}) 

package stellar.sdk.model.op

import java.time.ZonedDateTime

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import stellar.sdk.model.response.ResponseParser


case class Transacted[+O <: Operation](id: Long,
                                      txnHash: String,
                                      createdAt: ZonedDateTime,
                                      operation: O)

object TransactedOperationDeserializer extends ResponseParser[Transacted[Operation]]({ o: JObject =>
  implicit val formats = DefaultFormats + OperationDeserializer

  def date(key: String) = ZonedDateTime.parse((o \ key).extract[String])

  Transacted(
    id = (o \ "id").extract[String].toLong,
    txnHash = (o \ "transaction_hash").extract[String],
    createdAt = date("created_at"),
    operation = o.extract[Operation])
}) 

package stellar.sdk.model.response

import java.time.ZonedDateTime

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import stellar.sdk.model.{Amount, NativeAmount}

case class LedgerResponse(id: String, hash: String, previousHash: Option[String], sequence: Long, successTransactionCount: Int,
                          failureTransactionCount: Int, operationCount: Int, closedAt: ZonedDateTime,
                          totalCoins: NativeAmount, feePool: NativeAmount, baseFee: NativeAmount, baseReserve: NativeAmount,
                          maxTxSetSize: Int) {

  def transactionCount: Int = successTransactionCount + failureTransactionCount

}

object LedgerRespDeserializer extends ResponseParser[LedgerResponse]({ o: JObject =>
  implicit val formats = DefaultFormats

  LedgerResponse(
    id = (o \ "id").extract[String],
    hash = (o \ "hash").extract[String],
    previousHash = (o \ "prev_hash").extractOpt[String],
    sequence = (o \ "sequence").extract[Long],
    successTransactionCount = (o \ "successful_transaction_count").extract[Int],
    failureTransactionCount = (o \ "failed_transaction_count").extract[Int],
    operationCount = (o \ "operation_count").extract[Int],
    closedAt = ZonedDateTime.parse((o \ "closed_at").extract[String]),
    totalCoins = Amount.toBaseUnits((o \ "total_coins").extract[String]).map(NativeAmount.apply).get,
    feePool = Amount.toBaseUnits((o \ "fee_pool").extract[String]).map(NativeAmount.apply).get,
    baseFee = NativeAmount((o \ "base_fee").extractOpt[Long].getOrElse((o \ "base_fee_in_stroops").extract[Long])),
    baseReserve = {
      val old: Option[Long] = (o \ "base_reserve").extractOpt[String].map(_.toDouble).map(Amount.toBaseUnits).map(_.get)
      NativeAmount(old.getOrElse((o \ "base_reserve_in_stroops").extract[Long]))
    },
    maxTxSetSize = (o \ "max_tx_set_size").extract[Int]
  )
}) 

package stellar.sdk.model.response

import java.time.ZonedDateTime

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import stellar.sdk._
import stellar.sdk.model._

case class OfferResponse(id: Long, seller: PublicKeyOps, selling: Amount, buying: Asset, price: Price,
                         lastModifiedLedger: Long, lastModifiedTime: ZonedDateTime) {

  override def toString = {
    s"Offer $id: ${seller.accountId} selling $selling, buying $buying @ rate $price"
  }
}

object OfferRespDeserializer extends ResponseParser[OfferResponse]({ o: JObject =>
  implicit val formats = DefaultFormats
  val id = (o \ "id").extract[String].toLong

  def account(accountKey: String = "account") = KeyPair.fromAccountId((o \ accountKey).extract[String])

  def asset(prefix: String = "", issuerKey: String = "asset_issuer") = {
    def assetCode = (o \ prefix \ "asset_code").extract[String]

    def assetIssuer = KeyPair.fromAccountId((o \ prefix \ issuerKey).extract[String])

    (o \ prefix \ "asset_type").extract[String] match {
      case "native" => NativeAsset
      case "credit_alphanum4" => IssuedAsset4(assetCode, assetIssuer)
      case "credit_alphanum12" => IssuedAsset12(assetCode, assetIssuer)
      case t => throw new RuntimeException(s"Unrecognised asset type '$t'")
    }
  }

  def doubleFromString(key: String) = (o \ key).extract[String].toDouble

  def amount(prefix: String = "") = {
    val units = Amount.toBaseUnits(doubleFromString("amount")).get
    asset(prefix) match {
      case nna: NonNativeAsset => IssuedAmount(units, nna)
      case NativeAsset => NativeAmount(units)
    }
  }

  def price = {
    val priceObj = o \ "price_r"
    Price(
      (priceObj \ "n").extract[Int],
      (priceObj \ "d").extract[Int]
    )
  }

  def lastModifiedLedger = (o \ "last_modified_ledger").extract[Long]

  def lastModifiedTime = ZonedDateTime.parse((o \ "last_modified_time").extract[String])

  OfferResponse(id, account("seller"), amount("selling"), asset("buying"), price, lastModifiedLedger, lastModifiedTime)
}) 

package stellar.sdk.model.response

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import stellar.sdk.model._
import stellar.sdk.util.ByteArrays.{hexToBytes, trimmedByteArray}
import stellar.sdk.{KeyPair, PublicKey}

case class FederationResponse(address: String,
                              account: PublicKey,
                              memo: Memo = NoMemo)

object FederationResponseDeserialiser extends ResponseParser[FederationResponse]({ o: JObject =>
  implicit val formats = DefaultFormats

//  println(JsonMethods.pretty(JsonMethods.render(o)))

  FederationResponse(
    // reference server erroneously fails to set `stellar_address` for forward lookups
    address = (o \ "stellar_address").extractOpt[String].orNull,
    // reference server erroneously fails to set `account_id` for reverse lookups
    account = (o \ "account_id").extractOpt[String].map(KeyPair.fromAccountId).orNull,
    memo = (o \ "memo_type").extractOpt[String] match {
      case Some("id") => MemoId((o \ "memo").extract[String].toLong)
      case Some("text") => MemoText((o \ "memo").extract[String])
      case Some("hash") => MemoHash(trimmedByteArray(hexToBytes((o \ "memo").extract[String])))
      case _ => NoMemo
    }
  )
}) 

package stellar.sdk.model.response

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject
import stellar.sdk.Network
import stellar.sdk.model.ledger.{LedgerEntryChange, LedgerEntryChanges, TransactionLedgerEntries}
import stellar.sdk.model.result._
import stellar.sdk.model.{NativeAmount, SignedTransaction}

sealed abstract class TransactionPostResponse(envelopeXDR: String, resultXDR: String) {
  val isSuccess: Boolean

  def transaction(implicit network: Network): SignedTransaction = SignedTransaction.decodeXDR(envelopeXDR)

  def feeCharged: NativeAmount

  
case class TransactionRejected(status: Int, detail: String,
                               resultCode: String, opResultCodes: Seq[String],
                               envelopeXDR: String, resultXDR: String)
  extends TransactionPostResponse(envelopeXDR, resultXDR) {

  override val isSuccess: Boolean = false

  // -- unroll nested XDR deserialised object into this object for convenience
  lazy val result: TransactionNotSuccessful = TransactionResult.decodeXDR(resultXDR).asInstanceOf[TransactionNotSuccessful]

  def feeCharged: NativeAmount = result.feeCharged
}



object TransactionPostResponseDeserializer extends ResponseParser[TransactionPostResponse]({
  o: JObject =>
    implicit val formats = DefaultFormats

    (o \ "type").extractOpt[String] match {

      case Some("https://stellar.org/horizon-errors/transaction_failed") =>
        TransactionRejected(
          status = (o \ "status").extract[Int],
          detail = (o \ "detail").extract[String],
          resultCode = (o \ "extras" \ "result_codes" \ "transaction").extract[String],
          opResultCodes = (o \ "extras" \ "result_codes" \ "operations").extract[Seq[String]],
          resultXDR = (o \ "extras" \ "result_xdr").extract[String],
          envelopeXDR = (o \ "extras" \ "envelope_xdr").extract[String]
        )

      case _ =>
        TransactionApproved(
          hash = (o \ "hash").extract[String],
          ledger = (o \ "ledger").extract[Long],
          envelopeXDR = (o \ "envelope_xdr").extract[String],
          resultXDR = (o \ "result_xdr").extract[String],
          resultMetaXDR = (o \ "result_meta_xdr").extract[String]
        )
    }
}) 

package stellar.sdk.model.response

import java.nio.charset.StandardCharsets.UTF_8

import org.json4s.{DefaultFormats, Formats}
import org.json4s.JsonAST.{JArray, JObject}
import stellar.sdk._
import stellar.sdk.model.Amount.toBaseUnits
import stellar.sdk.model._
import stellar.sdk.util.ByteArrays

case class AccountResponse(id: PublicKey,
                           lastSequence: Long,
                           subEntryCount: Int,
                           thresholds: Thresholds,
                           authRequired: Boolean,
                           authRevocable: Boolean,
                           balances: List[Balance],
                           signers: List[Signer],
                           data: Map[String, Array[Byte]]) {

  def toAccount: Account = Account(AccountId(id.publicKey), lastSequence + 1)

  def decodedData: Map[String, String] = data.map { case (k, v) => k -> new String(v, UTF_8) }
}

object AccountRespDeserializer extends ResponseParser[AccountResponse]({ o: JObject =>
  implicit val formats: Formats = DefaultFormats
  val id = KeyPair.fromAccountId((o \ "id").extract[String])
  val seq = (o \ "sequence").extract[String].toLong
  val subEntryCount = (o \ "subentry_count").extract[Int]
  val lowThreshold = (o \ "thresholds" \ "low_threshold").extract[Int]
  val mediumThreshold = (o \ "thresholds" \ "med_threshold").extract[Int]
  val highThreshold = (o \ "thresholds" \ "high_threshold").extract[Int]
  val authRequired = (o \ "flags" \ "auth_required").extract[Boolean]
  val authRevocable = (o \ "flags" \ "auth_revocable").extract[Boolean]
  val JArray(jsBalances) = o \ "balances"
  val balances = jsBalances.map {
    case balObj: JObject =>
      val units = toBaseUnits((balObj \ "balance").extract[String].toDouble).get
      val amount = (balObj \ "asset_type").extract[String] match {
        case "credit_alphanum4" =>
          Amount(units, IssuedAsset4(
            code = (balObj \ "asset_code").extract[String],
            issuer = KeyPair.fromAccountId((balObj \ "asset_issuer").extract[String])
          ))
        case "credit_alphanum12" =>
          Amount(units, IssuedAsset12(
            code = (balObj \ "asset_code").extract[String],
            issuer = KeyPair.fromAccountId((balObj \ "asset_issuer").extract[String])
          ))
        case "native" => NativeAmount(units)
        case t => throw new RuntimeException(s"Unrecognised asset type: $t")
      }
      val limit = (balObj \ "limit").extractOpt[String].map(BigDecimal(_)).map(toBaseUnits).map(_.get)
      val buyingLiabilities = toBaseUnits(BigDecimal((balObj \ "buying_liabilities").extract[String])).get
      val sellingLiabilities = toBaseUnits(BigDecimal((balObj \ "selling_liabilities").extract[String])).get
      val authorised = (balObj \ "is_authorized").extractOpt[Boolean].getOrElse(false)
      val authorisedToMaintainLiabilities = (balObj \ "is_authorized_to_maintain_liabilities")
        .extractOpt[Boolean].getOrElse(false)

      Balance(amount, limit, buyingLiabilities, sellingLiabilities, authorised, authorisedToMaintainLiabilities)
    case _ => throw new RuntimeException(s"Expected js object at 'balances'")
  }
  val JArray(jsSigners) = o \ "signers"
  val signers = jsSigners.map {
    case signerObj: JObject =>
      val key = StrKey.decodeFromString((signerObj \ "key").extract[String]).asInstanceOf[SignerStrKey]
      val weight = (signerObj \ "weight").extract[Int]
      Signer(key, weight)
    case _ => throw new RuntimeException(s"Expected js object at 'signers'")
  }
  val JObject(dataFields) = o \ "data"
  val data = dataFields.map{ case (k, v) => k -> ByteArrays.base64(v.extract[String]) }.toMap

  AccountResponse(id, seq, subEntryCount, Thresholds(lowThreshold, mediumThreshold, highThreshold),
    authRequired, authRevocable, balances, signers, data)

}) 

package stellar.sdk.model.response

import org.json4s.DefaultFormats
import org.json4s.JsonAST.JObject


case class NetworkInfo(horizonVersion: String,
                       coreVersion: String,
                       earliestLedger: Long,
                       latestLedger: Long,
                       passphrase: String,
                       currentProtocolVersion: Int,
                       supportedProtocolVersion: Int)

object NetworkInfoDeserializer extends ResponseParser[NetworkInfo]({ o: JObject =>
  implicit val formats = DefaultFormats

  NetworkInfo(
    horizonVersion = (o \ "horizon_version").extract[String],
    coreVersion = (o \ "core_version").extract[String],
    earliestLedger = (o \ "history_elder_ledger").extract[Long],
    latestLedger = (o \ "history_latest_ledger").extract[Long],
    passphrase = (o \ "network_passphrase").extract[String],
    currentProtocolVersion = (o \ "current_protocol_version").extract[Int],
    supportedProtocolVersion = (o \ "core_supported_protocol_version").extract[Int]
  )
})