org.json4s.JObject Scala Examples

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.DefaultFormats
import org.json4s.JArray
import org.json4s.JNothing
import org.json4s.JObject
import org.json4s.JString
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonExtractions extends App {

  class Filter() {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(inputFile: File, jValue: JValue): Unit = {
      println(s"Extracting from ${inputFile.getName}")
      val extractions: JValue = (jValue \\ "extractions")

      extractions match {
        case JArray(extractions: List[_]) => // Type erasure removes the [JObject]
          extractions.foreach { extraction =>
            val jString = (extraction \ "text")
            val text = jString.extract[String]
            val oneLiner = text
                .replace("\n", "\\n")
                .replace("\t", "\\t")

            println("\t" + oneLiner)
          }
        case JObject(_) =>
        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val inputFiles = FileUtils.findFiles(inputDir, extension)
  val filter = new Filter()

  inputFiles.foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)
    filter.filter(inputFile, json)
  }
} 

package io.coral.cluster

import org.json4s.JObject
import org.json4s.JsonAST.{JNull, JNothing}
import org.json4s.JsonDSL._


case class Machine(// The friendly name of the machine, if any
				   alias: Option[String],
				   // The IP address of the machine
				   ip: String,
				   // The port on which the machine can be reached
				   port: Int,
				   // The role of the machine
				   roles: List[String],
				   // The status of the machine
				   status: Option[String]) {
	def toJson(): JObject = {
		("alias" -> alias.orNull) ~
		("ip" -> ip) ~
		("port" -> port) ~
		("roles" -> roles) ~
		("status" -> status)
	}
} 

package io.coral.lib

import org.json4s.{JArray, JValue, JObject}
import org.json4s.JsonAST.JNothing
import scala.util.parsing.combinator.{PackratParsers, JavaTokenParsers}
import scala.util.parsing.input.CharSequenceReader

object JsonExpressionParser extends JavaTokenParsers with PackratParsers {
	abstract class FieldElement
	// Represents the complete list of identifiers ("field.array[0].reference['elem']")
	// A FieldReference is a concatenation of FieldElements.
	// A FieldElement is either a simple identifier, an array
	// access element or a dictionary access element.
	case class FieldReference(items: List[FieldElement])
	// Represents a simple identifier between dots
	case class JsonIdentifier(id: String) extends FieldElement
	// Represents an array access identifier ("field[0]")
	case class ArrayAccess(id: JsonIdentifier, index: Int) extends FieldElement
	// Represents a dictionary access identifier ("field['inner']")
	case class DictionaryAccess(id: JsonIdentifier, field: String) extends FieldElement
	object ReferenceAll extends FieldElement

	
	def getFieldValue(json: JObject, id: FieldReference): JValue = {
		// tempJson holds the result we want to return
		var tempJson: JValue = json

		id.items.foreach({
			case ReferenceAll => tempJson
			case i: JsonIdentifier =>
				tempJson = tempJson \ i.id
			case a: ArrayAccess =>
				val obj = tempJson \ a.id.id
				obj match {
					case array: JArray =>
						if (a.index < array.arr.length)
							tempJson = array(a.index)
						else return JNothing
					case _ => return JNothing
				}
			case d: DictionaryAccess =>
				tempJson = tempJson \ d.id.id \ d.field
			case _ =>
		})

		tempJson
	}

	type P[+T] = PackratParser[T]

	lazy val local_field_reference: P[FieldReference] =
		repsep(field_element, ".") ^^ { case i => FieldReference(i) }
	lazy val field_element: P[FieldElement] =
		reference_all | array_access | dictionary_access | json_identifier
	lazy val json_identifier: P[JsonIdentifier] =
		ident ^^ { case i => JsonIdentifier(i) }
	lazy val array_access: P[ArrayAccess] =
		json_identifier ~ "[" ~ wholeNumber ~ "]" ^^ {
			case id ~ "[" ~ index ~ "]" =>
				ArrayAccess(id, index.toInt)
		}
	lazy val dictionary_access: P[DictionaryAccess] =
		json_identifier ~ "[" ~ "'" ~ ident ~ "'" ~ "]" ^^ {
			case id ~ "[" ~ "'" ~ field ~ "'" ~ "]" =>
				DictionaryAccess(id, field)
		}
	lazy val reference_all: P[FieldElement] = "*" ^^ { case _ => ReferenceAll }
} 

package io.coral.api.security

import java.util.UUID
import org.json4s.JObject
import io.coral.api.Runtime

object Permission {
	implicit val formats = org.json4s.DefaultFormats

	def isOwnerPermission(p: Permission, runtime: Runtime): Boolean = {
		p.user == runtime.owner &&
		p.method == "*" &&
		p.uri == s"/api/runtimes/${runtime.name}
case class Permission(// The unique identifier of the permission
					  id: UUID,
					  // The owner name of the permission
					  user: UUID,
					  // The name of the runtime that the permission applies to
					  runtime: UUID,
					  // The method of the permission
					  method: String,
					  // The URI of the permission
					  uri: String,
					  // Whether allowed or denied
					  allowed: Boolean) 

package io.coral.actors.transform

import akka.actor.Props
import io.coral.actors.{SimpleEmitTrigger, CoralActor}
import io.coral.lib.Random
import org.json4s.JsonAST.JNothing
import org.json4s.{JObject, JValue}

object SampleActor {
	implicit val formats = org.json4s.DefaultFormats

	def getParams(json: JValue) = {
		for {
			fraction <- (json \ "params" \ "fraction").extractOpt[Double]
		} yield {
			fraction
		}
	}

	def apply(json: JValue): Option[Props] = {
		getParams(json).map(_ => Props(classOf[SampleActor], json, Random))
	}
}

class SampleActor(json: JObject, random: Random)
	extends CoralActor(json)
	with SimpleEmitTrigger {
	val fraction: Double = SampleActor.getParams(json).get
	var randomStream: Stream[Boolean] = random.binomial(fraction)

	def next(): Boolean = {
		val value = randomStream.head
		randomStream = randomStream.tail
		value
	}

	override def simpleEmitTrigger(json: JObject): Option[JValue] = {
		next() match {
			case false => Some(JNothing)
			case true => Some(json)
		}
	}
} 

package com.ibm.aardpfark.pfa.expression

import com.ibm.aardpfark.pfa.types.WithSchema
import org.json4s.{JField, JObject, JValue}


private[pfa] class CellRetrieval[T <: WithSchema](name: String, path: Seq[PFAExpression] = Seq())
  extends PFAExpression {
  import org.json4s.JsonDSL._

  override def json: JValue = {
    JObject(JField("cell", name), JField("path", path.map(_.json)))
  }
}

private[pfa] class AttrRetrieval(attr: PFAExpression, path: Seq[PFAExpression] = Seq())
  extends PFAExpression {
  import org.json4s.JsonDSL._

  override def json: JValue = {
    JObject(JField("attr", attr.json), JField("path", path.map(_.json)))
  }
}

trait AttributeRetrieval {

  object Attr {
    def apply(ref: PFAExpression, path: PFAExpression*) = new AttrRetrieval(ref, path)
  }

} 

package com.ibm.aardpfark.pfa.expression

import com.ibm.aardpfark.pfa.document.{PFAExpressionSerializer, ParamSerializer, SchemaSerializer}
import org.json4s.native.JsonMethods.parse
import org.json4s.native.Serialization
import org.json4s.native.Serialization.write
import org.json4s.{JDouble, JField, JInt, JObject, JString, JValue, NoTypeHints}


class FunctionCall(name: String, args: Any*) extends PFAExpression {
  import com.ibm.aardpfark.pfa.dsl._
  import org.json4s.JsonDSL._

  override def json: JValue = {
    val jArgs = args.map {
      case n: Double =>
        JDouble(n)
      case i: Int =>
        JInt(i)
      case s: String =>
        JString(s)
      case expr: PFAExpression =>
        expr.json
      case fnDef: FunctionDef =>
        implicit val formats = Serialization.formats(NoTypeHints) +
          new SchemaSerializer +
          new PFAExpressionSerializer +
          new ParamSerializer
        parse(write(fnDef))
    }
    JObject(JField(name, jArgs) :: Nil)
  }
} 

package com.ibm.aardpfark.pfa

import com.ibm.aardpfark.pfa.expression.PFAExpression
import org.json4s.JsonAST.{JBool, JLong}
import org.json4s.{JDouble, JField, JInt, JNull, JObject, JString, JValue}
import org.json4s.JsonDSL._

package object types {
  private[pfa] trait API {

    sealed trait Literal extends PFAExpression

    implicit class StringLiteral(s: String) extends Literal {
      override def json: JValue = "string" -> s
    }

    implicit class DoubleLiteral(d: Double) extends Literal {
      override def json: JValue = "double" -> d
    }

    implicit class FloatLiteral(f: Float) extends Literal {
      override def json: JValue = "float" -> f
    }

    implicit class IntLiteral(i: Int) extends Literal {
      override def json: JValue = "int" -> i
    }

    implicit class LongLiteral(l: Long) extends Literal {
      override def json: JValue = "long" -> l
    }

    implicit class BooleanLiteral(b: Boolean) extends Literal {
      override def json: JValue = b
    }

    case object NullLiteral extends Literal {
      override def json: JValue = JNull
    }

  }
} 

package io.radicalbit.nsdb.web

import io.radicalbit.nsdb.common.statement.RelativeComparisonValue
import org.json4s.{CustomSerializer, JObject, JString}
import org.json4s.JsonAST.{JField, JInt, JLong}

case object CustomSerializerForTest
    extends CustomSerializer[RelativeComparisonValue](_ =>
      ({
        case JObject(
            List(JField("value", JLong(0L)),
                 JField("operator", JString(operator)),
                 JField("quantity", JInt(quantity)),
                 JField("unitMeasure", JString(unitMeasure)))) =>
          RelativeComparisonValue(0L, operator, quantity.intValue, unitMeasure)
      }, {
        case RelativeComparisonValue(_, operator, quantity: Long, unitMeasure) =>
          JObject(
            List(JField("value", JLong(0L)),
                 JField("operator", JString(operator)),
                 JField("quantity", JLong(quantity)),
                 JField("unitMeasure", JString(unitMeasure))))

      })) 

package io.radicalbit.nsdb.web

import io.radicalbit.nsdb.common._
import io.radicalbit.nsdb.common.protocol.Bit
import org.json4s.JsonAST.{JDouble, JField, JInt, JLong}
import org.json4s.{CustomSerializer, JObject, JString, JsonAST}


case object BitSerializer
    extends CustomSerializer[Bit](
      _ =>
        (
          {
            case _ =>
              throw new IllegalAccessException(
                "BitSerializer can be used only for serialization and not for deserialization")
          }, {
            case bit: Bit =>
              def extractJValue(nsdbType: NSDbType): JsonAST.JValue = {
                nsdbType match {
                  case NSDbDoubleType(rawValue) => JDouble(rawValue)
                  case NSDbIntType(rawValue)    => JInt(rawValue)
                  case NSDbLongType(rawValue)   => JLong(rawValue)
                  case NSDbStringType(rawValue) => JString(rawValue)
                }
              }
              JObject(
                List(
                  JField("timestamp", JLong(bit.timestamp)),
                  JField("value", extractJValue(bit.value)),
                  JField("dimensions",
                         JObject(bit.dimensions.map { case (k, v)   => JField(k, extractJValue(v)) }.toList)),
                  JField("tags", JObject(bit.tags.map { case (k, v) => JField(k, extractJValue(v)) }.toList))
                )
              )
          }
      )) 

package com.salesforce.op.stages

import com.salesforce.op.features._
import com.salesforce.op.features.types._
import com.salesforce.op.stages.OpPipelineStageReaderWriter._
import com.salesforce.op.test.PassengerSparkFixtureTest
import com.salesforce.op.utils.reflection.ReflectionUtils
import com.salesforce.op.utils.spark.RichDataset._
import org.apache.spark.ml.{Model, Transformer}
import org.apache.spark.sql.types.{DataType, Metadata, MetadataBuilder}
import org.json4s.JsonAST.JValue
import org.json4s.jackson.JsonMethods.{compact, parse, pretty, render}
import org.json4s.{JArray, JObject}
import org.scalatest.FlatSpec
import org.slf4j.LoggerFactory


// TODO: consider adding a read/write test for a spark wrapped stage as well
private[stages] abstract class OpPipelineStageReaderWriterTest
  extends FlatSpec with PassengerSparkFixtureTest {

  val meta = new MetadataBuilder().putString("foo", "bar").build()
  val expectedFeaturesLength = 1
  def stage: OpPipelineStageBase with Transformer
  val expected: Array[Real]
  val hasOutputName = true

  private val log = LoggerFactory.getLogger(this.getClass)
  private lazy val savePath = tempDir + "/" + this.getClass.getSimpleName + "-" + System.currentTimeMillis()
  private lazy val writer = new OpPipelineStageWriter(stage)
  private lazy val stageJsonString: String = writer.writeToJsonString(savePath)
  private lazy val stageJson: JValue = parse(stageJsonString)
  private lazy val isModel = stage.isInstanceOf[Model[_]]
  private val FN = FieldNames

  Spec(this.getClass) should "write stage uid" in {
    log.info(pretty(stageJson))
    (stageJson \ FN.Uid.entryName).extract[String] shouldBe stage.uid
  }
  it should "write class name" in {
    (stageJson \ FN.Class.entryName).extract[String] shouldBe stage.getClass.getName
  }
  it should "write params map" in {
    val params = extractParams(stageJson).extract[Map[String, Any]]
    if (hasOutputName) {
      params should have size 4
      params.keys shouldBe Set("inputFeatures", "outputMetadata", "inputSchema", "outputFeatureName")
    } else {
      params should have size 3
      params.keys shouldBe Set("inputFeatures", "outputMetadata", "inputSchema")
    }
  }
  it should "write outputMetadata" in {
    val params = extractParams(stageJson)
    val metadataStr = compact(render(extractParams(stageJson) \ "outputMetadata"))
    val metadata = Metadata.fromJson(metadataStr)
    metadata shouldBe stage.getMetadata()
  }
  it should "write inputSchema" in {
    val schemaStr = compact(render(extractParams(stageJson) \ "inputSchema"))
    val schema = DataType.fromJson(schemaStr)
    schema shouldBe stage.getInputSchema()
  }
  it should "write input features" in {
    val jArray = (extractParams(stageJson) \ "inputFeatures").extract[JArray]
    jArray.values should have length expectedFeaturesLength
    val obj = jArray(0).extract[JObject]
    obj.values.keys shouldBe Set("name", "isResponse", "isRaw", "uid", "typeName", "stages", "originFeatures")
  }
  it should "write model ctor args" in {
    if (stage.isInstanceOf[Model[_]]) {
      val ctorArgs = (stageJson \ FN.CtorArgs.entryName).extract[JObject]
      val (_, args) = ReflectionUtils.bestCtorWithArgs(stage)
      ctorArgs.values.keys shouldBe args.map(_._1).toSet
    }
  }
  it should "load stage correctly" in {
    val reader = new OpPipelineStageReader(stage)
    val stageLoaded = reader.loadFromJsonString(stageJsonString, path = savePath)
    stageLoaded shouldBe a[OpPipelineStageBase]
    stageLoaded shouldBe a[Transformer]
    stageLoaded.getOutput() shouldBe a[FeatureLike[_]]
    val _ = stage.asInstanceOf[Transformer].transform(passengersDataSet)
    val transformed = stageLoaded.asInstanceOf[Transformer].transform(passengersDataSet)
    transformed.collect(stageLoaded.getOutput().asInstanceOf[FeatureLike[Real]]) shouldBe expected
    stageLoaded.uid shouldBe stage.uid
    stageLoaded.operationName shouldBe stage.operationName
    stageLoaded.getInputFeatures() shouldBe stage.getInputFeatures()
    stageLoaded.getInputSchema() shouldBe stage.getInputSchema()
  }

  private def extractParams(stageJson: JValue): JValue = {
    val defaultParamsMap = stageJson \ FN.DefaultParamMap.entryName
    val paramsMap = stageJson \ FN.ParamMap.entryName
    defaultParamsMap.merge(paramsMap)
  }

} 

package is.hail.expr.ir

import is.hail.types.virtual._
import is.hail.expr.JSONAnnotationImpex
import is.hail.utils._
import org.json4s.{CustomSerializer, DefaultFormats, Formats, JObject, JValue}
import org.json4s.JsonDSL._

class NativeReaderOptionsSerializer() extends CustomSerializer[NativeReaderOptions](
  format =>
    ({ case jObj: JObject =>
      implicit val fmt = format
      val filterIntervals = (jObj \ "filterIntervals").extract[Boolean]
      val intervalPointType = IRParser.parseType((jObj \ "intervalPointType").extract[String])
      val intervals = {
        val jv = jObj \ "intervals"
        val ty = TArray(TInterval(intervalPointType))
        JSONAnnotationImpex.importAnnotation(jv, ty).asInstanceOf[IndexedSeq[Interval]]
      }
      NativeReaderOptions(intervals, intervalPointType, filterIntervals)
    }, { case opts: NativeReaderOptions =>
      implicit val fmt = format
      val ty = TArray(TInterval(opts.intervalPointType))
      (("name" -> opts.getClass.getSimpleName) ~
        ("intervals" -> JSONAnnotationImpex.exportAnnotation(opts.intervals, ty)) ~
        ("intervalPointType" -> opts.intervalPointType.parsableString()) ~
        ("filterIntervals" -> opts.filterIntervals))
    })
)

object NativeReaderOptions {
  def fromJValue(jv: JValue): NativeReaderOptions = {
    implicit val formats: Formats = DefaultFormats

    val filterIntervals = (jv \ "filterIntervals").extract[Boolean]
    val intervalPointType = IRParser.parseType((jv \ "intervalPointType").extract[String])
    val intervals = {
      val jvIntervals = jv \ "intervals"
      val ty = TArray(TInterval(intervalPointType))
      JSONAnnotationImpex.importAnnotation(jvIntervals, ty).asInstanceOf[IndexedSeq[Interval]]
    }
    NativeReaderOptions(intervals, intervalPointType, filterIntervals)
  }
}

case class NativeReaderOptions(
  intervals: IndexedSeq[Interval],
  intervalPointType: Type,
  filterIntervals: Boolean = false) {
  def toJson: JValue = {
    val ty = TArray(TInterval(intervalPointType))
    JObject(
      "name" -> "NativeReaderOptions",
      "intervals" -> JSONAnnotationImpex.exportAnnotation(intervals, ty),
      "intervalPointType" -> intervalPointType.parsableString(),
      "filterIntervals" -> filterIntervals)
  }
} 

package org.clulab.wm.eidos.serialization.json

import org.clulab.serialization.json.stringify
import org.clulab.wm.eidos.serialization.jsonld.TidyJObject
import org.clulab.wm.eidos.test.TestUtils.Test
import org.clulab.wm.eidos.utils.PlayUtils
import org.json4s.{JField, JObject}
import org.json4s.JsonDSL._


class TestJSONFormat extends Test {
  private val occupiedList = List[Int](1, 2, 3, 4)
  private val desertedList = List.empty[Int]
  private val occupiedSet = Set[Int](1, 2, 3, 4)
  private val desertedSet = Set.empty[Int]
  private val something = Option[String]("hello")
  private val nothing = Option[String](null)
  private val sloppyRecursive = List("", null)
  private val tidyRecursive = TidyJObject(List(
    "first" -> "",
    "last" -> null
  ))

  val sloppyJObject: JObject =
    ("string" -> "Hello, world!") ~
        ("blank" -> "") ~
        // ("absence" -> null) ~
        ("occupiedList" -> occupiedList) ~
        ("desertedList" -> desertedList) ~
        ("occupiedSet" -> occupiedSet) ~
        ("desertedSet" -> desertedSet) ~
        ("something" -> something) ~
        ("nothing" -> nothing) ~
        ("sloppyRecursive" -> sloppyRecursive)

  val tidyJObject: TidyJObject = TidyJObject(List(
    "string" -> "Hello, world!",
    "blank" -> "",
    "absence" -> null,
    "occupiedList" -> occupiedList,
    "desertedList" -> desertedList,
    "occupiedSet" -> occupiedSet,
    "desertedSet" -> desertedSet,
    "something" -> something,
    "nothing" -> nothing,
    "tidyRecursive" -> tidyRecursive
  ))

  def hasDirtyField(text: String): Boolean = {
    val line = text.replace('\n', ' ').replace('\r', ' ')

    line.matches(".*(blank|absence|null|nothing|desertedList).*")
  }

  behavior of "pretty string"

  it should "not record nulls, Nones, or Emptys" in {
    val sloppyString = stringify(sloppyJObject, pretty = true)
    val tidyString = stringify(tidyJObject, pretty = true)

//    println("pretty sloppy = " + sloppyString)
//    println("pretty tidy = " + tidyString)

    hasDirtyField(sloppyString) should be (true)
    hasDirtyField(tidyString) should be (false)
  }

  behavior of "ugly string"

  it should "not record nulls, Nones, or Emptys" in {
    val sloppyString = stringify(sloppyJObject, pretty = false)
    val tidyString = stringify(tidyJObject, pretty = false)

//    println("ugly sloppy = " + sloppyString)
//    println("ugly tidy = " + tidyString)

    hasDirtyField(sloppyString) should be (true)
    hasDirtyField(tidyString) should be (false)
  }

  behavior of "webapp string"

  it should "not record nulls, Nones, or Emptys" in {
    val sloppyString = PlayUtils.toPlayJson(sloppyJObject).toString
    val tidyString = PlayUtils.toPlayJson(tidyJObject).toString

//    println("webapp sloppy = " + sloppyString)
//    println("webapp tidy = " + tidyString)

    hasDirtyField(sloppyString) should be (true)
    hasDirtyField(tidyString) should be (false)
  }
} 

package com.webank.wedatasphere.linkis.rpc.transform

import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
import org.json4s.{CustomSerializer, JArray, JObject}
import org.json4s.jackson.Serialization.write
import org.json4s.jackson.JsonMethods.parse

//TODO is now only the simplest implementation, and there is a need to optimize it later.(TODO 现在只做最简单的实现，后续有需要再优化)

object JavaCollectionSerializer extends CustomSerializer[java.util.List[_]](implicit formats => ( {
  case j: JArray=> BDPJettyServerHelper.gson.fromJson(write(j), classOf[java.util.List[_]])
}, {
  case list: java.util.List[_] => parse(BDPJettyServerHelper.gson.toJson(list))
}
)
)

object JavaMapSerializer extends CustomSerializer[java.util.Map[_, _]](implicit formats => ( {
  case j: JObject => BDPJettyServerHelper.gson.fromJson(write(j), classOf[java.util.Map[_, _]])
}, {
  case map: java.util.Map[_, _] => parse(BDPJettyServerHelper.gson.toJson(map))
}
)
) 

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.serialization.json.stringify
import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileEditor
import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.StringUtils
import org.json4s.DefaultFormats
import org.json4s.JObject
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonSource extends App {

  class Filter(outputDir: String) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(jValue: JValue, inputFile: File): Unit = {
      val extractions: JValue = jValue \ "_source"

      extractions match {
        case jObject: JObject =>
          val json = stringify(jObject, pretty = true)
          val path = FileEditor(inputFile).setDir(outputDir).get

          FileUtils.printWriterFromFile(path).autoClose { pw =>
            pw.println(json)
          }

        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val outputDir = args(1)
  val filter = new Filter(outputDir)
  val inputFiles = FileUtils.findFiles(inputDir, "json")

  inputFiles.sortBy(_.getName).foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)

    filter.filter(json, inputFile)
  }
} 

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.serialization.json.stringify
import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileEditor
import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.DefaultFormats
import org.json4s.JObject
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonPretty extends App {

  class Filter(outputDir: String) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(jValue: JValue, inputFile: File): Unit = {
      val extractions: JValue = jValue

      extractions match {
        case jObject: JObject =>
          val json = stringify(jObject, pretty = true)
          val path = FileEditor(inputFile).setDir(outputDir).get

          FileUtils.printWriterFromFile(path).autoClose { pw =>
            pw.println(json)
          }

        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val outputDir = args(1)
  val filter = new Filter(outputDir)
  val inputFiles = FileUtils.findFiles(inputDir, "json")

  inputFiles.sortBy(_.getName).foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)

    filter.filter(json, inputFile)
  }
} 

package org.clulab.wm.eidos.apps

import java.io.File
import java.io.PrintWriter

import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.Sinker
import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.TsvWriter
import org.json4s.DefaultFormats
import org.json4s.JArray
import org.json4s.JObject
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonCanonicalNames extends App {

  class Filter(tsvWriter: TsvWriter) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    tsvWriter.println("file", "id", "text", "canonicalName")

    def filter(jValue: JValue, inputFile: File): Unit = {
      val extractions: JValue = jValue \\ "extractions"

      extractions match {
        case JArray(extractions: List[_]) => // Type erasure removes the [JObject]
          extractions.foreach { extraction =>
            val id = (extraction \ "@id").extract[String]
            val text = (extraction \ "text").extract[String]
            val canonicalName = (extraction \ "canonicalName").extract[String]

            tsvWriter.println(inputFile.getName, id, text, canonicalName)
          }
        case JObject(_) =>
        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val outputFile = args(2)

  new TsvWriter(Sinker.printWriterFromFile(outputFile)).autoClose { tsvWriter =>
    val filter = new Filter(tsvWriter)
    val inputFiles = FileUtils.findFiles(inputDir, extension)

    inputFiles.sortBy(_.getName).foreach { inputFile =>
      val text = FileUtils.getTextFromFile(inputFile)
      val json = JsonMethods.parse(text)

      filter.filter(json, inputFile)
    }
  }
} 

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.DefaultFormats
import org.json4s.JArray
import org.json4s.JNothing
import org.json4s.JObject
import org.json4s.JString
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonGeoAndTime extends App {

  class Filter() {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(inputFile: File, jValue: JValue): Unit = {
      println(s"Extracting from ${inputFile.getName}")

      def filterGeo(): Unit = {
        val geoLocations: JValue = (jValue \\ "geolocs" \ "text")

        geoLocations match {
          case JArray(geoLocations: List[_]) => // Type erasure removes the [JString]
            geoLocations.foreach { geoLocation =>
              val text = geoLocation.extract[String]
              val oneLiner = text
                  .replace("\n", "\\n")
                  .replace("\t", "\\t")

              println("\tGeo\t" + oneLiner)
            }
          case JNothing =>
          case _ => throw new RuntimeException(s"Unexpected geoLocations value: $geoLocations")
        }
      }

      def filterTime(): Unit = {
        val timexes: JValue = (jValue \\ "timexes" \ "text")

        timexes match {
          case JArray(timexes: List[_]) => // Type erasure removes the [JString]
            timexes.foreach { timex =>
              val text = timex.extract[String]
              val oneLiner = text
                  .replace("\n", "\\n")
                  .replace("\t", "\\t")

              println("\tTime\t" + oneLiner)
            }
          case JNothing =>
          case _ => throw new RuntimeException(s"Unexpected geoLocations value: $timexes")
        }
      }

      filterGeo()
      filterTime()
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val inputFiles = FileUtils.findFiles(inputDir, extension)
  val filter = new Filter()

  inputFiles.foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)
    filter.filter(inputFile, json)
  }
} 

package stellar.sdk.inet

import okhttp3.HttpUrl
import org.json4s.native.JsonMethods
import org.json4s.{DefaultFormats, Formats, JObject, JValue}

import scala.concurrent.duration.Duration
import scala.util.Try

case class HorizonServerError(uri: HttpUrl, body: JObject)(implicit val formats: Formats) extends Exception(
  s"Server error when communicating with Horizon. $uri -> ${
    implicit val formats: Formats = DefaultFormats
    Try((body \ "detail").extract[String]).getOrElse(JsonMethods.compact(JsonMethods.render(body)))
  }"
)

case class HorizonEntityNotFound(uri: HttpUrl, body: JValue)(implicit val formats: Formats) extends Exception(
  s"Requested entity was not found in Horizon. $uri -> ${
    implicit val formats: Formats = DefaultFormats
    Try((body \ "detail").extract[String]).getOrElse(JsonMethods.compact(JsonMethods.render(body)))
  }"
)

case class HorizonRateLimitExceeded(uri: HttpUrl, retryAfter: Duration)(implicit val formats: Formats) extends Exception(
  s"Horizon request rate limit was exceeded. Try again in $retryAfter"
)

case class HorizonBadRequest(uri: HttpUrl, body: String) extends Exception(
  s"Bad request. $uri -> ${
    implicit val formats: Formats = DefaultFormats
    Try(
      (JsonMethods.parse(body) \ "extras" \ "reason").extract[String]
    ).getOrElse(body)
  }")

case class FailedResponse(cause: String) extends Exception(cause) 

package stellar.sdk.model.response

import org.json4s.native.JsonMethods.{pretty, render}
import org.json4s.{CustomSerializer, JObject}

import scala.util.control.NonFatal

class ResponseParser[T](f: JObject => T)(implicit m: Manifest[T]) extends CustomSerializer[T](_ => ({
  case o: JObject =>
    try {
      f(o)
    } catch {
      case NonFatal(t) => throw ResponseParseException(pretty(render(o)), t)
    }
}, PartialFunction.empty))

case class ResponseParseException(doc: String, cause: Throwable)
  extends Exception(s"Unable to parse document:\n$doc", cause) 

package stellar.sdk.model.response

import org.json4s.native.JsonMethods
import org.json4s.{DefaultFormats, JObject}
import stellar.sdk.model.NativeAmount

case class FeeStatsResponse(lastLedger: Long,
                            lastLedgerBaseFee: NativeAmount,
                            ledgerCapacityUsage: Double,
                            maxFees: FeeStats,
                            chargedFees: FeeStats) {

  @deprecated("Use `chargedFees.min` instead.", "v0.11.0")
  def minAcceptedFee: NativeAmount = chargedFees.min

  @deprecated("Use `chargedFees.mode` instead.", "v0.11.0")
  def modeAcceptedFee: NativeAmount = chargedFees.mode

  @deprecated("Use `chargedFees.percentiles` instead.", "v0.11.0")
  def acceptedFeePercentiles: Map[Int, NativeAmount] = chargedFees.percentiles

}

case class FeeStats(min: NativeAmount,
                    mode: NativeAmount,
                    max: NativeAmount,
                    percentiles: Map[Int, NativeAmount])

object FeeStatsRespDeserializer extends ResponseParser[FeeStatsResponse]({ o: JObject =>
  implicit val formats = DefaultFormats + FeeStatsDeserializer

  def amount(field: String): NativeAmount = NativeAmount((o \ field).extract[String].toLong)

  val lastLedger = (o \ "last_ledger").extract[String].toLong
  val lastLedgerBaseFee = amount("last_ledger_base_fee")
  val ledgerCapacityUsage = (o \ "ledger_capacity_usage").extract[String].toDouble
  val maxFees = (o \ "max_fee").extract[FeeStats]
  val chargedFees = (o \ "fee_charged").extract[FeeStats]

  FeeStatsResponse(lastLedger, lastLedgerBaseFee, ledgerCapacityUsage, maxFees, chargedFees)
})

object FeeStatsDeserializer extends ResponseParser[FeeStats]({ o: JObject =>
  implicit val formats = DefaultFormats

  def amount(field: String): NativeAmount = NativeAmount((o \ field).extract[String].toLong)

  FeeStats(
    min = amount("min"),
    mode = amount("mode"),
    max = amount("max"),
    percentiles = Map(
      10 -> amount("p10"),
      20 -> amount("p20"),
      30 -> amount("p30"),
      40 -> amount("p40"),
      50 -> amount("p50"),
      60 -> amount("p60"),
      70 -> amount("p70"),
      80 -> amount("p80"),
      90 -> amount("p90"),
      95 -> amount("p95"),
      99 -> amount("p99")
    ))
}) 

package org.apress.prospark

import scala.reflect.runtime.universe

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SQLContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.json4s.DefaultFormats
import org.json4s.JDouble
import org.json4s.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.compact
import org.json4s.native.JsonMethods.parse
import org.json4s.native.JsonMethods.render
import org.json4s.string2JsonInput

object CdrDataframeExamplesNAApp {

  case class Cdr(squareId: Int, timeInterval: Long, countryCode: Int,
    smsInActivity: Float, smsOutActivity: Float, callInActivity: Float,
    callOutActivity: Float, internetTrafficActivity: Float)

  def main(args: Array[String]) {
    if (args.length != 4) {
      System.err.println(
        "Usage: CdrDataframeExamplesNAApp <appname> <batchInterval> <hostname> <port>")
      System.exit(1)
    }
    val Seq(appName, batchInterval, hostname, port) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val ssc = new StreamingContext(conf, Seconds(batchInterval.toInt))

    val sqlC = new SQLContext(ssc.sparkContext)
    import sqlC.implicits._
    implicit val formats = DefaultFormats

    val cdrStream = ssc.socketTextStream(hostname, port.toInt)
      .map(_.split("\\t", -1))
      .foreachRDD(rdd => {
        val cdrs = seqToCdr(rdd).toDF()
        cdrs.na.drop("any").show()
        cdrs.na.fill(0, Array("squareId")).show()
        cdrs.na.replace("squareId", Map(0 -> 1)).show()
        println("Correlation: " + cdrs.stat.corr("smsOutActivity", "callOutActivity"))
        println("Covariance: " + cdrs.stat.cov("smsInActivity", "callInActivity"))
        cdrs.stat.crosstab("squareId", "countryCode").show()
        cdrs.stat.freqItems(Array("squareId", "countryCode"), 0.1).show()
        cdrs.stat.crosstab("callOutActivity", "callInActivity").show()
      })

    ssc.start()
    ssc.awaitTermination()
  }

  def seqToCdr(rdd: RDD[Array[String]]): RDD[Cdr] = {
    rdd.map(c => c.map(f => f match {
      case x if x.isEmpty() => "0"
      case x => x
    })).map(c => Cdr(c(0).toInt, c(1).toLong, c(2).toInt, c(3).toFloat,
      c(4).toFloat, c(5).toFloat, c(6).toFloat, c(7).toFloat))
  }
} 

package org.apress.prospark

import scala.reflect.runtime.universe

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SQLContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.StreamingContext
import org.json4s.DefaultFormats
import org.json4s.JDouble
import org.json4s.JObject
import org.json4s.jvalue2extractable
import org.json4s.jvalue2monadic
import org.json4s.native.JsonMethods.compact
import org.json4s.native.JsonMethods.parse
import org.json4s.native.JsonMethods.render
import org.json4s.string2JsonInput

object CdrDataframeExamples3App {

  case class Cdr(squareId: Int, timeInterval: Long, countryCode: Int,
    smsInActivity: Float, smsOutActivity: Float, callInActivity: Float,
    callOutActivity: Float, internetTrafficActivity: Float)

  def main(args: Array[String]) {
    if (args.length != 5) {
      System.err.println(
        "Usage: CdrDataframeExamples3App <appname> <batchInterval> <hostname> <port> <gridJsonPath>")
      System.exit(1)
    }
    val Seq(appName, batchInterval, hostname, port, gridJsonPath) = args.toSeq

    val conf = new SparkConf()
      .setAppName(appName)
      .setJars(SparkContext.jarOfClass(this.getClass).toSeq)

    val ssc = new StreamingContext(conf, Seconds(batchInterval.toInt))

    val sqlC = new SQLContext(ssc.sparkContext)
    import sqlC.implicits._
    implicit val formats = DefaultFormats

    val gridFile = scala.io.Source.fromFile(gridJsonPath).mkString
    val gridGeo = (parse(gridFile) \ "features")
    val gridStr = gridGeo.children.map(r => {
      val c = (r \ "geometry" \ "coordinates").extract[List[List[List[Float]]]].flatten.flatten.map(r => JDouble(r))
      val l = List(("id", r \ "id"), ("x1", c(0)), ("y1", c(1)), ("x2", c(2)), ("y2", c(3)),
        ("x3", c(4)), ("y3", c(5)), ("x4", c(6)), ("y4", c(7)))
      compact(render(JObject(l)))
    })

    val gridDF = sqlC.read.json(ssc.sparkContext.makeRDD(gridStr))

    val cdrStream = ssc.socketTextStream(hostname, port.toInt)
      .map(_.split("\\t", -1))
      .foreachRDD(rdd => {
        val cdrs = seqToCdr(rdd).toDF()
        cdrs.join(gridDF, $"squareId" === $"id").show()
      })

    ssc.start()
    ssc.awaitTermination()
  }

  def seqToCdr(rdd: RDD[Array[String]]): RDD[Cdr] = {
    rdd.map(c => c.map(f => f match {
      case x if x.isEmpty() => "0"
      case x => x
    })).map(c => Cdr(c(0).toInt, c(1).toLong, c(2).toInt, c(3).toFloat,
      c(4).toFloat, c(5).toFloat, c(6).toFloat, c(7).toFloat))
  }
} 

package com.yahoo.maha.core

import java.util.TimeZone

import com.yahoo.maha.core.DruidDerivedFunction._
import org.joda.time.DateTimeZone
import org.json4s.JObject
import org.scalatest.{FunSuiteLike, Matchers}

class DerivedFunctionTest extends FunSuiteLike with Matchers {
  test("Create a DECODE_DIM failure cases") {
    val minLengthCatch = intercept[IllegalArgumentException] {
      new DECODE_DIM("fieldName", "tooFewArgs")
    }
    assert(minLengthCatch.getMessage.contains("Usage: DECODE( expression , search , result [, search , result]... [, default] )"))
  }

  test("Create value DECODE_DIM") {
    val newDecode = new DECODE_DIM("fieldName", "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
    val newDecodeWithoutDefault = new DECODE_DIM("fieldName", "arg1", "decodeVal1", "arg2", "decodeVal2")
    assert(newDecode.apply("arg1") == Some("decodeVal1"))
    assert(newDecode.apply("arg3") == Some("default"))
    assert(newDecodeWithoutDefault.apply("arg3") == None)
    assert(newDecode.apply.isDefinedAt("arg20"))
  }

  test("Attempt LOOKUP_WITH_DECODE fail") {
    val minLengthCatch = intercept[IllegalArgumentException] {
      new LOOKUP_WITH_DECODE("fieldNameSpace", "valueCol", dimensionOverrideMap = Map.empty, "tooFewArgs")
    }
    assert(minLengthCatch.getMessage.contains("Usage: DECODE( expression , search , result [, search , result]... [, default] )"))
  }

  test("Failure to get interval date with blank format") {
    val thrown = intercept[IllegalArgumentException]{
      GET_INTERVAL_DATE.checkFormat("")
    }
    assert(thrown.getMessage.contains("Format for get_interval_date must be d|w|m|day|yr not"))
  }

  test("All Derived Functions should generate proper JSON Strings.") {
    val gid = GET_INTERVAL_DATE("fieldName", "yyyyMMdd")
    val dow = DAY_OF_WEEK("fieldName")
    val dtf = DATETIME_FORMATTER("fieldName", 0, 10)
    val dd = DECODE_DIM("fieldName", "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
    val js = JAVASCRIPT("fieldName", "function(x) { return x > 0; }")
    val rgx = REGEX("fieldName", "blah", 0, true, "t")
    val lu = LOOKUP("namespace", "val", Map("a" -> "b"))
    val lwd = LOOKUP_WITH_DECODE("namespace", "valCol", Map("b" -> "a"), "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
    val lwe = LOOKUP_WITH_EMPTY_VALUE_OVERRIDE("namespace", "valCol", "ovr", Map("c" -> "d"))
    val lwo = LOOKUP_WITH_DECODE_ON_OTHER_COLUMN("namespace", "valCol", "valToCheck", "valIfMatched", "valIfNot", Map("2" -> "4", "b" -> "a"))
    val ltf = LOOKUP_WITH_TIMEFORMATTER("namespace", "valCol", "yyyyMMdd", "yyyy", Map("do" -> "dont"), Some("override"))
    val ldr = LOOKUP_WITH_DECODE_RETAIN_MISSING_VALUE("namespace", "valCol", true, true, Map("rtn" -> "not"), "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
    val dtz = DRUID_TIME_FORMAT("format", DateTimeZone.forID("Asia/Jakarta"))
    val dpg = DRUID_TIME_FORMAT_WITH_PERIOD_GRANULARITY("format", "P1D", DateTimeZone.forID("Asia/Jakarta"))
    val rc = TIME_FORMAT_WITH_REQUEST_CONTEXT("yyyy")
    val lwt = LOOKUP_WITH_TIMESTAMP("namespace", "val", "fmt", Map.empty, Some("ovrVal"), asMillis = false)

    val resultArray = List(gid, dow, dtf, dd, js, rgx, lu, lwd, lwe, lwo, ltf, ldr, dtz, dpg, rc, lwt)

    val expectedJSONs = List(
      """{"function_type":"GET_INTERVAL_DATE","fieldName":"fieldName","format":"yyyyMMdd"}""",
      """{"function_type":"DAY_OF_WEEK","fieldName":"fieldName"}""",
      """{"function_type":"DATETIME_FORMATTER","fieldName":"fieldName","index":0,"length":10}""",
      """{"function_type":"DECODE_DIM","fieldName":"fieldName","args":"arg1,decodeVal1,arg2,decodeVal2,default"}""",
      """{"function_type":"JAVASCRIPT","fieldName":"fieldName","function":"function(x) { return x > 0; }"}""",
      """{"function_type":"REGEX","fieldName":"fieldName","expr":"blah","index":0,"replaceMissingValue":true,"replaceMissingValueWith":"t"}""",
      """{"function_type":"LOOKUP","lookupNamespace":"namespace","valueColumn":"val","dimensionOverrideMap":{"a":"b"}}""",
      """{"function_type":"LOOKUP_WITH_DECODE","lookupNamespace":"namespace","valueColumn":"valCol","dimensionOverrideMap":{"b":"a"},"args":"arg1,decodeVal1,arg2,decodeVal2,default"}""",
      """{"function_type":"LOOKUP_WITH_EMPTY_VALUE_OVERRIDE","lookupNamespace":"namespace","valueColumn":"valCol","overrideValue":"ovr","dimensionOverrideMap":{"c":"d"}}""",
      """{"function_type":"LOOKUP_WITH_DECODE_ON_OTHER_COLUMN","lookupNamespace":"namespace","columnToCheck":"valCol","valueToCheck":"valToCheck","columnIfValueMatched":"valIfMatched","columnIfValueNotMatched":"valIfNot","dimensionOverrideMap":{"2":"4","b":"a"}}""",
      """{"function_type":"LOOKUP_WITH_TIMEFORMATTER","lookupNamespace":"namespace","valueColumn":"valCol","inputFormat":"yyyyMMdd","resultFormat":"yyyy","dimensionOverrideMap":{"do":"dont"}}""",
      """{"function_type":"LOOKUP_WITH_DECODE_RETAIN_MISSING_VALUE","lookupNamespace":"namespace","valueColumn":"valCol","retainMissingValue":true,"injective":true,"dimensionOverrideMap":{"rtn":"not"},"args":"arg1,decodeVal1,arg2,decodeVal2,default"}""",
      """{"function_type":"DRUID_TIME_FORMAT","format":"format","zone":"Asia/Jakarta"}""",
      """{"function_type":"DRUID_TIME_FORMAT_WITH_PERIOD_GRANULARITY","format":"format","period":"P1D","zone":"Asia/Jakarta"}""",
      """{"function_type":"TIME_FORMAT_WITH_REQUEST_CONTEXT","format":"yyyy"}""",
      """{"function_type":"LOOKUP_WITH_TIMESTAMP","lookupNamespace":"namespace","valueColumn":"val","resultFormat":"fmt","dimensionOverrideMap":{},"overrideValue":"ovrVal","asMillis":false}"""
    )

    import org.json4s._
    import org.json4s.jackson.JsonMethods._
    implicit val formats = DefaultFormats

    val allJSONs: List[JObject] = resultArray.map(expn => expn.asJSON)
    val allJsonStrings: List[String] = allJSONs.map(json => compact(json))

    assert(allJsonStrings.forall(str => expectedJSONs.contains(str)))
  }
} 

// Copyright 2017, Yahoo Holdings Inc.
// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
package com.yahoo.maha.core

import com.yahoo.maha.core.BaseExpressionTest.PRESTO_TIMESTAMP_TO_FORMATTED_DATE
import com.yahoo.maha.core.HiveExpression._
import org.json4s.JObject
import org.scalatest.{FunSuite, Matchers}


class ColumnAnnotationTest extends FunSuite with Matchers {
  test("successfully find HiveShardingExpression with instance") {
    val set: Set[ColumnAnnotation] = Set(HiveShardingExpression(null))
    set.contains(HiveShardingExpression.instance) === true
  }

  test("successfully find ForeignKey with instance") {
    val set: Set[ColumnAnnotation] = Set(ForeignKey("fid"))
    set.contains(ForeignKey.instance) === true
  }

  test("successfully find DayColumn with instance") {
    val set: Set[ColumnAnnotation] = Set(DayColumn("YYYYMMDD"))
    set.contains(DayColumn.instance) === true
  }

  test("successfully find PrestoShardingExpression with instance") {
    val set: Set[ColumnAnnotation] = Set(PrestoShardingExpression(null))
    set.contains(PrestoShardingExpression.instance) === true
  }

  test("Instantiate  Hive/PrestoShardingExpression") {
    implicit val cc: ColumnContext = new ColumnContext
    val derivedMin : HiveDerivedExpression = HiveDerivedExpression.fromExpression(MIN("{thing}"))
    val shardingExpr : HiveShardingExpression = new HiveShardingExpression(derivedMin)
    assert(shardingExpr.instance == HiveShardingExpression.instance)

    val prestoMin : PrestoDerivedExpression = PrestoDerivedExpression.fromExpression(PRESTO_TIMESTAMP_TO_FORMATTED_DATE("{created_date}", "YYYY-MM-dd"))
    val prestoShardingExpr : PrestoShardingExpression = new PrestoShardingExpression(prestoMin)
    assert(prestoShardingExpr.instance == PrestoShardingExpression.instance)
  }

  test("Instantiate ForeignKey and DayColumn") {
    val fk : ForeignKey = new ForeignKey("public_name")
    assert(fk.publicDimName == "public_name")
    assert(fk.instance == ForeignKey.instance)
    val dc : DayColumn = new DayColumn("format")
    assert(dc.fmt == "format")
    assert(dc.instance == DayColumn.instance)
  }

  test("Column annotations should convert to JSON properly.") {
    implicit val cc: ColumnContext = new ColumnContext
    val fk: ForeignKey = new ForeignKey("pd")
    val dc : DayColumn = new DayColumn("format")
    val derivedMin : HiveDerivedExpression = HiveDerivedExpression.fromExpression(MIN("{thing}"))
    val prestoMin : PrestoDerivedExpression = PrestoDerivedExpression.fromExpression(PRESTO_TIMESTAMP_TO_FORMATTED_DATE("{created_date}", "YYYY-MM-dd"))

    val hs: HiveShardingExpression = new HiveShardingExpression(derivedMin)
    val ps: PrestoShardingExpression = new PrestoShardingExpression(prestoMin)

    val expns = Set(fk, dc, hs, ps)

    import org.json4s._
    import org.json4s.jackson.JsonMethods._
    implicit val formats = DefaultFormats

    //All actual Annotations
    val allJSONs: String = expns.map(expn => compact(expn.asJSON)).mkString(",")

    //All instances with null args (null checking)
    val instances: String = expns.map(expn => compact(expn.instance.asJSON)).mkString(",")

    val allAnnotations = List(
      """{"annotation":"ForeignKey","publicDimName":"pd"}"""
      ,"""{"annotation":"DayColumn","fmt":"format"}"""
      ,"""{"annotation":"HiveShardingExpression","expression":"HiveDerivedExpression(""" //split into two to avoid columnContext.
      ,""",MIN(COL({thing},false,false)))"}"""
      ,"""{"annotation":"PrestoShardingExpression","expression":"PrestoDerivedExpression"""
      ,""",PRESTO_TIMESTAMP_TO_FORMATTED_DATE(COL({created_date},false,false),YYYY-MM-dd))"}"""
    )
    val allnstances = List(
      """{"annotation":"ForeignKey","publicDimName":"instance"}"""
      ,"""{"annotation":"DayColumn","fmt":"instance"}"""
      ,"""{"annotation":"HiveShardingExpression","expression":null}"""
      ,"""{"annotation":"PrestoShardingExpression","expression":null}"""
    )
    assert(allAnnotations.forall(
      annotation =>
        allJSONs.contains(annotation))
    )
    assert(allnstances.forall(
      instance =>
        instances.contains(instance))
    )

  }

} 

package algolia.responses

import org.json4s.{DefaultFormats, JObject}

case class BrowseResult(
    cursor: Option[String],
    hits: Seq[JObject],
    processingTimeMS: Int,
    query: String,
    params: String,
    nbHits: Option[Int],
    page: Option[Int],
    histPerPage: Option[Int],
    nbPages: Option[Int]
) {

  implicit val formats: DefaultFormats = org.json4s.DefaultFormats

  def asHit[T <: Hit: Manifest]: Seq[T] = hits.map(_.extract[T])

  def as[T: Manifest]: Seq[T] = hits.map(_.extract[T])

  def asWithObjectID[T <: ObjectID: Manifest]: Seq[T] = hits.map(_.extract[T])

}