com.google.gson.Gson Scala Examples

package com.webank.wedatasphere.linkis.entrance.utils

import java.util

import com.google.gson.Gson
import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.exception.JobHistoryFailedException
import com.webank.wedatasphere.linkis.protocol.query.cache.{CacheTaskResult, RequestReadCache}
import com.webank.wedatasphere.linkis.protocol.query.{RequestPersistTask, RequestQueryTask, RequestUpdateTask, ResponsePersist}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState


  def forceKill(taskID:Long):Unit = {
    val requestUpdateTask = new RequestUpdateTask
    requestUpdateTask.setTaskID(taskID)
    requestUpdateTask.setStatus("Cancelled")
    sender.ask(requestUpdateTask)
  }

  private def getTaskByTaskID(taskID:Long):RequestPersistTask = {
    val requestQueryTask = new RequestQueryTask()
    requestQueryTask.setTaskID(taskID)
    requestQueryTask.setSource(null)
    val task = Utils.tryCatch{
      val taskResponse = sender.ask(requestQueryTask)
      taskResponse match {
        case responsePersist:ResponsePersist => val status = responsePersist.getStatus
          if (status != SUCCESS_FLAG){
            logger.error(s"query from jobHistory status failed, status is $status")
            throw JobHistoryFailedException("query from jobHistory status failed")
          }else{
            val data = responsePersist.getData
            data.get(TASK_MAP_KEY) match {
              case tasks:util.List[util.Map[String, Object]] => tasks.get(0) match {
                case map:util.Map[String, Object] => val gson = new Gson()
                  val json = gson.toJson(map)
                  val requestPersistTask = gson.fromJson(json, classOf[RequestPersistTask])
                  requestPersistTask
                case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct RequestPersistTask type taskId is $taskID")
              }
              case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct List type taskId is $taskID")
            }
          }
        case _ => logger.error("get query response incorrectly")
          throw JobHistoryFailedException("get query response incorrectly")
      }
    }{
      case errorException:ErrorException => throw errorException
      case e:Exception => val e1 = JobHistoryFailedException(s"query taskId $taskID error")
        e1.initCause(e)
        throw e
    }
    task
  }


} 

package com.webank.wedatasphere.linkis.filesystem

import com.google.gson.Gson
import com.webank.wedatasphere.linkis.filesystem.action.OpenScriptFromBMLAction
import com.webank.wedatasphere.linkis.filesystem.conf.WorkspaceClientConf
import com.webank.wedatasphere.linkis.filesystem.request.{WorkspaceClient, WorkspaceHttpConf}
import com.webank.wedatasphere.linkis.filesystem.response.ScriptFromBMLResponse
import com.webank.wedatasphere.linkis.filesystem.result.ScriptFromBMLResult
import com.webank.wedatasphere.linkis.httpclient.authentication.AuthenticationStrategy
import com.webank.wedatasphere.linkis.httpclient.config.{ClientConfig, ClientConfigBuilder}
import com.webank.wedatasphere.linkis.httpclient.dws.DWSHttpClient
import com.webank.wedatasphere.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy
import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig
import com.webank.wedatasphere.linkis.httpclient.response.Result



class WorkspaceClientImpl extends WorkspaceClient with WorkspaceHttpConf {

  def this(user: String, token: String,gatewayAddress:String) {
    this
    this.user = user
    this.token = token
    val maxConnection: Int = 100
    val readTimeout: Int = 10000
    val connectiontimeout = 30000
    authenticationStrategy = new TokenAuthenticationStrategy()
    clientConfig = ClientConfigBuilder.newBuilder().addUJESServerUrl( "http://" + gatewayAddress)
      .connectionTimeout(connectiontimeout).discoveryEnabled(false)
      .loadbalancerEnabled(false).maxConnectionSize(maxConnection)
      .retryEnabled(false).readTimeout(readTimeout)
      .setAuthenticationStrategy(authenticationStrategy).
      setAuthTokenKey(WorkspaceClientConf.tokenKey).setAuthTokenValue(this.token).build()
    dwsClientConfig = new DWSClientConfig(clientConfig)
    dwsClientConfig.setDWSVersion(WorkspaceClientConf.dwsVersion)
    dwsClientName = "Workspace-Client"
    dwsClient = new DWSHttpClient(dwsClientConfig, dwsClientName)
  }
  @throws[IllegalArgumentException]
  override def requestOpenScriptFromBML(resourceId: String, version: String, fileName: String): ScriptFromBMLResponse = {
    val action: OpenScriptFromBMLAction = new OpenScriptFromBMLAction()
    action.setUser(user)
    action.setParameter("resourceId", resourceId)
    action.setParameter("version", version)
    action.setParameter("fileName", fileName)
    val result: Result = dwsClient.execute(action)
    result match {
      case r: ScriptFromBMLResult => {
        if(r.getStatus!= 0) throw new IllegalArgumentException(r.getMessage)
        ScriptFromBMLResponse(r.scriptContent, r.metadata)
      }
      case _ => null
    }
  }

  override protected var user: String = _
  override protected var token: String = _
  override protected var authenticationStrategy: AuthenticationStrategy = _
  override protected var clientConfig: ClientConfig = _
  override protected var dwsClientConfig: DWSClientConfig = _
  override protected var dwsClientName: String = _
  override protected var dwsClient: DWSHttpClient = _
} 

import MockedResponse._
import com.lyft.networking.apiObjects.CostEstimate
import com.uber.sdk.rides.client.model.PriceEstimate
import models._
import org.scalatest.{FlatSpec, Matchers}

class ModelSpec extends FlatSpec with Matchers {
  behavior of "Models"
  "UberPriceModel" should "return CabPrice" in {
    import com.google.gson.Gson
    val g = new Gson

    val priceEstimate = g.fromJson(priceEstimateJson, classOf[PriceEstimate])
    val source = Location("source_test", 45f, 34f)
    val destination = Location("destination_test", 55f, 34f)

    UberPriceModel(priceEstimate, source, destination) should matchPattern {
      case CabPrice("Uber", "a1111c8c-c720-46c3-8534-2fcdd730040d", "uberX", _, Some(6.17f), 1.0, _, "source_test", "destination_test", _) =>
    }
  }

  "LyftPriceModel" should "return CabPrice" in {
    import com.google.gson.Gson

    val g = new Gson

    val costEstimate = g.fromJson(costEstimateJson, classOf[CostEstimate])
    val source = Location("source_test", 45f, 34f)
    val destination = Location("destination_test", 55f, 34f)

    LyftPriceModel(costEstimate, source, destination) should matchPattern {
      case CabPrice("Lyft", "lyft_plus", "Lyft Plus", _, Some(3.29f), 1.25, _, "source_test", "destination_test", _) =>
    }
  }

  "WeatherModel" should "return Weather" in {
    val location = Location("source_test", 45f, 34f)
    WeatherModel(weatherResponse, location) should matchPattern {
      case Weather("source_test", Some(40.38f), Some(0.64f), Some(995.66f), Some(0.83f), None, None, Some(11.05f), 1543448090) =>
    }
  }
} 

import com.amazonaws.services.dynamodbv2.document.internal.InternalUtils
import com.amazonaws.services.dynamodbv2.streamsadapter.model.RecordAdapter
import com.amazonaws.services.kinesis.model.Record
import com.google.gson.Gson
import org.apache.spark.sql._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.kinesis.dynamostream.KinesisInitialPositions.Latest
import org.apache.spark.streaming.kinesis.dynamostream.KinesisInputDStream
import org.apache.spark.streaming.{Milliseconds, Seconds, StreamingContext}

object Trials extends App {

  import org.apache.log4j.{Level, Logger}

  Logger.getLogger("org").setLevel(Level.ERROR)
  Logger.getLogger("akka").setLevel(Level.ERROR)

  //session setup
  System.setProperty("hadoop.home.dir", "C:\\winutils")
  val sparkSession = SparkSession.builder()
    .master("local[*]")
    .appName("test")
    .getOrCreate()
  val sc = sparkSession.sparkContext
  val ssc = new StreamingContext(sc, Seconds(10))
  val sqlContext = sparkSession.sqlContext

  //creates an array of strings from raw byte array
  def kinesisRecordHandler: Record => Array[String] = (record: Record) => new String(record.getData.array()).split(",")

  //converts records to map of key value pair and then json
  def recordHandler = (record: Record) => {
    val gson = new Gson
    val sRecord = record.asInstanceOf[RecordAdapter].getInternalObject
    val map = InternalUtils.toSimpleMapValue(sRecord.getDynamodb.getNewImage)
    gson.toJson(map)
  }

  case class CabPrice(cab_type: String, product_id: String, name: String, price: String, distance: String, surge_multiplier: String, time_stamp: String, source: String, destination: String, id: String)

  val stream_cab = KinesisInputDStream.builder
    .streamingContext(ssc)
    .streamName("cab_rides")
    .regionName("us-east-1")
    .initialPosition(new Latest())
    .checkpointAppName("cab_rides-app")
    .checkpointInterval(Milliseconds(1000))
    .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
    .buildWithMessageHandler(recordHandler)


  val stream_weather = KinesisInputDStream.builder
    .streamingContext(ssc)
    .streamName("weather")
    .regionName("us-east-1")
    .initialPosition(new Latest())
    .checkpointAppName("cab_rides-app")
    .checkpointInterval(Milliseconds(1000))
    .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
    .buildWithMessageHandler(recordHandler)


  //creating dataframe, can be stored as temp view
  val cabSchema = Encoders.product[CabPrice].schema
  stream_cab.foreachRDD(rdd => {
    import sqlContext.implicits._
    //val xx: Dataset[String] = rdd.toDS()

    val df: DataFrame = sqlContext.read.schema(cabSchema).json(rdd.toDS())
    df.show()

  })
  ssc.start()
  ssc.awaitTermination()

} 

package com.cloudera.sprue

import java.io._
import org.apache.commons._
import org.apache.http._
import org.apache.http.client._
import org.apache.http.client.methods.HttpPost
import java.util.ArrayList
import org.apache.http.client.entity.UrlEncodedFormEntity
import com.google.gson.Gson
import java.util.HashMap
import java.lang.reflect.Type
import com.google.gson.reflect.TypeToken
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.spark.sql.Row

 
case class MetricsTags(state: String)
case class OpenTSDBMessageElement(metric: String, timestamp: Long, value: Long, tags: MetricsTags)

object TSDBUpdater {
    val client = new DefaultHttpClient()
    // val client = HttpClientBuilder.create.build
}


class TSDBUpdater (url : String) extends Serializable {
       
  def loadPatientStats (row : Row) {
      
     val metricList = new ArrayList[OpenTSDBMessageElement]()
     val jmap = new MetricsTags(row.getString(0))
     val evalTimestamp = row.getLong(1)
     
     val sirsMetric = new OpenTSDBMessageElement("sirs", evalTimestamp, row.getLong(2), jmap)
     metricList.add(sirsMetric)
     
     val sepsisMetric = new OpenTSDBMessageElement("sepsis", evalTimestamp, row.getLong(3), jmap)
     metricList.add(sepsisMetric)
     
     val severeSepsisMetric = new OpenTSDBMessageElement("severeSepsis", evalTimestamp, row.getLong(4), jmap)
     metricList.add(severeSepsisMetric)

    val septicShockMetric = new OpenTSDBMessageElement("septicShock", evalTimestamp, row.getLong(5), jmap)
     metricList.add(septicShockMetric)
     
    val organMetric = new OpenTSDBMessageElement("organDysfunctionSyndrome", evalTimestamp, row.getLong(6), jmap)
     metricList.add(organMetric)

    val metricsAsJson = new Gson().toJson(metricList)
      
    val post = new HttpPost(url)
    
    post.setHeader("Content-type", "application/json");
    post.setEntity(new StringEntity(metricsAsJson));

    val response = TSDBUpdater.client.execute(post) 
  //  println("response =====" + response.toString())
  }
} 

package com.stratio.sparta.plugin.input.twitter

import java.io.{Serializable => JSerializable}

import com.google.gson.Gson
import com.stratio.sparta.sdk.pipeline.input.Input
import com.stratio.sparta.sdk.properties.ValidatingPropertyMap._
import org.apache.spark.sql.Row
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.twitter.TwitterUtils
import twitter4j.TwitterFactory
import twitter4j.conf.ConfigurationBuilder

import scala.util.{Failure, Success, Try}


class TwitterJsonInput(properties: Map[String, JSerializable]) extends Input(properties) {

  System.setProperty("twitter4j.oauth.consumerKey", properties.getString("consumerKey"))
  System.setProperty("twitter4j.oauth.consumerSecret", properties.getString("consumerSecret"))
  System.setProperty("twitter4j.oauth.accessToken", properties.getString("accessToken"))
  System.setProperty("twitter4j.oauth.accessTokenSecret", properties.getString("accessTokenSecret"))

  val cb = new ConfigurationBuilder()
  val tf = new TwitterFactory(cb.build())
  val twitterApi = tf.getInstance()
  val trends = twitterApi.getPlaceTrends(1).getTrends.map(trend => trend.getName)
  val terms: Option[Seq[String]] = Try(properties.getString("termsOfSearch")) match {
    case Success("") => None
    case Success(t: String) => Some(t.split(",").toSeq)
    case Failure(_) => None
  }
  val search = terms.getOrElse(trends.toSeq)

  def initStream(ssc: StreamingContext, sparkStorageLevel: String): DStream[Row] = {
    TwitterUtils.createStream(ssc, None, search, storageLevel(sparkStorageLevel))
      .map(stream => {
        val gson = new Gson()
        Row(gson.toJson(stream))
      }
      )
  }
} 

package com.expedia.www.haystack.trace.reader.stores.readers.es

import com.expedia.www.haystack.commons.metrics.MetricsSupport
import com.expedia.www.haystack.trace.commons.clients.es.AWSSigningJestClientFactory
import com.expedia.www.haystack.trace.commons.config.entities.AWSRequestSigningConfiguration
import com.expedia.www.haystack.trace.reader.config.entities.ElasticSearchClientConfiguration
import com.expedia.www.haystack.trace.reader.metrics.AppMetricNames
import com.expedia.www.haystack.trace.reader.stores.readers.es.ESUtils._
import com.google.gson.Gson
import io.searchbox.client.config.HttpClientConfig
import io.searchbox.client.{JestClient, JestClientFactory}
import io.searchbox.core.{Search, SearchResult}
import org.slf4j.LoggerFactory

import scala.concurrent.{ExecutionContextExecutor, Future, Promise}
import scala.util.Try

class ElasticSearchReader(config: ElasticSearchClientConfiguration, awsRequestSigningConfig: AWSRequestSigningConfiguration)(implicit val dispatcher: ExecutionContextExecutor) extends MetricsSupport with AutoCloseable {
  private val LOGGER = LoggerFactory.getLogger(classOf[ElasticSearchReader])
  private val readTimer = metricRegistry.timer(AppMetricNames.ELASTIC_SEARCH_READ_TIME)
  private val readFailures = metricRegistry.meter(AppMetricNames.ELASTIC_SEARCH_READ_FAILURES)

  // initialize the elastic search client
  private val esClient: JestClient = {
    LOGGER.info("Initializing the http elastic search client with endpoint={}", config.endpoint)

    val factory = {
      if (awsRequestSigningConfig.enabled) {
        LOGGER.info("using AWSSigningJestClientFactory for es client")
        new AWSSigningJestClientFactory(awsRequestSigningConfig)
      } else {
        LOGGER.info("using JestClientFactory for es client")
        new JestClientFactory()
      }
    }

    val builder = new HttpClientConfig.Builder(config.endpoint)
      .multiThreaded(true)
      .connTimeout(config.connectionTimeoutMillis)
      .readTimeout(config.readTimeoutMillis)

    if (config.username.isDefined && config.password.isDefined) {
      builder.defaultCredentials(config.username.get, config.password.get)
    }

    factory.setHttpClientConfig(builder.build())
    factory.getObject
  }

  def search(request: Search): Future[SearchResult] = {
    val promise = Promise[SearchResult]()
    val time = readTimer.time()
    try {
      LOGGER.debug(s"elastic search query requested: ${request.toString}', query: '${request.toJson}'")
      esClient.executeAsync(request, new ElasticSearchReadResultListener(request, promise, time, readFailures))
      promise.future
    } catch {
      case ex: Exception =>
        readFailures.mark()
        time.stop()
        LOGGER.error(s"Failed to read from elasticsearch for request=${request.toJson} with exception", ex)
        Future.failed(ex)
    }
  }

  def count(request: Search): Future[SearchResult] = {
    val promise = Promise[SearchResult]()
    val time = readTimer.time()
    try {
      LOGGER.debug(s"elastic count query requested: ${request.toString}', query: '${request.toJson}'")
      esClient.executeAsync(request, new ElasticSearchCountResultListener(request, promise, time, readFailures))
      promise.future
    } catch {
      case ex: Exception =>
        readFailures.mark()
        time.stop()
        LOGGER.error(s"Failed to read from elasticsearch for request=${request.getData(new Gson())} with exception", ex)
        Future.failed(ex)
    }
  }

  override def close(): Unit = Try(esClient.shutdownClient())
} 

package com.expedia.www.haystack.trace.reader.unit.stores.readers.es.query

import com.expedia.www.haystack.trace.reader.config.entities.ServiceMetadataIndexConfiguration
import com.expedia.www.haystack.trace.reader.stores.readers.es.query.ServiceMetadataQueryGenerator
import com.expedia.www.haystack.trace.reader.unit.BaseUnitTestSpec
import com.google.gson.Gson

class ServiceMetadataQueryGeneratorSpec extends BaseUnitTestSpec {
  private val indexType = "metadata"
  private val serviceMetadataIndexConfiguration = ServiceMetadataIndexConfiguration(
    enabled = true,
    indexName = "service_metadata",
    indexType = indexType)

  describe("ServiceMetadataQueryGenerator") {
    it("should generate valid aggregation queries for service names") {
      Given("a query generator")
      val queryGenerator = new ServiceMetadataQueryGenerator(serviceMetadataIndexConfiguration)

      When("asked for aggregated service name")
      val query = queryGenerator.generateSearchServiceQuery()

      Then("generate a valid query")
      query.getType should be(indexType)
      query.getData(new Gson()) shouldEqual "{\n  \"size\" : 0,\n  \"aggregations\" : {\n    \"distinct_services\" : {\n      \"terms\" : {\n        \"field\" : \"servicename\",\n        \"size\" : 10000,\n        \"min_doc_count\" : 1,\n        \"shard_min_doc_count\" : 0,\n        \"show_term_doc_count_error\" : false,\n        \"order\" : [\n          {\n            \"_count\" : \"desc\"\n          },\n          {\n            \"_key\" : \"asc\"\n          }\n        ]\n      }\n    }\n  }\n}"
      query.toString shouldEqual "Search{uri=service_metadata/metadata/_search, method=POST}"
    }

    it("should generate valid aggregation queries for operation names") {
      Given("a query generator and a service name")
      val queryGenerator = new ServiceMetadataQueryGenerator(serviceMetadataIndexConfiguration)
      val serviceName = "test_service"
      When("asked for aggregated operation names")
      val query = queryGenerator.generateSearchOperationQuery(serviceName)

      Then("generate a valid query")
      query.getType should be(indexType)
    }
  }
} 

package com.expedia.www.haystack.trace.indexer.unit

import java.util
import java.util.Collections

import com.codahale.metrics.Timer
import com.expedia.www.haystack.commons.metrics.MetricsSupport
import com.expedia.www.haystack.commons.retries.RetryOperation
import com.expedia.www.haystack.trace.indexer.metrics.AppMetricNames
import com.expedia.www.haystack.trace.indexer.writers.es.ElasticSearchResultHandler
import com.google.gson.Gson
import io.searchbox.core.BulkResult
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException
import org.scalatest.easymock.EasyMockSugar
import org.scalatest.{FunSpec, Matchers}

class ElasticSearchResultHandlerSpec extends FunSpec with Matchers with EasyMockSugar with MetricsSupport {
  private val esWriteFailureMeter = metricRegistry.meter(AppMetricNames.ES_WRITE_FAILURE)

  describe("Trace Index Result Handler") {

    it("should complete with success if no failures reported") {
      val retryCallback = mock[RetryOperation.Callback]
      val timer = mock[Timer.Context]
      val bulkResult = mock[BulkResult]

      expecting {
        retryCallback.onResult(bulkResult)
        timer.close()
        bulkResult.getFailedItems.andReturn(Collections.emptyList()).anyTimes()
      }

      whenExecuting(retryCallback, timer, bulkResult) {
        val handler = new ElasticSearchResultHandler(timer, esWriteFailureMeter, retryCallback)
        handler.completed(bulkResult)
        esWriteFailureMeter.getCount shouldBe 0
      }
    }

    it("should complete with success but mark the failures if happen") {
      val retryCallback = mock[RetryOperation.Callback]
      val timer = mock[Timer.Context]
      val bulkResult = mock[BulkResult]
      val outer = new BulkResult(new Gson())
      val resultItem = new outer.BulkResultItem("op", "index", "type", "1", 400,
        "error", 1, "errorType", "errorReason")

      expecting {
        retryCallback.onResult(bulkResult)
        timer.close()
        bulkResult.getFailedItems.andReturn(util.Arrays.asList(resultItem)).anyTimes()
      }

      whenExecuting(retryCallback, timer, bulkResult) {
        val handler = new ElasticSearchResultHandler(timer, esWriteFailureMeter, retryCallback)
        val initialFailures = esWriteFailureMeter.getCount
        handler.completed(bulkResult)
        esWriteFailureMeter.getCount - initialFailures shouldBe 1
      }
    }

    it("should report failure and mark the number of failures, and perform retry on any exception") {
      val retryCallback = mock[RetryOperation.Callback]
      val timer = mock[Timer.Context]
      val bulkResult = mock[BulkResult]

      val error = new RuntimeException
      expecting {
        retryCallback.onError(error, retry = true)
        timer.close()
      }

      whenExecuting(retryCallback, timer, bulkResult) {
        val handler = new ElasticSearchResultHandler(timer, esWriteFailureMeter, retryCallback)
        val initialFailures = esWriteFailureMeter.getCount
        handler.failed(error)
        esWriteFailureMeter.getCount - initialFailures shouldBe 1
      }
    }

    it("should report failure and mark the number of failures and perform function on elastic search specific exception") {
      val retryCallback = mock[RetryOperation.Callback]
      val timer = mock[Timer.Context]
      val bulkResult = mock[BulkResult]

      val error = new EsRejectedExecutionException("too many requests")

      expecting {
        retryCallback.onError(error, retry = true)
        timer.close()
      }

      whenExecuting(retryCallback, timer, bulkResult) {
        val handler = new ElasticSearchResultHandler(timer, esWriteFailureMeter, retryCallback)
        val initialFailures = esWriteFailureMeter.getCount
        handler.failed(error)
        esWriteFailureMeter.getCount - initialFailures shouldBe 1
      }
    }
  }
} 

package com.expedia.www.haystack.trace.indexer.unit

import com.expedia.www.haystack.trace.indexer.writers.es.ThreadSafeBulkBuilder
import com.google.gson.Gson
import io.searchbox.core.Index
import org.scalatest.{FunSpec, Matchers}

class ThreadSafeBulkBuilderSpec extends FunSpec with Matchers {
  private val gson = new Gson()

  describe("Thread safe bulk builder") {
    it("should return the bulk object when index operations exceeds the configured maxDocument count") {
      val builder = new ThreadSafeBulkBuilder(maxDocuments = 3, 1000)
      var bulkOp = builder.addAction(new Index.Builder("source1").build(), 10, forceBulkCreate = false)
      bulkOp shouldBe 'empty

      bulkOp = builder.addAction(new Index.Builder("source2").build(), 10, forceBulkCreate = false)
      bulkOp shouldBe 'empty

      bulkOp = builder.addAction(new Index.Builder("source3").build(), 10, forceBulkCreate = false)
      var bulkJson = bulkOp.get.getData(gson)
      bulkJson shouldEqual
        """{"index":{}}
                             |source1
                             |{"index":{}}
                             |source2
                             |{"index":{}}
                             |source3
                             |""".stripMargin

      builder.getDocsCount shouldBe 0
      builder.getTotalSizeInBytes shouldBe 0

      bulkOp = builder.addAction(new Index.Builder("source4").build(), 10, forceBulkCreate = true)
      bulkJson = bulkOp.get.
        getData(gson)
      bulkJson shouldEqual
        """{"index":{}}
          |source4
          |""".stripMargin
     }

    it("should return the bulk after size of the index operations exceed the configured threshold") {
      val builder = new ThreadSafeBulkBuilder(maxDocuments = 10, 100)
      var bulkOp = builder.addAction(new Index.Builder("source1").build(), 30, forceBulkCreate = false)
      bulkOp shouldBe 'empty

      bulkOp = builder.addAction(new Index.Builder("source2").build(), 30, forceBulkCreate = false)
      bulkOp shouldBe 'empty

      bulkOp = builder.addAction(new Index.Builder("source3").build(), 80, forceBulkCreate = false)
      val bulkJson = bulkOp.get.getData(gson)
      bulkJson shouldEqual """{"index":{}}
                             |source1
                             |{"index":{}}
                             |source2
                             |{"index":{}}
                             |source3
                             |""".stripMargin

      builder.getDocsCount shouldBe 0
      builder.getTotalSizeInBytes shouldBe 0
    }

    it("should return the bulk if forceBulkCreate attribute is set") {
      val builder = new ThreadSafeBulkBuilder(maxDocuments = 10, 1000)
      var bulkOp = builder.addAction(new Index.Builder("source1").build(), 30, forceBulkCreate = false)
      bulkOp shouldBe 'empty

      bulkOp = builder.addAction(new Index.Builder("source2").build(), 30, forceBulkCreate = false)
      bulkOp shouldBe 'empty

      bulkOp = builder.addAction(new Index.Builder("source3").build(), 80, forceBulkCreate = false)
      bulkOp shouldBe 'empty

      bulkOp = builder.addAction(new Index.Builder("source4").build(), 80, forceBulkCreate = true)
      val bulkJson = bulkOp.get.getData(gson)
      bulkJson shouldEqual """{"index":{}}
                             |source1
                             |{"index":{}}
                             |source2
                             |{"index":{}}
                             |source3
                             |{"index":{}}
                             |source4
                             |""".stripMargin

      builder.getDocsCount shouldBe 0
      builder.getTotalSizeInBytes shouldBe 0
    }
  }
} 

package biz.meetmatch.modules.util

import biz.meetmatch.modules.Module
import biz.meetmatch.util.Utils
import com.google.gson.Gson
import org.apache.spark.sql.SparkSession
import org.rogach.scallop.Scallop
import org.slf4j.LoggerFactory

import scalaj.http.Http

object WarmupZeppelin extends Module {

  override def execute(scallopts: Scallop)(implicit sparkSession: SparkSession): Unit = {
    val interpreterId = Utils.getConfig("zeppelin.interpreter.id")

    logger.info(s"Restarting the spark interpreter using url http://localhost:8080/api/interpreter/setting/restart/$interpreterId...")
    val interpreterResponse = Http(s"http://localhost:8080/api/interpreter/setting/restart/$interpreterId")
      .method("put")
      .timeout(connTimeoutMs = 10000, readTimeoutMs = 60000)
      .asParamMap
    logger.info("Response code: " + interpreterResponse.code)

    if (interpreterResponse.code == 200) {
      val bootstrapNotebookId = Utils.getConfig("zeppelin.bootstrapNotebookId.id")

      logger.info(s"Executing the Bootstrap notebook using url http://localhost:8080/api/notebook/job/$bootstrapNotebookId...")
      val notebookResponse = Http(s"http://localhost:8080/api/notebook/job/$bootstrapNotebookId")
        .postForm
        .timeout(connTimeoutMs = 10000, readTimeoutMs = 60000)
        .asParamMap
      logger.info("Response code: " + notebookResponse.code)

      if (notebookResponse.code == 200) {
        def stillRunningParagraphs(): Boolean = {
          logger.info("Checking if the Bootstrap notebook has finished...")
          val checkNotebookResponse = Http("http://localhost:8080/api/notebook/job/2BKFC4D5W")
            .timeout(connTimeoutMs = 10000, readTimeoutMs = 120000)
            .asString
          logger.info("Response code: " + checkNotebookResponse.code)
          val notebookJobResponse = new Gson().fromJson(checkNotebookResponse.body, classOf[GetNotebookJobResponse])
          notebookJobResponse.body.exists(_.status != "FINISHED")
        }

        while (stillRunningParagraphs()) {
          logger.info("Keep on polling...")
          Thread.sleep(5000)
        }
        logger.info("The Bootstrap notebook has finished.")
      }
    }
  }

  case class GetNotebookJobResponse(status: String, body: Array[ParagraphStatus])

  case class ParagraphStatus(id: String, started: String, finished: String, status: String)

  override protected val logger = LoggerFactory.getLogger(this.getClass)
} 

package ingraph.tests

import java.util

import com.google.gson.Gson
import ingraph.csv.EdgeMetaData
import ldbc.LdbcUpdateLoader

import scala.collection.JavaConverters._

object LdbcSnbTestCase  {
  private val gson = new Gson()
}

class LdbcSnbTestCase(val workload: String,
                      val sf: String,
                      val query: Int,
                      val csvDir: String
                     ) extends TestCase with CSVData {
  val updateQueryPrefix: String = "../queries/ldbc-snb-interactive/interactive-update-"
  val updateQueryPostfix: String = ".cypher"
  val csvPostfix = "_0_0.csv"

  override def name: String = f"${workload}-${query}%02d"

  override def querySpecification: String = {
    def convert(v: Any): String = {
      v match {
        case d: Double => f"$d%.0f"
        case s: String => "'" + s.toString + "'"
        case seq: util.ArrayList[Any] => "[" + seq.toArray.map(convert).mkString(",") + "]"
        case _ => v.toString
      }
    }

    val baseQuerySpecification: String = readToString(s"../queries/ldbc-snb-${workload}/${name}.cypher")
    val parameters: Map[String, String] = LdbcSnbTestCase.gson
      .fromJson(readToString(f"../graphs/ldbc-snb-bi/${query}%02d/parameters"), classOf[java.util.Map[String, Object]])
      .asScala
      .toMap
      .map { case (k, v) => (k, convert(v)) }
    parameters.foldLeft(baseQuerySpecification)((a, b) =>
      a.replaceAllLiterally("$" + b._1.toString, b._2.toString))
  }

  override def vertexCsvPaths: Map[String, List[String]] = {
    Map(
      "comment" -> List("Message", "Comment"),
      "forum" -> List("Forum"),
      "organisation" -> List("Organisation"),
      "person" -> List("Person"),
      "place" -> List("Place"),
      "post" -> List("Message", "Post"),
      "tagclass" -> List("TagClass"),
      "tag" -> List("Tag")
    ) map {
      case (file, labels) => (csvDir + file + csvPostfix) -> labels
    }
  }

  override def edgeCsvPaths: Map[String, EdgeMetaData] = {
    Map(
      "comment_hasCreator_person"      -> EdgeMetaData("Message",      "HAS_CREATOR",    "Person"),
      "comment_isLocatedIn_place"      -> EdgeMetaData("Message",      "IS_LOCATED_IN",  "Place"),
      "comment_replyOf_comment"        -> EdgeMetaData("Message",      "REPLY_OF",       "Message"),
      "comment_replyOf_post"           -> EdgeMetaData("Message",      "REPLY_OF",       "Message"),
      "forum_containerOf_post"         -> EdgeMetaData("Forum",        "CONTAINER_OF",   "Message"),
      "forum_hasMember_person"         -> EdgeMetaData("Forum",        "HAS_MEMBER",     "Person"),
      "forum_hasModerator_person"      -> EdgeMetaData("Forum",        "HAS_MODERATOR",  "Person"),
      "forum_hasTag_tag"               -> EdgeMetaData("Forum",        "HAS_TAG",        "Tag"),
      "person_hasInterest_tag"         -> EdgeMetaData("Person",       "HAS_INTEREST",   "Tag"),
      "person_isLocatedIn_place"       -> EdgeMetaData("Person",       "IS_LOCATED_IN",  "Place"),
      "person_knows_person"            -> EdgeMetaData("Person",       "KNOWS",          "Person"),
      "person_likes_comment"           -> EdgeMetaData("Person",       "LIKES",          "Message"),
      "person_likes_post"              -> EdgeMetaData("Person",       "LIKES",          "Message"),
      "place_isPartOf_place"           -> EdgeMetaData("Place",        "IS_PART_OF",     "Place"),
      "post_hasCreator_person"         -> EdgeMetaData("Message",      "HAS_CREATOR",    "Person"),
      "comment_hasTag_tag"             -> EdgeMetaData("Message",      "HAS_TAG",        "Tag"),
      "post_hasTag_tag"                -> EdgeMetaData("Message",      "HAS_TAG",        "Tag"),
      "post_isLocatedIn_place"         -> EdgeMetaData("Message",      "IS_LOCATED_IN",  "Place"),
      "tagclass_isSubclassOf_tagclass" -> EdgeMetaData("TagClass",     "IS_SUBCLASS_OF", "TagClass"),
      "tag_hasType_tagclass"           -> EdgeMetaData("Tag",          "HAS_TYPE",       "TagClass"),
      "organisation_isLocatedIn_place" -> EdgeMetaData("Organisation", "IS_LOCATED_IN",  "Place"),
      "person_studyAt_organisation"    -> EdgeMetaData("Person",       "STUDY_OF",       "Organisation"),
      "person_workAt_organisation"     -> EdgeMetaData("Person",       "WORK_AT",        "Organisation")
    ) map {
      case (file, labels) => (csvDir + file + csvPostfix) -> labels
    }
  }

  val loader = new LdbcUpdateLoader(csvDir, updateQueryPrefix, updateQueryPostfix)
  val updates = loader.generateQuerySpecifications()

}