org.apache.http.client.methods.HttpGet Scala Examples

package com.stratio.benchmark.generator.utils

import org.apache.http.HttpStatus
import org.apache.http.client.methods.{HttpDelete, HttpGet, HttpPost, HttpPut}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
import org.apache.log4j.Logger
import org.json4s.DefaultFormats
import org.json4s.native.JsonMethods._

import scala.io.Source

trait HttpUtil   {

  private val logger = Logger.getLogger(this.getClass)

  
  def createPolicy(policyContent: String, endpoint: String)(implicit defaultFormats: DefaultFormats): String = {

    val policyName = (parse(policyContent) \ "name").extract[String]

    // If the policy exists when it launches the benchmark, it should stop and delete it.
    getPolicyId(policyName, endpoint) match {
      case Some(id) =>
        stopPolicy(id, endpoint)
        deletePolicy(id, endpoint)
      case None => logger.debug(s"No policy with name $policyName exists in Sparta yet.")
    }

    val client = HttpClientBuilder.create().build()
    val post = new HttpPost(s"$endpoint/policyContext")
    post.setHeader("Content-type", "application/json")
    post.setEntity(new StringEntity(policyContent))
    val response = client.execute(post)

   if(response.getStatusLine.getStatusCode != HttpStatus.SC_OK)
     throw new IllegalStateException(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}")
   else {
     val entity = response.getEntity
     val policyId = (parse(EntityUtils.toString(entity)) \ "policyId").extract[String]
     policyId
   }
  }

  def getPolicyId(name: String, endpoint: String)(implicit defaultFormats: DefaultFormats): Option[String] = {
    val client = HttpClientBuilder.create().build()
    val get = new HttpGet(s"$endpoint/policy/findByName/$name")

    val response = client.execute(get)

    response.getStatusLine.getStatusCode match {
      case HttpStatus.SC_OK =>
        Option((parse(EntityUtils.toString(response.getEntity)) \ "id").extract[String])
      case _ => None
    }
  }

  def stopPolicy(id: String, endpoint: String): Unit = {
    val client = HttpClientBuilder.create().build()
    val put = new HttpPut(s"$endpoint/policyContext")
    put.setHeader("Content-Type", "application/json")
    val entity = new StringEntity(s"""{"id":"$id", "status":"Stopping"}""")
    put.setEntity(entity)
    val response = client.execute(put)

    if(response.getStatusLine.getStatusCode != HttpStatus.SC_CREATED) {
      logger.info(Source.fromInputStream(response.getEntity.getContent).mkString(""))
      logger.info(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}")
    }
  }

  def deletePolicy(id: String, endpoint: String): Unit = {
    val client = HttpClientBuilder.create().build()
    val delete = new HttpDelete(s"$endpoint/policy/$id")
    val response = client.execute(delete)

    if(response.getStatusLine.getStatusCode != HttpStatus.SC_OK)
      logger.info(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}")
  }
} 

package com.sumologic.sumobot.plugins.pagerduty
import com.sumologic.sumobot.plugins.HttpClientWithTimeOut
import net.liftweb.json._
import org.apache.http.client.methods.HttpGet

import scala.collection.mutable.ArrayBuffer

class PagerDutySchedulesManager(settings: PagerDutySettings) {
  private[this] val perPage = 100

  def getAllOnCalls: Seq[PagerDutyOnCall] = {
    val client = HttpClientWithTimeOut.client()
    try {
      var total = Integer.MAX_VALUE
      var page = 0
      var offset = 0
      val onCallsList = ArrayBuffer[PagerDutyOnCall]()
      while (page * perPage < total) {
        val url = s"${settings.url}/oncalls?offset=$offset&limit=$perPage&total=true"
        val getReq = new HttpGet(url)

        getReq.addHeader("Accept", "application/vnd.pagerduty+json;version=2")
        getReq.addHeader("Authorization", s"Token token=${settings.token}")

        val response = client.execute(getReq)
        val entity = response.getEntity


        if (entity != null) {
          val inputStream = entity.getContent
          val str = Some(scala.io.Source.fromInputStream(inputStream).mkString).getOrElse {
            return Seq()
          }

          val json = JsonParser.parse(str)

          try {
            val jsonOnCalls = json \ "oncalls"
            implicit val formats = DefaultFormats.withHints(ShortTypeHints(List(classOf[PagerDutyOnCallUser], classOf[PagerDutyEscalationPolicy])))
            total = (json \ "total").extract[Int]
            page += 1
            offset = page * perPage

            val oncalls: List[PagerDutyOnCall] = jsonOnCalls.children.map(_.extract[PagerDutyOnCall])
            onCallsList ++= oncalls
          } catch {
            case e: Exception =>
              println(s"Failed to parse $str")
              throw e
          }
        }
      }
      onCallsList.toList
    } finally {
      client.close()
    }
  }
} 

package io.github.shogowada.scala.jsonrpc.example.test.utils

import java.net.ServerSocket

import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClientBuilder

import scala.util.Try

trait BaseTarget {

  lazy val port = freePort()
  lazy val url = s"http://localhost:$port"
  lazy val jarLocation = System.getProperty("jarLocation")

  def healthCheckUrl = url

  def freePort(): Int = {
    val server = new ServerSocket(0)
    val localPort = server.getLocalPort
    server.close()
    localPort
  }

  private def startProcess(jarLocation: String, port: Int): Process = {
    new ProcessBuilder(
      "java", s"-Dport=$port", "-jar", jarLocation
    ).start()
  }

  private def waitUntilReady(): Unit = {
    Range(0, 10).toStream
        .map(_ => {
          Thread.sleep(1000)
          val client = HttpClientBuilder.create().build()
          val maybeCode = Try {
            val response = client.execute(new HttpGet(healthCheckUrl))
            val code = response.getStatusLine.getStatusCode
            response.close()
            code
          }.toOption
          client.close()
          maybeCode
        })
        .filter(code => code.contains(200))
        .head
  }

  val target = startProcess(jarLocation, port)

  Runtime.getRuntime.addShutdownHook(new Thread() {
    override def run(): Unit = {
      target.destroy()
    }
  })

  waitUntilReady()
} 

package works.weave.socks.aws.orders.dataaccess.web

import com.amazonaws.util.IOUtils
import com.fasterxml.jackson.core.`type`.TypeReference
import java.lang.reflect.ParameterizedType
import java.lang.reflect.Type
import java.net.URI
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClientBuilder
import org.slf4j.LoggerFactory
import works.weave.socks.aws.orders.ProjectDefaultJacksonMapper

object JSONHTTP {

  val Log = LoggerFactory.getLogger(getClass)

  val objectMapper = ProjectDefaultJacksonMapper.build() // after (_.bla)

  def get[T : Manifest : NotNothing](uri : URI) : T = {
    val client = HttpClientBuilder.create.build
    val get = new HttpGet(uri)
    get.addHeader("Accept", "application/json")
    val response = client.execute(get)
    val responseString = IOUtils.toString(response.getEntity.getContent)
    Log.info(s"Got status ${response.getStatusLine.getStatusCode}")
    require(response.getStatusLine.getStatusCode == 200)
    Log.info(s"Got response from URI $uri: $responseString")
    objectMapper.readValue(responseString, typeReference[T])
  }

  def typeReference[T : Manifest] = new TypeReference[T] {
    override def getType = typeFromManifest(manifest[T])
  }

  def typeFromManifest(m : Manifest[_]) : Type = {
    if (m.typeArguments.isEmpty) { m.runtimeClass }
    else new ParameterizedType {
      def getRawType = m.runtimeClass
      def getActualTypeArguments = m.typeArguments.map(typeFromManifest).toArray
      def getOwnerType = null
    }
  }

  // Trick to disable nothign for type param
  sealed trait NotNothing[-T]

  object NotNothing {
    implicit object notNothing extends NotNothing[Any]
    implicit object `The error is because the missing type parameter was resolved to Nothing` extends NotNothing[Nothing]
  }

} 

package com.gravity.goose.utils

import com.gravity.goose.text.{StringReplacement, HashUtils}
import java.net.{URI, MalformedURLException, URL}
import org.apache.http.client.methods.HttpGet


  def getCleanedUrl(urlToCrawl: String): Option[ParsingCandidate] = {

    val finalURL =
      if (urlToCrawl.contains("#!")) ESCAPED_FRAGMENT_REPLACEMENT.replaceAll(urlToCrawl) else urlToCrawl

    try {
      val url = new URL(finalURL)
      val linkhash = HashUtils.md5(finalURL)
      Some(ParsingCandidate(finalURL, linkhash, url))
    }
    catch {
      case e: MalformedURLException => {
        warn("{0} - is a malformed URL and cannot be processed", urlToCrawl)
        None
      }
      case unknown: Exception => {
        critical("Unable to process URL: {0} due to an unexpected exception:\n\tException Type: {1}\n\tException Message: {2}\n\tException Stack:\n{3}",
          urlToCrawl,
          unknown.getClass.getCanonicalName,
          unknown.getMessage,
          unknown.getStackTraceString)

        None
      }
    }
  }

  def tryToURL(url: String): Option[URL] = {
    val finalUrl = if (url.contains("#!")) {
      ESCAPED_FRAGMENT_REPLACEMENT.replaceAll(url)
    } else {
      url
    }

    try {
      Some(new URL(finalUrl))
    } catch {
      case _: Exception => None
    }
  }

  def tryToURI(url: String): Option[URI] = {
    val finalUrl = if (url.contains("#!")) {
      ESCAPED_FRAGMENT_REPLACEMENT.replaceAll(url)
    } else {
      url
    }

    try {
      Some(URI.create(finalUrl))
    } catch {
      case _: Exception => None
    }
  }

  def tryToHttpGet(url: String): Option[HttpGet] = {
    tryToURI(url) match {
      case Some(uri) => Some(new HttpGet(uri))
      case None => None
    }
  }
} 

package org.crowdcrawler.proxycrawler.checker

import java.net.URI
import java.nio.charset.StandardCharsets
import java.security.cert.X509Certificate

import org.apache.http.HttpHost
import org.apache.http.annotation.ThreadSafe
import org.apache.http.client.methods.HttpGet
import org.apache.http.conn.ssl.{NoopHostnameVerifier, SSLConnectionSocketFactory}
import org.apache.http.impl.client.HttpClients
import org.apache.http.ssl.{TrustStrategy, SSLContexts}
import org.apache.http.util.EntityUtils


@ThreadSafe
private[checker] object HttpsProxyChecker extends AbstractProxyChecker {
  // trust all certificates including self-signed certificates
  private[checker] val SSL_CONTEXT = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
    def isTrusted(chain: Array[X509Certificate], authType: String) = true
  }).build()
  private val CLIENT = {
    val connectionFactory = new SSLConnectionSocketFactory(SSL_CONTEXT, NoopHostnameVerifier.INSTANCE)
    HttpClients.custom().setSSLSocketFactory(connectionFactory).setMaxConnTotal(AbstractProxyChecker.MAX_CONN)
      .disableRedirectHandling().build()
  }
  private val TARGET_URL = new URI("https://www.google.com")


  def check(host: String, port: Int): (Int, Int) = {
    val request = new HttpGet(TARGET_URL)
    AbstractProxyChecker.configureRequest(request, Some(new HttpHost(host, port, "http")))

    val response = CLIENT.execute(request)

    val statusCode = response.getStatusLine.getStatusCode
    val html = EntityUtils.toString(response.getEntity, StandardCharsets.UTF_8)
    if (statusCode == 200 && html.contains("<title>Google</title>")) (statusCode, html.getBytes.length)
    else (statusCode, -1)
  }
} 

package org.crowdcrawler.proxycrawler.checker

import java.io.IOException

import org.apache.http.{HttpHeaders, HttpHost}
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.HttpGet
import org.crowdcrawler.proxycrawler.ProxyCrawler


private[checker] trait AbstractProxyChecker {
  
  @throws(classOf[IOException])
  def check(host: String, port: Int): (Int, Int)
}


private[checker] object AbstractProxyChecker {
  val TIMEOUT = 30000  // 30000 milliseconds
  val MAX_CONN = 100000
  val REQUEST_CONFIG = RequestConfig.custom.setConnectTimeout(TIMEOUT).setSocketTimeout(TIMEOUT)
    .setRedirectsEnabled(false).setRelativeRedirectsAllowed(false).setCircularRedirectsAllowed(false)
    .build()


  def configureRequest(request: HttpGet, proxy: Option[HttpHost] = None): Unit = {
    ProxyCrawler.DEFAULT_HEADERS.foreach { case (key, value) =>
      request.setHeader(key, value)
    }
    // disable keep-alive
    request.setHeader(HttpHeaders.CONNECTION, "close")
    val requestConfig = if (proxy.isDefined) {
      RequestConfig.copy(AbstractProxyChecker.REQUEST_CONFIG).setProxy(proxy.get).build()
    } else {
      REQUEST_CONFIG
    }
    request.setConfig(requestConfig)
  }
} 

package org.crowdcrawler.proxycrawler.checker

import java.net
import java.net.{InetSocketAddress, Socket, URI}
import java.nio.charset.StandardCharsets
import javax.net.ssl.{HostnameVerifier, SSLContext}

import org.apache.http.annotation.ThreadSafe
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.protocol.HttpClientContext
import org.apache.http.config.RegistryBuilder
import org.apache.http.conn.socket.{ConnectionSocketFactory, PlainConnectionSocketFactory}
import org.apache.http.conn.ssl.{NoopHostnameVerifier, SSLConnectionSocketFactory}
import org.apache.http.impl.client.HttpClients
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
import org.apache.http.protocol.HttpContext
import org.apache.http.util.EntityUtils


@ThreadSafe
private[checker] object SocksProxyChecker extends AbstractProxyChecker {
  private class MyHttpConnectionSocketFactory extends PlainConnectionSocketFactory {
    override def createSocket(context: HttpContext): Socket = {
      val socksAddress = context.getAttribute("socks.address").asInstanceOf[InetSocketAddress]
      val proxy = new net.Proxy(net.Proxy.Type.SOCKS, socksAddress)
      new Socket(proxy)
    }
  }

  private class MyHttpsConnectionSocketFactory(sslContext: SSLContext, verifier: HostnameVerifier)
    extends SSLConnectionSocketFactory(sslContext) {
    override def createSocket(context: HttpContext): Socket = {
      val socksAddress = context.getAttribute("socks.address").asInstanceOf[InetSocketAddress]
      val proxy = new net.Proxy(net.Proxy.Type.SOCKS, socksAddress)
      new Socket(proxy)
    }
  }

  private val CLIENT = {
    val reg = RegistryBuilder.create[ConnectionSocketFactory]()
      .register("http", new MyHttpConnectionSocketFactory())
      .register("https",
        new MyHttpsConnectionSocketFactory(HttpsProxyChecker.SSL_CONTEXT, NoopHostnameVerifier.INSTANCE))
      .build()
    val cm = new PoolingHttpClientConnectionManager(reg)
    cm.setMaxTotal(AbstractProxyChecker.MAX_CONN)
    HttpClients.custom().setConnectionManager(cm).disableRedirectHandling().build()
  }
  private val TARGET_URL = new URI("http://www.baidu.com")


  def check(host: String, port: Int): (Int, Int) = {
    val request = new HttpGet(TARGET_URL)
    AbstractProxyChecker.configureRequest(request)

    val httpContext = {
      val socksAddress = new InetSocketAddress(host, port)
      val context = HttpClientContext.create()
      context.setAttribute("socks.address", socksAddress)
      context
    }

    val response = CLIENT.execute(request, httpContext)

    val statusCode = response.getStatusLine.getStatusCode
    val html = EntityUtils.toString(response.getEntity, StandardCharsets.UTF_8)
    if (statusCode == 200 && html.contains("<title>百度一下")) (statusCode, html.getBytes.length) else (statusCode, -1)
  }
} 

package org.crowdcrawler.proxycrawler.checker

import java.net.URI
import java.nio.charset.StandardCharsets

import org.apache.http.annotation.ThreadSafe
import org.apache.http.HttpHost
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils


@ThreadSafe
private[checker] object HttpProxyChecker extends AbstractProxyChecker {
  private val CLIENT  = HttpClients.custom().setMaxConnTotal(AbstractProxyChecker.MAX_CONN)
    .disableRedirectHandling().build()
  private val TARGET_URL = new URI("http://www.baidu.com")


  def check(host: String, port: Int): (Int, Int) = {
    val request = new HttpGet(TARGET_URL)
    AbstractProxyChecker.configureRequest(request, Some(new HttpHost(host, port, "http")))

    val response = CLIENT.execute(request)

    val statusCode = response.getStatusLine.getStatusCode
    val html = EntityUtils.toString(response.getEntity, StandardCharsets.UTF_8)
    if (statusCode == 200 && html.contains("<title>百度一下")) (statusCode, html.getBytes.length) else (statusCode, -1)
  }
} 

package org.crowdcrawler.proxycrawler

import java.io.IOException
import java.net.URI
import java.security.cert.X509Certificate

import com.typesafe.scalalogging.Logger
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import org.apache.http.ssl.{TrustStrategy, SSLContexts}
import org.apache.http.conn.ssl.{NoopHostnameVerifier, SSLConnectionSocketFactory}
import org.apache.http.util.EntityUtils
import org.crowdcrawler.proxycrawler.crawler.plugins.AbstractPlugin

import org.apache.http.HttpHeaders
import org.slf4j.LoggerFactory

import scala.collection.immutable
import scala.collection.mutable


class ProxyCrawler(plugins: List[AbstractPlugin]) {
  *;q=0.8"),
    (HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, sdch"),
    (HttpHeaders.ACCEPT_LANGUAGE, "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4"),
    (HttpHeaders.CONNECTION, "keep-alive")
  )

  private val CLIENT = {
    // trust all certificates including self-signed certificates
    val sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
      def isTrusted(chain: Array[X509Certificate], authType: String) = true
    }).build()
    val connectionFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE)
    HttpClients.custom().setSSLSocketFactory(connectionFactory).build()
  }

  def apply(classNames: String*): ProxyCrawler = {
    val plugins = mutable.ListBuffer.empty[AbstractPlugin]
    for (className <- classNames) {
      val clazz = Class.forName("org.crowdcrawler.proxycrawler.crawler.plugins." + className)
      plugins += clazz.newInstance().asInstanceOf[AbstractPlugin]
    }
    new ProxyCrawler(plugins.toList)
  }

  private def createRequest(uri: URI, headers: immutable.Map[String, String]): HttpGet = {
    val request = new HttpGet(uri)
    for (header <- headers) {
      request.setHeader(header._1, header._2)
    }
    request
  }
} 

package fr.vsct.dt.maze.helpers

import com.typesafe.scalalogging.StrictLogging
import fr.vsct.dt.maze.core.Commands.expectThat
import fr.vsct.dt.maze.core.Predef._
import fr.vsct.dt.maze.core.{Predicate, Result}
import org.apache.http._
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.entity.{ContentType, StringEntity}
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.message.{BasicHttpResponse, BasicStatusLine}
import org.apache.http.protocol.HttpContext
import org.scalatest.FlatSpec

import scala.beans.BeanProperty


class HttpExecutionTest extends FlatSpec {

  
  class MockHttpClient(val response: String) extends CloseableHttpClient with StrictLogging {
    var init = false

    override def doExecute(target: HttpHost, request: HttpRequest, context: HttpContext): CloseableHttpResponse = {
      if (!init) throw new IllegalStateException("Client is not initialized")
      logger.info("Doing actual http call")
      val r = if(request.getRequestLine.getUri == "http://some-url.com") {
        val t = new BasicCloseableHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK"))
        t.setEntity(new StringEntity(response, "UTF-8"))
        t
      } else {
        val t = new BasicCloseableHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_BAD_REQUEST, "KO"))
        t.setEntity(new StringEntity("""{"status": "ko"}""", ContentType.APPLICATION_JSON))
        t
      }
      r
    }

    @Deprecated
    override def getConnectionManager = null

    @Deprecated
    override def getParams = null

    override def close(): Unit = {}

    class BasicCloseableHttpResponse(statusLine: StatusLine) extends BasicHttpResponse(statusLine) with CloseableHttpResponse {
      override def close(): Unit = {}
    }

  }

  "a http check" should "not do an effective call until apply is effectively called" in {

    Http.client = new MockHttpClient("Youppy !")

    val requestOk = new HttpGet("http://some-url.com")

    val check1: Predicate = Http.execute(requestOk).status is 200
    val check2: Predicate = Http.execute(requestOk).response is "Youppy !"

    val check3 = check1 || check2

    val check4 = !check3

    Http.client.asInstanceOf[MockHttpClient].init = true

    assert(check1.get() == Result.success)
    assert(check2.get() == Result.success)
    assert(check3.get() == Result.success)
    assert(check4.get() == Result.failure(s"Expected ${check3.label} to be false"))
    expectThat(Http.get("http://some-error-url.com").status is 400)
    expectThat(Http.get("http://some-url.com").isOk)
    expectThat(!Http.get("http://some-error-url.com").isOk)
    expectThat(Http.get("http://some-error-url.com").responseAs(classOf[Stupid]) is Stupid(status = "ko"))

  }

}

case class Stupid(@BeanProperty status: String) 

package cmwell.analytics.util

import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import org.apache.http.HttpResponse
import org.apache.http.client.methods.{HttpGet, HttpPost}
import org.apache.http.client.{ClientProtocolException, ResponseHandler}
import org.apache.http.entity.{ContentType, StringEntity}
import org.apache.http.impl.client.{CloseableHttpClient, HttpClients}
import org.apache.http.util.EntityUtils


object HttpUtil {

  private val mapper = new ObjectMapper()

  private val responseHandler = new ResponseHandler[String]() {

    override def handleResponse(response: HttpResponse): String = {
      val status = response.getStatusLine.getStatusCode
      if (status >= 200 && status < 300) {
        val entity = response.getEntity

        if (entity != null)
          EntityUtils.toString(entity)
        else
          throw new ClientProtocolException("Unexpected null response")
      }
      else {
        throw new ClientProtocolException("Unexpected response status: " + status)
      }
    }
  }

  def get(url: String, client: CloseableHttpClient): String = {
    val httpGet = new HttpGet(url)
    client.execute(httpGet, responseHandler)
  }

  def get(url: String): String = {

    val client =  HttpClients.createDefault
    try {
      get(url, client)
    }
    finally {
      client.close()
    }
  }

  def getJson(url: String): JsonNode = {
    val client =  HttpClients.createDefault
    try {
      getJson(url, client)
    }
    finally {
      client.close()
    }
  }

  def getJson(url: String, client: CloseableHttpClient): JsonNode = {
    mapper.readTree(get(url, client))
  }

  def getJson(url: String, content: String, contentType: String): JsonNode = {
    val httpPost = new HttpPost(url)
    val entity = new StringEntity(content, ContentType.create(contentType, "UTF-8"))
    httpPost.setEntity(entity)

    val client = HttpClients.createDefault
    try {
      mapper.readTree(client.execute(httpPost, responseHandler))
    }
    finally {
      client.close()
    }
  }
} 

package cmwell.tools.neptune.export

import java.net.{URL, URLDecoder, URLEncoder}
import java.time.Instant

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.http.util.EntityUtils
import org.slf4j.LoggerFactory

object CmWellConsumeHandler {

  protected lazy val logger = LoggerFactory.getLogger("cm_well_consumer")
  val maxRetry = 5

  private val sleepTimeout = 10000

  def bulkConsume(cluster: String, position: String, format: String, updateMode:Boolean, retryCount:Int= 0): CloseableHttpResponse = {
    val withMeta = if(updateMode) "&with-meta" else ""
    val url = "http://" + cluster + "/_bulk-consume?position=" + position + "&format=" + format + withMeta
    val client = new DefaultHttpClient
    client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
    val get = new HttpGet(url)
    logger.info("Going to bulk consume,url= " + url)
    val response = client.execute(get)
    val statusCode = response.getStatusLine.getStatusCode
    if (statusCode != 200 && statusCode != 204) {
      if(statusCode == 503) {
        logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
        Thread.sleep(sleepTimeout)
        bulkConsume(cluster, position, format, updateMode)
      }
      else{
        if (retryCount < maxRetry) {
          logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...,retry count=" + retryCount)
          Thread.sleep(sleepTimeout)
          bulkConsume(cluster, position, format, updateMode, retryCount + 1)
        } else {
          throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
        }
      }
    }
    response
  }

  def retrivePositionFromCreateConsumer(cluster: String, lengthHint: Int, qp: Option[String], updateMode:Boolean, automaticUpdateMode:Boolean, toolStartTime:Instant, retryCount:Int = 0): String = {
    val withDeletedParam = if(updateMode || automaticUpdateMode) "&with-deleted" else ""
    //initial mode
    val qpTillStartTime = if(!updateMode && !automaticUpdateMode)  URLEncoder.encode(",system.lastModified<") + toolStartTime.toString else ""
    //automatic update mode
    val qpAfterStartTime = if(!updateMode && automaticUpdateMode) URLEncoder.encode(",system.lastModified>>" )+ toolStartTime.toString else ""
    val createConsumerUrl = "http://" + cluster + "/?op=create-consumer&qp=-system.parent.parent_hierarchy:/meta/" + qp.getOrElse("") + qpTillStartTime + qpAfterStartTime + "&recursive&length-hint=" + lengthHint + withDeletedParam
    logger.info("create-consumer-url=" + createConsumerUrl)
    val get = new HttpGet(createConsumerUrl)
    val client = new DefaultHttpClient
    client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
    val response = client.execute(get)
    val res = response.getAllHeaders.find(_.getName == "X-CM-WELL-POSITION").map(_.getValue).getOrElse("")
    logger.info("create-Consumer http status=" + response.getStatusLine.getStatusCode)
    val statusCode = response.getStatusLine.getStatusCode
    if (statusCode != 200) {
      if(statusCode == 503){
        logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
        Thread.sleep(sleepTimeout)
        retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime)
      }else {
        if (retryCount < maxRetry) {
          logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry..., retry count=" + retryCount)
          Thread.sleep(sleepTimeout)
          retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime, retryCount+1)
        }
        else {
          throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
        }
      }
    }
    res
  }


} 

package utils

import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
import scala.collection.mutable


object LdContextResolver {
  private val ldContexts = mutable.Map[String, Map[String, Any]]()

  def resolveContext(ldContextLoc: String, ldContextAcc: mutable.Map[String, String]): Unit = synchronized {
    var ldContext:Map[String, Any] = Map[String,Any]()

    if (ldContexts.contains(ldContextLoc)) {
      Console.println(s"loading @context from cache: ${ldContextLoc}")
      ldContext = ldContexts(ldContextLoc)
    }
    else {
      Console.println(s"Resolving JSON-LD @context: ${ldContextLoc}")

      val getRequest = new HttpGet(ldContextLoc)

      // send the GET request
      val httpClient = HttpClientBuilder.create().build()
      val result = httpClient.execute(getRequest)

      if (result.getStatusLine.getStatusCode == 200) {
        val ldContextStr = EntityUtils.toString(result.getEntity, "UTF-8")
        ldContext = ParserUtil.parse(ldContextStr).asInstanceOf[Map[String, Any]]

        ldContexts += (ldContextLoc -> ldContext)
      }
    }

    val firstLevel = ldContext.getOrElse("@context", None)

    if (firstLevel == None) {
      Console.println(s"It seems ${ldContextLoc} does not contain a valid JSON-LD @context")
      return
    }

   if (firstLevel.isInstanceOf[List[Any]]) {
      val list = firstLevel.asInstanceOf[List[Any]]

      list.foreach(l => {
        if (l.isInstanceOf[String]) {
          resolveContext(l.asInstanceOf[String], ldContextAcc)
        }
        else if (l.isInstanceOf[Map[String, String]]) {
          ldContextAcc ++= l.asInstanceOf[Map[String, String]]
        }
      })
   }
   else if (firstLevel.isInstanceOf[Map[String,Any]]) {
     val contextKeys = firstLevel.asInstanceOf[Map[String,Any]]
     contextKeys.keySet.foreach(contextKey => {
       val contextValue = contextKeys(contextKey)

       if (contextValue.isInstanceOf[String]) {
         ldContextAcc += (contextKey -> contextValue.asInstanceOf[String])
       }
       // Supporting { "@type": "http://example.org/MyType", "@id": "http://example.org/id234" }
       else if (contextValue.isInstanceOf[Map[String,String]]) {
         val contextValueMap = contextValue.asInstanceOf[Map[String,String]]
         ldContextAcc += (contextKey -> contextValueMap("@id"))
       }
       else {
         Console.println(s"Cannot resolve @context: ${ldContextLoc}")
         return
       }
     })
   }
   else {
      Console.println(s"Cannot resolve @context: ${ldContextLoc}")
      return
    }
  }

} 

package org.apache.livy.test

import java.io.File
import java.net.URI
import java.util.concurrent.{TimeUnit, Future => JFuture}
import javax.servlet.http.HttpServletResponse

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.scalatest.BeforeAndAfterAll

import org.apache.http.client.methods.HttpGet

import org.apache.livy._
import org.apache.livy.client.common.HttpMessages._
import org.apache.livy.sessions.SessionKindModule
import org.apache.livy.test.framework.BaseIntegrationTestSuite
import org.apache.livy.test.jobs.spark2._

class Spark2JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logging {

  private var client: LivyClient = _
  private var sessionId: Int = _
  private val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .registerModule(new SessionKindModule())

  override def afterAll(): Unit = {
    super.afterAll()

    if (client != null) {
      client.stop(true)
    }

    livyClient.connectSession(sessionId).stop()
  }

  test("create a new session and upload test jar") {
    val prevSessionCount = sessionList().total
    val tempClient = createClient(livyEndpoint)

    try {
      // Figure out the session ID by poking at the REST endpoint. We should probably expose this
      // in the Java API.
      val list = sessionList()
      assert(list.total === prevSessionCount + 1)
      val tempSessionId = list.sessions(0).id

      livyClient.connectSession(tempSessionId).verifySessionIdle()
      waitFor(tempClient.uploadJar(new File(testLib)))

      client = tempClient
      sessionId = tempSessionId
    } finally {
      if (client == null) {
        try {
          if (tempClient != null) {
            tempClient.stop(true)
          }
        } catch {
          case e: Exception => warn("Error stopping client.", e)
        }
      }
    }
  }

  test("run spark2 job") {
    assume(client != null, "Client not active.")
    val result = waitFor(client.submit(new SparkSessionTest()))
    assert(result === 3)
  }

  test("run spark2 dataset job") {
    assume(client != null, "Client not active.")
    val result = waitFor(client.submit(new DatasetTest()))
    assert(result === 2)
  }

  private def waitFor[T](future: JFuture[T]): T = {
    future.get(60, TimeUnit.SECONDS)
  }

  private def sessionList(): SessionList = {
    val httpGet = new HttpGet(s"$livyEndpoint/sessions/")
    val r = livyClient.httpClient.execute(httpGet)
    val statusCode = r.getStatusLine().getStatusCode()
    val responseBody = r.getEntity().getContent
    val sessionList = mapper.readValue(responseBody, classOf[SessionList])
    r.close()

    assert(statusCode ==  HttpServletResponse.SC_OK)
    sessionList
  }

  private def createClient(uri: String): LivyClient = {
    new LivyClientBuilder().setURI(new URI(uri)).build()
  }
} 

package org.apress.prospark

import java.util.Timer
import java.util.TimerTask

import scala.reflect.ClassTag

import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.api.java.JavaDStream
import org.apache.spark.streaming.api.java.JavaDStream.fromDStream
import org.apache.spark.streaming.api.java.JavaStreamingContext
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver

class HttpInputDStream(
    @transient ssc_ : StreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends ReceiverInputDStream[String](ssc_) with Logging {

  def getReceiver(): Receiver[String] = {
    new HttpReceiver(storageLevel, url, interval)
  }
}

class HttpReceiver(
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends Receiver[String](storageLevel) with Logging {

  var httpClient: CloseableHttpClient = _
  var trigger: Timer = _

  def onStop() {
    httpClient.close()
    logInfo("Disconnected from Http Server")
  }

  def onStart() {
    httpClient = HttpClients.createDefault()
    trigger = new Timer()
    trigger.scheduleAtFixedRate(new TimerTask {
      def run() = doGet()
    }, 0, interval * 1000)

    logInfo("Http Receiver initiated")
  }

  def doGet() {
    logInfo("Fetching data from Http source")
    val response = httpClient.execute(new HttpGet(url))
    try {
      val content = EntityUtils.toString(response.getEntity())
      store(content)
    } catch {
      case e: Exception => restart("Error! Problems while connecting", e)
    } finally {
      response.close()
    }

  }

}

object HttpUtils {
  def createStream(
    ssc: StreamingContext,
    storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2,
    url: String,
    interval: Long): DStream[String] = {
    new HttpInputDStream(ssc, storageLevel, url, interval)
  }

  def createStream(
    jssc: JavaStreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long): JavaDStream[String] = {
    implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[String]]
    createStream(jssc.ssc, storageLevel, url, interval)
  }
} 

package org.apress.prospark

import java.util.Timer
import java.util.TimerTask

import scala.reflect.ClassTag

import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.api.java.JavaDStream
import org.apache.spark.streaming.api.java.JavaDStream.fromDStream
import org.apache.spark.streaming.api.java.JavaStreamingContext
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver

class HttpInputDStream(
    @transient ssc_ : StreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends ReceiverInputDStream[String](ssc_) with Logging {

  def getReceiver(): Receiver[String] = {
    new HttpReceiver(storageLevel, url, interval)
  }
}

class HttpReceiver(
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends Receiver[String](storageLevel) with Logging {

  var httpClient: CloseableHttpClient = _
  var trigger: Timer = _

  def onStop() {
    httpClient.close()
    logInfo("Disconnected from Http Server")
  }

  def onStart() {
    httpClient = HttpClients.createDefault()
    trigger = new Timer()
    trigger.scheduleAtFixedRate(new TimerTask {
      def run() = doGet()
    }, 0, interval * 1000)

    logInfo("Http Receiver initiated")
  }

  def doGet() {
    logInfo("Fetching data from Http source")
    val response = httpClient.execute(new HttpGet(url))
    try {
      val content = EntityUtils.toString(response.getEntity())
      store(content)
    } catch {
      case e: Exception => restart("Error! Problems while connecting", e)
    } finally {
      response.close()
    }

  }

}

object HttpUtils {
  def createStream(
    ssc: StreamingContext,
    storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2,
    url: String,
    interval: Long): DStream[String] = {
    new HttpInputDStream(ssc, storageLevel, url, interval)
  }

  def createStream(
    jssc: JavaStreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long): JavaDStream[String] = {
    implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[String]]
    createStream(jssc.ssc, storageLevel, url, interval)
  }
} 

package cn.piflow.bundle.http

import java.io.{BufferedReader, InputStreamReader, PrintWriter}
import java.net.{HttpURLConnection, InetAddress, URL, URLConnection}

import cn.piflow.Runner
import cn.piflow.conf.bean.FlowBean
import cn.piflow.conf.util.{FileUtil, OptionUtil}
import cn.piflow.util.{PropertyUtil, ServerIpUtil}
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.sql.SparkSession
import org.h2.tools.Server
import org.junit.Test

import scala.util.parsing.json.JSON

class GetUrlTest {

  @Test
  def testFlow(): Unit ={

    //parse flow json
    val file = "src/main/resources/flow/http/getUrl.json"
    val flowJsonStr = FileUtil.fileReader(file)
    val map = OptionUtil.getAny(JSON.parseFull(flowJsonStr)).asInstanceOf[Map[String, Any]]
    println(map)

    //create flow
    val flowBean = FlowBean(map)
    val flow = flowBean.constructFlow()


    val ip = InetAddress.getLocalHost.getHostAddress
    cn.piflow.util.FileUtil.writeFile("server.ip=" + ip, ServerIpUtil.getServerIpFile())
    val h2Server = Server.createTcpServer("-tcp", "-tcpAllowOthers", "-tcpPort","50001").start()
    //execute flow
    val spark = SparkSession.builder()
      .master("local[12]")
      .appName("hive")
      .config("spark.driver.memory", "4g")
      .config("spark.executor.memory", "8g")
      .config("spark.cores.max", "8")
      .config("hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
      .enableHiveSupport()
      .getOrCreate()

    val process = Runner.create()
      .bind(classOf[SparkSession].getName, spark)
      .bind("checkpoint.path", "")
      .bind("debug.path","")
      .start(flow);

    process.awaitTermination();
    val pid = process.pid();
    println(pid + "!!!!!!!!!!!!!!!!!!!!!")
    spark.close();
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetFlowGroupInfo {

  def main(args: Array[String]): Unit = {

    val url = "http://10.0.85.83:8001/group/info?groupId=group_91198855-afbc-4726-856f-31326173a90f"
    val client = HttpClients.createDefault()
    val getFlowGroupInfoData:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowGroupInfoData)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("Code is " + str)
  }

} 

package cn.piflow.api

import org.apache.commons.httpclient.URI
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost}
import org.apache.http.impl.client.HttpClients
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import org.omg.CORBA.NameValuePair

object HTTPClientGetFlowInfo {

  def main(args: Array[String]): Unit = {

    val url = "http://10.0.86.98:8001/flow/info?appID=application_1562293222869_0420"
    val client = HttpClients.createDefault()
    val getFlowDebugData:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowDebugData)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("Code is " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetGroups {

  def main(args: Array[String]): Unit = {

    val url = "http://10.0.86.98:8001/stop/groups"
    val client = HttpClients.createDefault()
    val getGroups:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getGroups)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("Groups is: " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetFlowCheckpoints {

  def main(args: Array[String]): Unit = {

    val url = "http://10.0.86.98:8001/flow/checkpoints?appID=process_9e291e46-fe25-4e7c-943d-0ff85dddb22d_1"
    val client = HttpClients.createDefault()
    val getFlowInfo:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowInfo)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("Flow checkpoint is " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetStops {
  def main(args: Array[String]): Unit = {

    //val url = "http://10.0.86.98:8002/stop/listWithGroup"
    val url = "http://10.0.86.98:8001/stop/list"
    val client = HttpClients.createDefault()
    val getGroups:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getGroups)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("Groups is: " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetFlowGroupProcess {

  def main(args: Array[String]): Unit = {

    val url = "http://10.0.85.83:8001/group/progress?groupId=group_527ff279-a278-41b0-a658-7595a576fbb9"
    val client = HttpClients.createDefault()
    val getFlowGroupProgressData:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowGroupProgressData)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("Code is " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetStopInfo {

  def main(args: Array[String]): Unit = {

    val url = "http://10.0.85.83:8001/stop/info?bundle=cn.piflow.bundle.csv.CsvParser"
    val client = HttpClients.createDefault()
    val getFlowInfo:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowInfo)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("result : " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.{HttpClientBuilder, HttpClients}
import org.apache.http.util.EntityUtils

object HTTPClientGetScheduleInfo {

  def main(args: Array[String]): Unit = {
    val url = "http://10.0.85.83:8001/schedule/info?scheduleId=schedule_9339f584-4ec5-4e12-a51d-4214182ff63a"
    val client = HttpClients.createDefault()
    val getFlowDebugData:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowDebugData)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("Code is " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetAllPlugin {

  def main(args: Array[String]): Unit = {

    val url = "http://10.0.85.83:8001/plugin/info"
    val client = HttpClients.createDefault()
    val getFlowInfo:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowInfo)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println("result : " + str)
  }

} 

package cn.piflow.api

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HTTPClientGetFlowDebugData {

  def main(args: Array[String]): Unit = {
    //
    val url = "http://10.0.86.191:8002/flow/debugData?appID=application_1562293222869_0090&stopName=Fork&port=out3"
    val client = HttpClients.createDefault()
    val getFlowInfo:HttpGet = new HttpGet(url)

    val response:CloseableHttpResponse = client.execute(getFlowInfo)
    val entity = response.getEntity
    val str = EntityUtils.toString(entity,"UTF-8")
    println( str)
  }
}