org.apache.http.impl.client.CloseableHttpClient Scala Examples

package org.apress.prospark

import java.util.Timer
import java.util.TimerTask

import scala.reflect.ClassTag

import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.api.java.JavaDStream
import org.apache.spark.streaming.api.java.JavaDStream.fromDStream
import org.apache.spark.streaming.api.java.JavaStreamingContext
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver

class HttpInputDStream(
    @transient ssc_ : StreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends ReceiverInputDStream[String](ssc_) with Logging {

  def getReceiver(): Receiver[String] = {
    new HttpReceiver(storageLevel, url, interval)
  }
}

class HttpReceiver(
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends Receiver[String](storageLevel) with Logging {

  var httpClient: CloseableHttpClient = _
  var trigger: Timer = _

  def onStop() {
    httpClient.close()
    logInfo("Disconnected from Http Server")
  }

  def onStart() {
    httpClient = HttpClients.createDefault()
    trigger = new Timer()
    trigger.scheduleAtFixedRate(new TimerTask {
      def run() = doGet()
    }, 0, interval * 1000)

    logInfo("Http Receiver initiated")
  }

  def doGet() {
    logInfo("Fetching data from Http source")
    val response = httpClient.execute(new HttpGet(url))
    try {
      val content = EntityUtils.toString(response.getEntity())
      store(content)
    } catch {
      case e: Exception => restart("Error! Problems while connecting", e)
    } finally {
      response.close()
    }

  }

}

object HttpUtils {
  def createStream(
    ssc: StreamingContext,
    storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2,
    url: String,
    interval: Long): DStream[String] = {
    new HttpInputDStream(ssc, storageLevel, url, interval)
  }

  def createStream(
    jssc: JavaStreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long): JavaDStream[String] = {
    implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[String]]
    createStream(jssc.ssc, storageLevel, url, interval)
  }
} 

package org.apress.prospark

import java.util.Timer
import java.util.TimerTask

import scala.reflect.ClassTag

import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.api.java.JavaDStream
import org.apache.spark.streaming.api.java.JavaDStream.fromDStream
import org.apache.spark.streaming.api.java.JavaStreamingContext
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver

class HttpInputDStream(
    @transient ssc_ : StreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends ReceiverInputDStream[String](ssc_) with Logging {

  def getReceiver(): Receiver[String] = {
    new HttpReceiver(storageLevel, url, interval)
  }
}

class HttpReceiver(
    storageLevel: StorageLevel,
    url: String,
    interval: Long) extends Receiver[String](storageLevel) with Logging {

  var httpClient: CloseableHttpClient = _
  var trigger: Timer = _

  def onStop() {
    httpClient.close()
    logInfo("Disconnected from Http Server")
  }

  def onStart() {
    httpClient = HttpClients.createDefault()
    trigger = new Timer()
    trigger.scheduleAtFixedRate(new TimerTask {
      def run() = doGet()
    }, 0, interval * 1000)

    logInfo("Http Receiver initiated")
  }

  def doGet() {
    logInfo("Fetching data from Http source")
    val response = httpClient.execute(new HttpGet(url))
    try {
      val content = EntityUtils.toString(response.getEntity())
      store(content)
    } catch {
      case e: Exception => restart("Error! Problems while connecting", e)
    } finally {
      response.close()
    }

  }

}

object HttpUtils {
  def createStream(
    ssc: StreamingContext,
    storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2,
    url: String,
    interval: Long): DStream[String] = {
    new HttpInputDStream(ssc, storageLevel, url, interval)
  }

  def createStream(
    jssc: JavaStreamingContext,
    storageLevel: StorageLevel,
    url: String,
    interval: Long): JavaDStream[String] = {
    implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[String]]
    createStream(jssc.ssc, storageLevel, url, interval)
  }
} 

package com.microsoft.azure.powerbi.clients

import com.microsoft.azure.powerbi.common._
import com.microsoft.azure.powerbi.exceptions._
import com.microsoft.azure.powerbi.models._
import org.apache.http.client.methods._
import org.apache.http.impl.client.CloseableHttpClient
import org.json4s.native.Serialization
import org.json4s.native.Serialization._
import org.json4s.ShortTypeHints

object PowerBIReportClient {

  def get(dashboardId: String,
          authenticationToken: String,
          groupId: String = null): PowerBIReportDetailsList = {

    implicit val formats = Serialization.formats(
      ShortTypeHints(
        List()
      )
    )

    var getRequestURL: String = null

    if(groupId == null || groupId.trim.isEmpty) {
      getRequestURL = PowerBIURLs.ReportsBeta
    } else {
      getRequestURL = PowerBIURLs.GroupsBeta + f"/$groupId/reports"
    }

    val getRequest: HttpGet = new HttpGet(getRequestURL)

    getRequest.addHeader("Authorization", f"Bearer $authenticationToken")

    val httpClient: CloseableHttpClient = HttpClientUtils.getCustomHttpClient

    var responseContent: String = null
    var statusCode: Int = -1
    var exceptionMessage: String = null

    try {
      val httpResponse = httpClient.execute(getRequest)
      statusCode = httpResponse.getStatusLine.getStatusCode

      val responseEntity = httpResponse.getEntity

      if (responseEntity != null) {

        val inputStream = responseEntity.getContent
        responseContent = scala.io.Source.fromInputStream(inputStream).getLines.mkString
        inputStream.close()
      }
    }
    catch {
      case e: Exception => exceptionMessage = e.getMessage
    }
    finally {
      httpClient.close()
    }

    if (statusCode == 200) {
      return read[PowerBIReportDetailsList](responseContent)
    }

    throw PowerBIClientException(statusCode, responseContent, exceptionMessage)
  }
} 

package com.microsoft.azure.powerbi.clients

import com.microsoft.azure.powerbi.common._
import com.microsoft.azure.powerbi.exceptions._
import com.microsoft.azure.powerbi.models._
import org.apache.http.client.methods._
import org.apache.http.impl.client.CloseableHttpClient
import org.json4s.ShortTypeHints
import org.json4s.native.Serialization
import org.json4s.native.Serialization._

object PowerBIGroupClient {

  def get(datasetId: String, authenticationToken: String): PowerBIGroupDetailsList = {

    implicit val formats = Serialization.formats(
      ShortTypeHints(
        List()
      )
    )

    val getRequest: HttpGet = new HttpGet(PowerBIURLs.Groups)

    getRequest.addHeader("Authorization", f"Bearer $authenticationToken")

    val httpClient: CloseableHttpClient = HttpClientUtils.getCustomHttpClient

    var responseContent: String = null
    var statusCode: Int = -1
    var exceptionMessage: String = null

    try {

      val httpResponse = httpClient.execute(getRequest)
      statusCode = httpResponse.getStatusLine.getStatusCode

      val responseEntity = httpResponse.getEntity

      if (responseEntity != null) {
        val inputStream = responseEntity.getContent
        responseContent = scala.io.Source.fromInputStream(inputStream).getLines.mkString
        inputStream.close()
      }
    }
    catch {
      case e: Exception => exceptionMessage = e.getMessage
    }
    finally {
      httpClient.close()
    }

    if (statusCode == 200) {
      return read[PowerBIGroupDetailsList](responseContent)
    }

    throw PowerBIClientException(statusCode, responseContent, exceptionMessage)
  }
} 

package com.microsoft.azure.powerbi.clients

import com.microsoft.azure.powerbi.common._
import com.microsoft.azure.powerbi.exceptions._
import com.microsoft.azure.powerbi.models._
import org.apache.http.client.methods._
import org.apache.http.impl.client.CloseableHttpClient
import org.json4s.native.Serialization
import org.json4s.native.Serialization._
import org.json4s.ShortTypeHints

object PowerBIDashboardClient {

  def get(authenticationToken: String): PowerBIDashboardDetailsList = {
    get(authenticationToken, null)
  }

  def get(authenticationToken: String, groupId: String): PowerBIDashboardDetailsList = {

    implicit val formats = Serialization.formats(
      ShortTypeHints(
        List()
      )
    )

    var getRequestURL: String = null

    if (groupId == null || groupId.trim.isEmpty) {
      getRequestURL = PowerBIURLs.DashboardsBeta
    } else {
      getRequestURL = PowerBIURLs.GroupsBeta + f"/$groupId/dashboards"
    }

    val getRequest: HttpGet = new HttpGet(getRequestURL)

    getRequest.addHeader("Authorization", f"Bearer $authenticationToken")

    val httpClient: CloseableHttpClient = HttpClientUtils.getCustomHttpClient

    var responseContent: String = null
    var statusCode: Int = -1
    var exceptionMessage: String = null

    try {
      val httpResponse = httpClient.execute(getRequest)

      statusCode = httpResponse.getStatusLine.getStatusCode

      val responseEntity = httpResponse.getEntity

      if (responseEntity != null) {
        val inputStream = responseEntity.getContent
        responseContent = scala.io.Source.fromInputStream(inputStream).getLines.mkString
        inputStream.close()
      }
    }
    catch {
      case e: Exception => exceptionMessage = e.getMessage
    }
    finally {
      httpClient.close()
    }

    if (statusCode == 200) {
      return read[PowerBIDashboardDetailsList](responseContent)
    }

    throw PowerBIClientException(statusCode, responseContent, exceptionMessage)
  }
} 

package com.microsoft.azure.powerbi.common

import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.{HttpClients, CloseableHttpClient}

object HttpClientUtils {

  def getCustomHttpClient: CloseableHttpClient = {

     val customRequestConfig: RequestConfig = RequestConfig.custom()
      .setSocketTimeout(PowerBIClientConstants.sockectTimeoutInSeconds * 1000)
      .setConnectTimeout(PowerBIClientConstants.connectionTimeoutInSeconds * 1000)
      .setConnectionRequestTimeout(PowerBIClientConstants.connectionRequestTimeoutInSeconds * 1000)
      .build()

    val customHttpClient: CloseableHttpClient = HttpClients.custom()
      .setDefaultRequestConfig(customRequestConfig).build()

    customHttpClient
  }
} 

// Copyright (C) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in project root for information.

package com.microsoft.ml.spark.cognitive

import org.apache.commons.io.IOUtils
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods._
import org.apache.http.impl.client.{CloseableHttpClient, HttpClientBuilder}
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager

import scala.concurrent.blocking
import scala.util.Try

object RESTHelpers {
  lazy val RequestTimeout = 60000

  lazy val RequestConfigVal: RequestConfig = RequestConfig.custom()
    .setConnectTimeout(RequestTimeout)
    .setConnectionRequestTimeout(RequestTimeout)
    .setSocketTimeout(RequestTimeout)
    .build()

  lazy val ConnectionManager = {
    val cm = new PoolingHttpClientConnectionManager()
    cm.setDefaultMaxPerRoute(Int.MaxValue)
    cm.setMaxTotal(Int.MaxValue)
    cm
  }

  lazy val Client: CloseableHttpClient = HttpClientBuilder
    .create().setConnectionManager(ConnectionManager)
    .setDefaultRequestConfig(RequestConfigVal).build()

  def retry[T](backoffs: List[Int], f: () => T): T = {
    try {
      f()
    } catch {
      case t: Throwable =>
        val waitTime = backoffs.headOption.getOrElse(throw t)
        println(s"Caught error: $t with message ${t.getMessage}, waiting for $waitTime")
        blocking {Thread.sleep(waitTime.toLong)}
        retry(backoffs.tail, f)
    }
  }

  //TODO use this elsewhere
  def safeSend(request: HttpRequestBase,
               backoffs: List[Int] = List(100, 500, 1000),
               expectedCodes: Set[Int] = Set(),
               close: Boolean = true): CloseableHttpResponse = {

    retry(List(100, 500, 1000), { () =>
      val response = Client.execute(request)
      try {
        if (response.getStatusLine.getStatusCode.toString.startsWith("2") ||
          expectedCodes(response.getStatusLine.getStatusCode)
        ) {
          response
        } else {
          val requestBodyOpt = Try(request match {
            case er: HttpEntityEnclosingRequestBase => IOUtils.toString(er.getEntity.getContent)
            case _ => ""
          }).get

          val responseBodyOpt = Try(IOUtils.toString(response.getEntity.getContent)).getOrElse("")

          throw new RuntimeException(
            s"Failed: " +
              s"\n\t response: $response " +
              s"\n\t requestUrl: ${request.getURI}" +
              s"\n\t requestBody: $requestBodyOpt" +
              s"\n\t responseBody: $responseBodyOpt")
        }
      } catch {
        case e: Exception =>
          response.close()
          throw e
      } finally {
        if (close) {
          response.close()
        }
      }
    })
  }

} 

package cmwell.analytics.util

import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import org.apache.http.HttpResponse
import org.apache.http.client.methods.{HttpGet, HttpPost}
import org.apache.http.client.{ClientProtocolException, ResponseHandler}
import org.apache.http.entity.{ContentType, StringEntity}
import org.apache.http.impl.client.{CloseableHttpClient, HttpClients}
import org.apache.http.util.EntityUtils


object HttpUtil {

  private val mapper = new ObjectMapper()

  private val responseHandler = new ResponseHandler[String]() {

    override def handleResponse(response: HttpResponse): String = {
      val status = response.getStatusLine.getStatusCode
      if (status >= 200 && status < 300) {
        val entity = response.getEntity

        if (entity != null)
          EntityUtils.toString(entity)
        else
          throw new ClientProtocolException("Unexpected null response")
      }
      else {
        throw new ClientProtocolException("Unexpected response status: " + status)
      }
    }
  }

  def get(url: String, client: CloseableHttpClient): String = {
    val httpGet = new HttpGet(url)
    client.execute(httpGet, responseHandler)
  }

  def get(url: String): String = {

    val client =  HttpClients.createDefault
    try {
      get(url, client)
    }
    finally {
      client.close()
    }
  }

  def getJson(url: String): JsonNode = {
    val client =  HttpClients.createDefault
    try {
      getJson(url, client)
    }
    finally {
      client.close()
    }
  }

  def getJson(url: String, client: CloseableHttpClient): JsonNode = {
    mapper.readTree(get(url, client))
  }

  def getJson(url: String, content: String, contentType: String): JsonNode = {
    val httpPost = new HttpPost(url)
    val entity = new StringEntity(content, ContentType.create(contentType, "UTF-8"))
    httpPost.setEntity(entity)

    val client = HttpClients.createDefault
    try {
      mapper.readTree(client.execute(httpPost, responseHandler))
    }
    finally {
      client.close()
    }
  }
} 

package fr.vsct.dt.maze.helpers

import com.typesafe.scalalogging.StrictLogging
import fr.vsct.dt.maze.core.Commands.expectThat
import fr.vsct.dt.maze.core.Predef._
import fr.vsct.dt.maze.core.{Predicate, Result}
import org.apache.http._
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.entity.{ContentType, StringEntity}
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.message.{BasicHttpResponse, BasicStatusLine}
import org.apache.http.protocol.HttpContext
import org.scalatest.FlatSpec

import scala.beans.BeanProperty


class HttpExecutionTest extends FlatSpec {

  
  class MockHttpClient(val response: String) extends CloseableHttpClient with StrictLogging {
    var init = false

    override def doExecute(target: HttpHost, request: HttpRequest, context: HttpContext): CloseableHttpResponse = {
      if (!init) throw new IllegalStateException("Client is not initialized")
      logger.info("Doing actual http call")
      val r = if(request.getRequestLine.getUri == "http://some-url.com") {
        val t = new BasicCloseableHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK"))
        t.setEntity(new StringEntity(response, "UTF-8"))
        t
      } else {
        val t = new BasicCloseableHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_BAD_REQUEST, "KO"))
        t.setEntity(new StringEntity("""{"status": "ko"}""", ContentType.APPLICATION_JSON))
        t
      }
      r
    }

    @Deprecated
    override def getConnectionManager = null

    @Deprecated
    override def getParams = null

    override def close(): Unit = {}

    class BasicCloseableHttpResponse(statusLine: StatusLine) extends BasicHttpResponse(statusLine) with CloseableHttpResponse {
      override def close(): Unit = {}
    }

  }

  "a http check" should "not do an effective call until apply is effectively called" in {

    Http.client = new MockHttpClient("Youppy !")

    val requestOk = new HttpGet("http://some-url.com")

    val check1: Predicate = Http.execute(requestOk).status is 200
    val check2: Predicate = Http.execute(requestOk).response is "Youppy !"

    val check3 = check1 || check2

    val check4 = !check3

    Http.client.asInstanceOf[MockHttpClient].init = true

    assert(check1.get() == Result.success)
    assert(check2.get() == Result.success)
    assert(check3.get() == Result.success)
    assert(check4.get() == Result.failure(s"Expected ${check3.label} to be false"))
    expectThat(Http.get("http://some-error-url.com").status is 400)
    expectThat(Http.get("http://some-url.com").isOk)
    expectThat(!Http.get("http://some-error-url.com").isOk)
    expectThat(Http.get("http://some-error-url.com").responseAs(classOf[Stupid]) is Stupid(status = "ko"))

  }

}

case class Stupid(@BeanProperty status: String)