scalaj.http.Http Scala Examples

package com.bot4s.telegram.clients

import java.net.Proxy
import java.nio.file.Files

import cats.instances.future._
import com.bot4s.telegram.api.RequestHandler
import com.bot4s.telegram.methods.{Request, JsonRequest, MultipartRequest, Response}
import com.bot4s.telegram.models.InputFile
import com.bot4s.telegram.marshalling
import io.circe.parser.parse
import io.circe.{Decoder, Encoder}
import scalaj.http.{Http, MultiPart}
import slogging.StrictLogging

import scala.concurrent.{ExecutionContext, Future, blocking}


class ScalajHttpClient(token: String, proxy: Proxy = Proxy.NO_PROXY, telegramHost: String = "api.telegram.org")
                      (implicit ec: ExecutionContext) extends RequestHandler[Future] with StrictLogging {

  val connectionTimeoutMs = 10000
  val readTimeoutMs = 50000

  private val apiBaseUrl = s"https://$telegramHost/bot$token/"

  def sendRequest[R, T <: Request[_]](request: T)(implicit encT: Encoder[T], decR: Decoder[R]): Future[R] = {
    val url = apiBaseUrl + request.methodName

    val scalajRequest = request match {
      case r: JsonRequest[_] =>
        Http(url)
          .postData(marshalling.toJson(request))
          .header("Content-Type", "application/json")

      case r: MultipartRequest[_] =>

        // InputFile.FileIds are encoded as query params.
        val (fileIds, files) = r.getFiles.partition {
          case (key, _: InputFile.FileId) => true
          case _ => false
        }

        val parts = files.map {
          case (camelKey, inputFile) =>
            val key = marshalling.snakenize(camelKey)
            inputFile match {
              case InputFile.FileId(id) =>
                throw new RuntimeException("InputFile.FileId cannot must be encoded as a query param")

              case InputFile.Contents(filename, contents) =>
                MultiPart(key, filename, "application/octet-stream", contents)

              case InputFile.Path(path) =>
                MultiPart(key, path.getFileName.toString(),
                  "application/octet-stream",
                  Files.newInputStream(path),
                  Files.size(path),
                  _ => ())

              case other =>
                throw new RuntimeException(s"InputFile $other not supported")
            }
        }

        val fields = parse(marshalling.toJson(request)).fold(throw _, _.asObject.map {
          _.toMap.mapValues {
            json =>
              json.asString.getOrElse(marshalling.printer.pretty(json))
          }
        })

        val fileIdsParams = fileIds.map {
          case (key, inputFile: InputFile.FileId) =>
            marshalling.snakenize(key) -> inputFile.fileId
        }

        val params = fields.getOrElse(Map())

        Http(url).params(params ++ fileIdsParams).postMulti(parts: _*)
    }

    import marshalling.responseDecoder

    Future {
      blocking {
        scalajRequest
          .timeout(connectionTimeoutMs, readTimeoutMs)
          .proxy(proxy)
          .asString
      }
    } map {
      x =>
        if (x.isSuccess)
          marshalling.fromJson[Response[R]](x.body)
        else
          throw new RuntimeException(s"Error ${x.code} on request")
    } map (processApiResponse[R])
  }

} 

import play.api.libs.json.{JsSuccess, JsValue, Json}

import scalaj.http.{Base64, Http, HttpRequest}

object GithubClient {
  def apply(): GithubClient = GithubClient(Env.read("GITHUB_ACCESS_TOKEN"))
}

case class GithubClient(accessToken: String) {
  import JsonFormatting._

  val host       = "https://api.github.com"
  val authHeader = "Authorization" -> s"token $accessToken"

  def updateFile(owner: String, repo: String, filePath: String, newContent: String, branch: String): Unit = {
    getCurrentSha(owner, repo, filePath, branch) match {
      case Some(currentSha) =>
        updateContentsOfFile(owner, repo, filePath, currentSha, newContent, branch)
        println(s"Updated file $filePath in other repo successfully.")
      case None =>
        println(s"Branch $branch in other repo does not seem to exist. Won't update file.")
    }
  }

  def getCurrentSha(owner: String, repo: String, filePath: String, branch: String): Option[String] = {
    val request = baseRequest(urlPath(owner, repo, filePath, branch))
    request.asJson(200, 404).validate[GetContentResponse](getContentReads) match {
      case JsSuccess(parsed, _) => Some(parsed.sha)
      case _                    => None
    }
  }

  def updateContentsOfFile(owner: String, repo: String, filePath: String, sha: String, newContent: String, branch: String): JsValue = {
    val request = baseRequest(urlPath(owner, repo, filePath))
    val payload = UpdateContentsRequest(
      message = s"Updated by the SBT Task in the open source repo to: $newContent",
      content = Base64.encodeString(newContent),
      sha = sha,
      branch = branch
    )
    request.put(Json.toJson(payload)(updateContentsWrites).toString).asJson(200)
  }

  def urlPath(owner: String, repo: String, filePath: String, branch: String): String = urlPath(owner, repo, filePath) + s"?ref=$branch"
  def urlPath(owner: String, repo: String, filePath: String): String                 = s"/repos/$owner/$repo/contents/$filePath"
  def baseRequest(path: String)                                                      = Http(s"$host$path").headers(authHeader).header("content-type", "application/json")

  implicit class HttpRequestExtensions(httpRequest: HttpRequest) {
    def asJson(allowedStatusCodes: Int*): JsValue = {
      val response          = httpRequest.asString
      val isAllowedResponse = allowedStatusCodes.contains(response.code)
      require(isAllowedResponse, s"The request did not result in an expected status code. Allowed status are $allowedStatusCodes. The response was: $response")
      Json.parse(response.body)
    }
  }
}

object JsonFormatting {
  import play.api.libs.json._

  case class GetContentResponse(sha: String)
  case class UpdateContentsRequest(message: String, content: String, sha: String, branch: String)

  implicit val getContentReads      = Json.reads[GetContentResponse]
  implicit val updateContentsWrites = Json.writes[UpdateContentsRequest]
}

object Env {
  def read(name: String) = sys.env.getOrElse(name, sys.error(s"Env var $name must be set"))
} 

package org.jetbrains.sbtidea.tasks

import java.net.URLEncoder
import java.nio.file.Path
import java.util.regex.Pattern

import org.jetbrains.sbtidea.PluginLogger
import org.jetbrains.sbtidea.download.BuildInfo
import org.jetbrains.sbtidea.download.plugin.LocalPluginRegistry
import com.eclipsesource.json._
import scalaj.http.Http

import scala.collection.JavaConverters._

class SearchPluginId(ideaRoot: Path, buildInfo: BuildInfo, useBundled: Boolean = true, useRemote: Boolean = true) {

  private val REPO_QUERY = "https://plugins.jetbrains.com/api/search/plugins?search=%s&build=%s"

  // true if plugin was found in the remote repo
  def apply(query: String): Map[String, (String, Boolean)] = {
    val local  = if (useBundled) searchPluginIdLocal(query) else Map.empty
    val remote = if (useRemote) searchPluginIdRemote(query) else Map.empty
    remote ++ local
  }

  private def searchPluginIdLocal(query: String): Map[String, (String, Boolean)] = {
    val pattern = Pattern.compile(query)
    val registry = new LocalPluginRegistry(ideaRoot)
    val allDescriptors = registry.getAllDescriptors
    allDescriptors
        .filter(descriptor => pattern.matcher(descriptor.name).find() || pattern.matcher(descriptor.id).find())
        .map(descriptor => descriptor.id -> (descriptor.name, false))
        .toMap
  }

  // Apparently we can't use json4s when cross-compiling for sbt because there are BOTH no shared versions AND binary compatibility
  private def searchPluginIdRemote(query: String): Map[String, (String, Boolean)] = {
    try {
      val param = URLEncoder.encode(query, "UTF-8")
      val url = REPO_QUERY.format(param, s"${buildInfo.edition.edition}-${buildInfo.getActualIdeaBuild(ideaRoot)}")
      val data = Http(url).asString.body
      val json = Json.parse(data)
      val values = json.asArray().values().asScala.map(_.asObject())
      val names = values.map(_.getString("name", "") -> true)
      val ids = values.map(_.getString("xmlId", ""))
      ids.zip(names).toMap
    } catch {
      case ex: Throwable =>
        PluginLogger.warn(s"Failed to query IJ plugin repo: $ex")
        Map.empty
    }
  }
} 

package biz.meetmatch.modules.util

import biz.meetmatch.modules.Module
import biz.meetmatch.util.Utils
import com.google.gson.Gson
import org.apache.spark.sql.SparkSession
import org.rogach.scallop.Scallop
import org.slf4j.LoggerFactory

import scalaj.http.Http

object WarmupZeppelin extends Module {

  override def execute(scallopts: Scallop)(implicit sparkSession: SparkSession): Unit = {
    val interpreterId = Utils.getConfig("zeppelin.interpreter.id")

    logger.info(s"Restarting the spark interpreter using url http://localhost:8080/api/interpreter/setting/restart/$interpreterId...")
    val interpreterResponse = Http(s"http://localhost:8080/api/interpreter/setting/restart/$interpreterId")
      .method("put")
      .timeout(connTimeoutMs = 10000, readTimeoutMs = 60000)
      .asParamMap
    logger.info("Response code: " + interpreterResponse.code)

    if (interpreterResponse.code == 200) {
      val bootstrapNotebookId = Utils.getConfig("zeppelin.bootstrapNotebookId.id")

      logger.info(s"Executing the Bootstrap notebook using url http://localhost:8080/api/notebook/job/$bootstrapNotebookId...")
      val notebookResponse = Http(s"http://localhost:8080/api/notebook/job/$bootstrapNotebookId")
        .postForm
        .timeout(connTimeoutMs = 10000, readTimeoutMs = 60000)
        .asParamMap
      logger.info("Response code: " + notebookResponse.code)

      if (notebookResponse.code == 200) {
        def stillRunningParagraphs(): Boolean = {
          logger.info("Checking if the Bootstrap notebook has finished...")
          val checkNotebookResponse = Http("http://localhost:8080/api/notebook/job/2BKFC4D5W")
            .timeout(connTimeoutMs = 10000, readTimeoutMs = 120000)
            .asString
          logger.info("Response code: " + checkNotebookResponse.code)
          val notebookJobResponse = new Gson().fromJson(checkNotebookResponse.body, classOf[GetNotebookJobResponse])
          notebookJobResponse.body.exists(_.status != "FINISHED")
        }

        while (stillRunningParagraphs()) {
          logger.info("Keep on polling...")
          Thread.sleep(5000)
        }
        logger.info("The Bootstrap notebook has finished.")
      }
    }
  }

  case class GetNotebookJobResponse(status: String, body: Array[ParagraphStatus])

  case class ParagraphStatus(id: String, started: String, finished: String, status: String)

  override protected val logger = LoggerFactory.getLogger(this.getClass)
} 

package com.karasiq.highcharts.generator

import java.io.{BufferedWriter, FileOutputStream, OutputStreamWriter, PrintWriter}
import java.nio.file._
import java.nio.file.attribute.BasicFileAttributes

import scala.util.control.Exception
import scalaj.http.{Http, HttpOptions}

import com.karasiq.highcharts.generator.writers.{ScalaClassWriter, ScalaJsClassBuilder}

case class HighchartsApiDoc(library: String) {
  private val defaultPackage = System.getProperty(s"highcharts-generator.$library.package", s"com.$library")

  private def httpGet(url: String): List[ConfigurationObject] = {
    val page = Http.get(url)
      .header("User-Agent", "Mozilla/5.0 (X11; OpenBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36")
      .header("Accept", "application/json")
      .options(HttpOptions.connTimeout(10000), HttpOptions.readTimeout(10000))

    val json = page.asString
    ConfigurationObject.fromJson(json)
  }

  private def writeFiles(pkg: String, configs: List[ConfigurationObject], rootObject: Option[String] = None): Unit = {
    val header =
      s"""
          |package $pkg
          |
          |import scalajs.js, js.`|`
          |import com.highcharts.CleanJsObject
          |import com.highcharts.HighchartsUtils._
          |
          |""".stripMargin

    val outputDir = Paths.get(System.getProperty("highcharts-generator.output", "src/main/scala"), pkg.split("\\."):_*)
    Files.createDirectories(outputDir)

    // Remove all files
    Files.walkFileTree(outputDir, new SimpleFileVisitor[Path] {
      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
        Files.delete(file)
        FileVisitResult.CONTINUE
      }
    })

    val classes = new ScalaJsClassBuilder().parse(configs, rootObject)
    val classWriter = new ScalaClassWriter
    classes.foreach { scalaJsClass ⇒
      val file = outputDir.resolve(scalaJsClass.scalaName + ".scala")
      println(s"Writing $file...")
      val writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file.toFile, true), "UTF-8")))
      Exception.allCatch.andFinally(writer.close()) {
        if (Files.size(file) == 0) {
          writer.print(header)
        }
        classWriter.writeClass(scalaJsClass) { line ⇒
          writer.println(line)
        }
        writer.flush()
      }
    }
  }

  def writeConfigs(): Unit = {
    val configs = httpGet(s"https://api.highcharts.com/$library/dump.json")
    writeFiles(s"$defaultPackage.config", configs, Some(s"${library.capitalize}Config"))
  }

  def writeApis(): Unit = {
    val configs = httpGet(s"https://api.highcharts.com/$library/object/dump.json")
    writeFiles(s"$defaultPackage.api", configs)
  }

  def writeAll(): Unit = {
    // TODO: https://github.com/highcharts/highcharts/issues/7227
    writeConfigs()
    // writeApis() // TODO: 404
  }
}

object Main extends App {
  HighchartsApiDoc("highcharts").writeAll()
  HighchartsApiDoc("highstock").writeAll()
  HighchartsApiDoc("highmaps").writeAll()
} 

package com.giorgioinf.twtml.web

import org.json4s._
import org.json4s.native.Serialization
import org.json4s.native.Serialization.{write,read}
import scala.reflect.Manifest
import scalaj.http.{Http,HttpRequest}

class WebClient (val server:String) {

    implicit val formats = Serialization.formats(ShortTypeHints(List(classOf[Config], classOf[Stats])))

    def this() = this("http://localhost:8888")

    private def request(kind:String = ""):HttpRequest = {
      Http(server + "/api" + kind)
        .header("content-type", "application/json")
        .header("accept", "application/json")
    }

    private def post(data:TypeData) {
      val json = write(data)
      request().postData(json).asString
    }

    private def get[A:Manifest](kind:String):A = {
      val json = request(kind).asString.body
      read[A](json)
    }

    def config(id:String, host:String, viz:List[String]) = {
      post(Config(id, host, viz))
    }

    def stats(count:Long, batch:Long, mse:Long,
        realStddev:Long, predStddev:Long) = {
      post(Stats(count, batch, mse, realStddev, predStddev))
    }

    def config():Config = {
      get[Config]("/config")
    }

    def stats():Stats = {
      get[Stats]("/stats")
    }
}

object WebClient {
    def apply(host: String = ""): WebClient = {
    host match {
      case "" => new WebClient()
      case _ => new WebClient(host)
    }
  }
} 

package com.giorgioinf.twtml.web

import org.json4s._
import org.json4s.native.Serialization
import org.json4s.native.Serialization.{write,read}
import scala.reflect.Manifest
import scalaj.http.{Http,HttpRequest}

class WebClient (val server:String) {

    implicit val formats = Serialization.formats(ShortTypeHints(List(classOf[Config], classOf[Stats])))

    def this() = this("http://localhost:8888")

    private def request(kind:String = ""):HttpRequest = {
      Http(server + "/api" + kind)
        .header("content-type", "application/json")
        .header("accept", "application/json")
    }

    private def post(data:TypeData) {
      val json = write(data)
      request().postData(json).asString
    }

    private def get[A:Manifest](kind:String):A = {
      val json = request(kind).asString.body
      read[A](json)
    }

    def config(id:String, host:String, viz:List[String]) = {
      post(Config(id, host, viz))
    }

    def stats(count:Long, batch:Long, mse:Long,
        realStddev:Long, predStddev:Long) = {
      post(Stats(count, batch, mse, realStddev, predStddev))
    }

    def config():Config = {
      get[Config]("/config")
    }

    def stats():Stats = {
      get[Stats]("/stats")
    }
}

object WebClient {
    def apply(host: String = ""): WebClient = {
    host match {
      case "" => new WebClient()
      case _ => new WebClient(host)
    }
  }
} 

package com.raphtory.tests

import scalaj.http.Http
import scalaj.http.HttpRequest
import spray.json._
object EtherAPITest extends App {
  val nodeIP       = System.getenv().getOrDefault("ETHEREUM_IP_ADDRESS", "http://127.0.0.1").trim
  val nodePort     = System.getenv().getOrDefault("ETHEREUM_PORT", "8545").trim
  val baseRequest  = Http(nodeIP + ":" + nodePort).header("content-type", "application/json")
  var currentBlock = System.getenv().getOrDefault("ETHEREUM_START_BLOCK_INDEX", "9000000").trim.toInt
  var highestBlock = System.getenv().getOrDefault("ETHEREUM_MAXIMUM_BLOCK_INDEX", "999999999").trim.toInt
  def request(command: String, params: String = ""): HttpRequest =
    baseRequest.postData(s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""")
  var total = 0L
  for (i <- 9000000 to 9700000) {
    val transactionCountHex = request("eth_getBlockTransactionCountByNumber", "\"0x" + currentBlock.toHexString + "\"")
      .execute()
      .body
      .toString
      .parseJson
      .asJsObject
    val transactionCount = Integer.parseInt(transactionCountHex.fields("result").toString().drop(3).dropRight(1), 16)
    total += transactionCount
    if (i % 1000 == 0)
      println(total)
  }
  println(total)
//  for(i <- 0 until transactionCount) {
//    println(s"At index $i")
//    println(s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""")
//    println(request("eth_getTransactionByBlockNumberAndIndex",s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""").execute().body.toString.parseJson.asJsObject)
//  }
} 

package com.raphtory.examples.blockchain.spouts

import java.io.File
import java.io.PrintWriter

import com.raphtory.core.components.Spout.SpoutTrait
import com.raphtory.examples.blockchain.BitcoinTransaction
import scalaj.http.Http
import scalaj.http.HttpRequest
import spray.json._

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.sys.process._

class BitcoinSpout extends SpoutTrait {

  var blockcount    = 1
  val rpcuser       = System.getenv().getOrDefault("BITCOIN_USERNAME", "").trim
  val rpcpassword   = System.getenv().getOrDefault("BITCOIN_PASSWORD", "").trim
  val serverAddress = System.getenv().getOrDefault("BITCOIN_NODE", "").trim
  val id            = "scala-jsonrpc"
  val baseRequest   = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain")

  //************* MESSAGE HANDLING BLOCK
  override def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock")
    case "parseBlock" =>
      try {
        getTransactions()
        blockcount += 1
        AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock")
      } catch {
        case e: java.net.SocketTimeoutException => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock")
      }
    case _ => println("message not recognized!")
  }

  def outputScript() = {
    val pw = new PrintWriter(new File("bitcoin.sh"))
    pw.write("""curl --user $1:$2 --data-binary $3 -H 'content-type: text/plain;' $4""")
    pw.close
    "chmod 777 bitcoin.sh" !
  }

  def curlRequest(command: String, params: String): String = {
    //val data = """{"jsonrpc":"1.0","id":"scala-jsonrpc","method":"getblockhash","params":[2]}"""
    val data = s"""{"jsonrpc":"1.0","id":"$id","method":"$command","params":[$params]}"""
    s"bash bitcoin.sh $rpcuser $rpcpassword $data $serverAddress" !!
  }

  def request(command: String, params: String = ""): HttpRequest =
    baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""")

  def getTransactions(): Unit = {
    val re        = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject
    val blockID   = re.fields("result")
    val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject
    val result    = blockData.fields("result")
    val time      = result.asJsObject.fields("time")
    for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements)
      sendTuple(BitcoinTransaction(time, blockcount, blockID, transaction))
    //val time = transaction.asJsObject.fields("time")

  }

}
//def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") 

package com.raphtory.examples.blockchain.spouts

import com.raphtory.core.components.Spout.SpoutTrait
import com.raphtory.examples.blockchain.LitecoinTransaction
import scalaj.http.Http
import scalaj.http.HttpRequest
import spray.json._

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.language.postfixOps

class LitecoinSpout extends SpoutTrait {

  var blockcount    = 1
  val rpcuser       = System.getenv().getOrDefault("LITECOIN_USERNAME", "").trim
  val rpcpassword   = System.getenv().getOrDefault("LITECOIN_PASSWORD", "").trim
  val serverAddress = System.getenv().getOrDefault("LITECOIN_NODE", "").trim
  val id            = "scala-jsonrpc"
  val baseRequest   = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain")

  //************* MESSAGE HANDLING BLOCK
  override def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout   => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock")
    case "parseBlock" => running()
    case _            => println("message not recognized!")
  }

  def running(): Unit =
    try {
      getTransactions()
      blockcount += 1
      AllocateSpoutTask(Duration(1, NANOSECONDS), "parseBlock")
      if (blockcount % 100 == 0) println("Currently Calling for block $blockcount")
    } catch {
      case e: java.net.SocketTimeoutException     =>
      case e: spray.json.DeserializationException => AllocateSpoutTask(Duration(10, SECONDS), "parseBlock")
    }

  def getTransactions(): Unit = {
    val re        = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject
    val blockID   = re.fields("result")
    val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject
    val result    = blockData.fields("result")
    val time      = result.asJsObject.fields("time")
    for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements)
      sendTuple(LitecoinTransaction(time, blockcount, blockID, transaction))
    //val time = transaction.asJsObject.fields("time")
  }

  def request(command: String, params: String = ""): HttpRequest =
    baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""")

}
//def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") 

package com.raphtory.examples.blockchain.spouts

import java.net.InetAddress
import java.util.NoSuchElementException

import com.raphtory.core.components.Spout.SpoutTrait
import com.raphtory.core.utils.Utils
import com.raphtory.tests.EtherAPITest.baseRequest
import com.raphtory.tests.EtherAPITest.currentBlock
import com.raphtory.tests.EtherAPITest.request

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.duration.MILLISECONDS
import scala.concurrent.duration.NANOSECONDS
import scala.concurrent.duration.SECONDS
import scala.language.postfixOps
import scala.sys.process._
import scalaj.http.Http
import scalaj.http.HttpRequest
import spray.json._
import akka.http.scaladsl.unmarshalling.Unmarshal
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.stream.ActorMaterializer
import spray.json.DefaultJsonProtocol._

import scala.concurrent.duration._
import scala.concurrent.Await

case class EthResult(blockHash:Option[String],blockNumber:Option[String],from:Option[String],gas:Option[String],gasPrice:Option[String],hash:Option[String],input:Option[String],nonce:Option[String],r:Option[String],s:Option[String],to:Option[String],transactionIndex:Option[String],v:Option[String],value:Option[String])
case class EthTransaction(id:Option[String],jsonrpc:Option[String],result:EthResult)
class EthereumGethSpout extends SpoutTrait {
  var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt
  var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "10026447").trim.toInt
  val nodeIP       = System.getenv().getOrDefault("SPOUT_ETHEREUM_IP_ADDRESS", "127.0.0.1").trim
  val nodePort     = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "8545").trim
  val baseRequest  = requestBuilder()

  implicit val materializer = ActorMaterializer()
  implicit val EthFormat = jsonFormat14(EthResult)
  implicit val EthTransactionFormat = jsonFormat3(EthTransaction)
  if (nodeIP.matches(Utils.IPRegex))
    println(s"Connecting to Ethereum RPC \n Address:$nodeIP \n Port:$nodePort")
  else
    println(s"Connecting to Ethereum RPC \n Address:${hostname2Ip(nodeIP)} \n Port:$nodePort")

  override protected def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout  => pullNextBlock()
    case "nextBlock" => pullNextBlock()
  }

  def pullNextBlock(): Unit = {
    if (currentBlock > highestBlock)
      return
    try {
      log.debug(s"Trying block $currentBlock")
      val transactionCountHex = executeRequest("eth_getBlockTransactionCountByNumber", "\"0x" + currentBlock.toHexString + "\"")
      val transactionCount = Integer.parseInt(transactionCountHex.fields("result").toString().drop(3).dropRight(1), 16)
      if(transactionCount>0){
        var transactions = "["
        for (i <- 0 until transactionCount)
          transactions = transactions + batchRequestBuilder("eth_getTransactionByBlockNumberAndIndex",s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""")+","
        val trasnactionBlock = executeBatchRequest(transactions.dropRight(1)+"]")
        val transList = trasnactionBlock.parseJson.convertTo[List[EthTransaction]]
        transList.foreach(t => { //try needed to ignore contracts //todo include them
          try{sendTuple(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}")}
          catch {case e:NoSuchElementException =>}

        })

      }
      currentBlock += 1
      AllocateSpoutTask(Duration(1, NANOSECONDS), "nextBlock")
    } catch {
      case e: NumberFormatException => AllocateSpoutTask(Duration(1, SECONDS), "nextBlock")
      case e: Exception             => e.printStackTrace(); AllocateSpoutTask(Duration(1, SECONDS), "nextBlock")
    }
  }


  def batchRequestBuilder(command:String,params:String):String = s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}"""
  def executeBatchRequest(data: String) = requestBatch(data).execute().body.toString
  def requestBatch(data: String): HttpRequest = baseRequest.postData(data)
  def requestBuilder() =
    if (nodeIP.matches(Utils.IPRegex))
      Http("http://" + nodeIP + ":" + nodePort).header("content-type", "application/json")
    else
      Http("http://" + hostname2Ip(nodeIP) + ":" + nodePort).header("content-type", "application/json")
  def request(command: String, params: String = ""): HttpRequest =
    baseRequest.postData(s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""")
  def executeRequest(command: String, params: String = "") =
    request(command, params).execute().body.toString.parseJson.asJsObject

  def hostname2Ip(hostname: String): String = InetAddress.getByName(hostname).getHostAddress()

} 

package com.raphtory.examples.blockchain.spouts

import com.raphtory.core.components.Spout.SpoutTrait
import com.raphtory.examples.blockchain.LitecoinTransaction
import org.joda.time.DateTime
import org.joda.time.DateTimeZone
import scalaj.http.Http
import scalaj.http.HttpRequest
import spray.json._

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.language.postfixOps

class DashcoinSpout extends SpoutTrait {

  var blockcount    = 1
  val rpcuser       = System.getenv().getOrDefault("DASHCOIN_USERNAME", "").trim
  val rpcpassword   = System.getenv().getOrDefault("DASHCOIN_PASSWORD", "").trim
  val serverAddress = System.getenv().getOrDefault("DASHCOIN_NODE", "").trim
  val id            = "scala-jsonrpc"
  val baseRequest   = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain")

  //************* MESSAGE HANDLING BLOCK
  override def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout   => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock")
    case "parseBlock" => running()
    case _            => println("message not recognized!")
  }

  def running(): Unit =
    try {
      for (i <- 1 to 10) {
        getTransactions()
        blockcount += 1
        if (blockcount % 1000 == 0) println(s"Parsed block $blockcount at ${DateTime.now(DateTimeZone.UTC).getMillis}")
      }
      AllocateSpoutTask(Duration(1, NANOSECONDS), "parseBlock")

    } catch {
      case e: java.net.SocketTimeoutException     =>
      case e: spray.json.DeserializationException => AllocateSpoutTask(Duration(10, SECONDS), "parseBlock")
    }

  def getTransactions(): Unit = {
    val re        = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject
    val blockID   = re.fields("result")
    val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject
    val result    = blockData.fields("result")
    val time      = result.asJsObject.fields("time")
    for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements)
      sendTuple(LitecoinTransaction(time, blockcount, blockID, transaction))
    //val time = transaction.asJsObject.fields("time")
  }

  def request(command: String, params: String = ""): HttpRequest =
    baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""")

}
//def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") 

package daf.catalogmanager

import java.net.URLEncoder
import java.security.AccessControlException

import it.gov.daf.common.config.Read
import json._
import org.slf4j.LoggerFactory
import play.api.Configuration
import play.api.libs.json.Json
import scalaj.http.{ Http, HttpResponse }

import scala.util.{ Failure, Try, Success => TrySuccess }

class CatalogManagerClient(serviceUrl: String) {

  val logger = LoggerFactory.getLogger("it.gov.daf.CatalogManager")

  private def callService(authorization: String, catalogId: String) = Try {
    Http(s"$serviceUrl/catalog-manager/v1/catalog-ds/get/${URLEncoder.encode(catalogId,"UTF-8")}")
      .header("Authorization", authorization)
      .asString
  }

  private def parseCatalog(response: HttpResponse[String]) =
    if (response.code == 401)  Failure { new AccessControlException("Unauthorized") }
    else if (response.isError) Failure { new RuntimeException(s"Error retrieving catalog data: [${response.code}] with body [${response.body}]") }
    else Try { Json.parse(response.body).as[MetaCatalog] }

  def getById(authorization: String, catalogId: String): Try[MetaCatalog] = for {
    response <- callService(authorization, catalogId)
    catalog  <- parseCatalog(response)
  } yield catalog

}

object CatalogManagerClient {

  def fromConfig(config: Configuration) = Read.string { "daf.catalog_url" }.!.read(config) match {
    case TrySuccess(baseUrl) => new CatalogManagerClient(baseUrl)
    case Failure(error)      => throw new RuntimeException("Unable to create catalog-manager client", error)
  }

} 

package http.support.tests.client

import http.support.tests.{UserCoding, User, Api}
import typedapi.client._
import typedapi.client.scalajhttp._
import scalaj.http.Http
import io.circe.parser._
import io.circe.syntax._
import org.specs2.mutable.Specification

final class ScalajHttpClientSupportSpec extends Specification {

  import UserCoding._

  sequential

  case class DecodeException(msg: String) extends Exception

  implicit val decoder = typedapi.util.Decoder[Id, User](json => decode[User](json).fold(
    error => Left(DecodeException(error.toString())),
    user  => Right(user)
  ))
  implicit val encoder = typedapi.util.Encoder[Id, User](user => user.asJson.noSpaces)

  val cm = ClientManager(Http, "http://localhost", 9001)

  val server = TestServer.start()

  "http4s client support" >> {
    val (p, s, q, header, fixed, clInH, clFixH, clColl, serMatchH, serSendH, m0, m1, m2, m3, m4, m5, _, _, _) = deriveAll(Api)

    "paths and segments" >> {
      p().run[Blocking](cm) === Right(User("foo", 27))
      s("jim").run[Blocking](cm) === Right(User("jim", 27))
    }
    
    "queries" >> {
      q(42).run[Blocking](cm) === Right(User("foo", 42))
    }
    
    "headers" >> {
      header(42).run[Blocking](cm) === Right(User("foo", 42))
      fixed().run[Blocking](cm) === Right(User("joe", 27))
      clInH("jim").run[Blocking](cm) === Right(User("jim", 27))
      clFixH().run[Blocking](cm) === Right(User("joe", 27))
      clColl(Map("coll" -> "joe", "collect" -> "jim")).run[Blocking](cm) === Right(User("collect: jim,coll: joe", 27))
      serMatchH().run[Blocking](cm) === Right(User("joe", 27))
      serSendH().run[Blocking](cm) === Right(User("joe", 27))
    }

    "methods" >> {
      m0().run[Blocking](cm) === Right(User("foo", 27))
      m1().run[Blocking](cm) === Right(User("foo", 27))
      m2(User("jim", 42)).run[Blocking](cm) === Right(User("jim", 42))
      m3().run[Blocking](cm) === Right(User("foo", 27))
      m4(User("jim", 42)).run[Blocking](cm) === Right(User("jim", 42))
      m5(List("because")).run[Blocking](cm) === Right(User("foo", 27))
    }

    "raw" >> {
      m0().run[Id].raw(cm).body === """{"name":"foo","age":27}"""
    }

    step {
      server.shutdown.unsafeRunSync()
    }
  }
} 

import scala.Console._
import scala.sys.process._
import org.specs2._
import org.specs2.execute._
import scalaj.http.Http
import java.io.{ByteArrayOutputStream, PrintWriter}

class BasicAppSpec extends mutable.Specification  {

  // The System Properties will contain the connection information for the running Docker Compose
  // services. The key into the map is "serviceName:containerPort" and it will return "host:hostPort" which is the
  // Docker Compose generated endpoint that can be connected to at runtime. You can use this to endpoint connect to
  // for testing. Each service will also inject a "serviceName:containerId" key with the value equal to the container id.
  // You can use this to emulate service failures by killing and restarting the container.
  val basicServiceName = "basic"
  val basicServiceHostKey = s"$basicServiceName:8080"
  val basicServiceContainerIdKey = s"$basicServiceName:containerId"
  val hostInfo = getHostInfo()
  val containerId = getContainerId()

  "Validate that the Docker Compose endpoint returns a success code and the string 'Hello, World!'" >> {
      println(s"Attempting to connect to: $hostInfo, container id is $containerId")

      eventually {
        val output = Http(s"http://$hostInfo").asString
        output.isSuccess mustEqual true
        output.body must contain ("Hello, World!")
      }
  }

  def getHostInfo(): String = getContainerSetting(basicServiceHostKey)
  def getContainerId(): String = getContainerSetting(basicServiceContainerIdKey)

  def getContainerSetting(key: String): String = {
    if (System.getProperty(key) !=  null) {
      System.getProperty(key)
    }
    else {
      throw new FailureException(Failure(s"Cannot find the expected Docker Compose service key '$key' in the System Properties"))
    }
  }
} 

import org.scalatest._
import scala.Console._
import scala.sys.process._
import scalaj.http.Http
import org.scalatest.Tag
import org.scalatest.concurrent._
import org.scalatest.exceptions._
import java.io.{ByteArrayOutputStream, PrintWriter}

class BasicAppSpec extends fixture.FunSuite with fixture.ConfigMapFixture with Eventually with IntegrationPatience with Matchers {

  // The configMap passed to each test case will contain the connection information for the running Docker Compose
  // services. The key into the map is "serviceName:containerPort" and it will return "host:hostPort" which is the
  // Docker Compose generated endpoint that can be connected to at runtime. You can use this to endpoint connect to
  // for testing. Each service will also inject a "serviceName:containerId" key with the value equal to the container id.
  // You can use this to emulate service failures by killing and restarting the container.
  val basicServiceName = "basic"
  val basicServiceHostKey = s"$basicServiceName:8080"
  val basicServiceContainerIdKey = s"$basicServiceName:containerId"

  test("Validate that the Docker Compose endpoint returns a success code and the string 'Hello, World!'") {
    configMap =>{
      println(configMap)
      val hostInfo = getHostInfo(configMap)
      val containerId = getContainerId(configMap)

      println(s"Attempting to connect to: $hostInfo, container id is $containerId")

      eventually {
        val output = Http(s"http://$hostInfo").asString
        output.isSuccess shouldBe true
        output.body should include ("Hello, World!")
      }
    }
  }

  test("Validate presence of docker config information in system properties") {
    configMap =>
      Option(System.getProperty(basicServiceHostKey)) shouldBe defined
  }

  def getHostInfo(configMap: ConfigMap): String = getContainerSetting(configMap, basicServiceHostKey)
  def getContainerId(configMap: ConfigMap): String = getContainerSetting(configMap, basicServiceContainerIdKey)

  def getContainerSetting(configMap: ConfigMap, key: String): String = {
    if (configMap.keySet.contains(key)) {
      configMap(key).toString
    }
    else {
      throw new TestFailedException(s"Cannot find the expected Docker Compose service key '$key' in the configMap", 10)
    }
  }
} 

import org.scalatest._
import scala.Console._
import scala.sys.process._
import scalaj.http.Http
import org.scalatest.Tag
import org.scalatest.concurrent._
import org.scalatest.exceptions._
import java.io.{ByteArrayOutputStream, PrintWriter}

//You can define a specific tag to indicate which test should be run against the Docker Compose instance
object DockerComposeTag extends Tag("DockerComposeTag")

class BasicAppSpec extends fixture.FunSuite with fixture.ConfigMapFixture with Eventually with IntegrationPatience with Matchers {

  // The configMap passed to each test case will contain the connection information for the running Docker Compose
  // services. The key into the map is "serviceName:containerPort" and it will return "host:hostPort" which is the
  // Docker Compose generated endpoint that can be connected to at runtime. You can use this to endpoint connect to
  // for testing. Each service will also inject a "serviceName:containerId" key with the value equal to the container id.
  // You can use this to emulate service failures by killing and restarting the container.
  val basicServiceName = "basic"
  val basicServiceHostKey = s"$basicServiceName:8080"
  val basicServiceContainerIdKey = s"$basicServiceName:containerId"

  test("Validate that the Docker Compose endpoint returns a success code and the string 'Hello, World!'", DockerComposeTag) {
    configMap =>{
      println(configMap)
      val hostInfo = getHostInfo(configMap)
      val containerId = getContainerId(configMap)

      println(s"Attempting to connect to: $hostInfo, container id is $containerId")

      eventually {
        val output = Http(s"http://$hostInfo").asString
        output.isSuccess shouldBe true
        output.body should include ("Hello, World!")
      }
    }
  }

  test("Example Untagged Test. Will not be run.") {
    configMap =>
  }

  test("Validate presence of docker config information in system properties", DockerComposeTag) {
    configMap =>
      Option(System.getProperty(basicServiceHostKey)) shouldBe defined
  }

  def getHostInfo(configMap: ConfigMap): String = getContainerSetting(configMap, basicServiceHostKey)
  def getContainerId(configMap: ConfigMap): String = getContainerSetting(configMap, basicServiceContainerIdKey)

  def getContainerSetting(configMap: ConfigMap, key: String): String = {
    if (configMap.keySet.contains(key)) {
      configMap(key).toString
    }
    else {
      throw new TestFailedException(s"Cannot find the expected Docker Compose service key '$key' in the configMap", 10)
    }
  }
} 

package com.codacy.analysis.core.utils

import io.circe.parser.parse
import io.circe.{Json, ParsingFailure}
import scalaj.http.{Http, HttpRequest, HttpResponse}

class HttpHelper(apiUrl: Option[String], extraHeaders: Map[String, String]) {

  private lazy val connectionTimeoutMs = 2000
  private lazy val readTimeoutMs = 5000

  private val remoteUrl = apiUrl.getOrElse("https://api.codacy.com") + "/2.0"

  def get(endpoint: String): Either[ParsingFailure, Json] = {
    val headers: Map[String, String] = Map("Content-Type" -> "application/json") ++ extraHeaders

    val response: HttpResponse[String] =
      Http(s"$remoteUrl$endpoint").headers(headers).timeout(connectionTimeoutMs, readTimeoutMs).asString

    parse(response.body)
  }

  def post(endpoint: String, dataOpt: Option[Json] = None): Either[ParsingFailure, Json] = {
    val headers: Map[String, String] = dataOpt.fold(Map.empty[String, String]) { _ =>
      Map("Content-Type" -> "application/json")
    } ++ extraHeaders

    val request: HttpRequest = dataOpt.map { data =>
      Http(s"$remoteUrl$endpoint").postData(data.toString)
    }.getOrElse(Http(s"$remoteUrl$endpoint"))
      .method("POST")
      .headers(headers)
      .timeout(connectionTimeoutMs, readTimeoutMs)

    parse(request.asString.body)
  }

} 

import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.receiver.Receiver
import org.jfarcand.wcs.{TextListener, WebSocket}
import scala.util.parsing.json.JSON
import scalaj.http.Http

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

class TFLArrivalPredictionsByLine() extends Receiver[String](StorageLevel.MEMORY_ONLY) with Runnable {
  private val tflUrl = "https://api.tfl.gov.uk/Line/circle/Arrivals?stopPointId=940GZZLUERC&app_id=a73727f3&app_key=dc8150560a2422afae2b70cf291c4327"
  @transient
  private var thread: Thread = _
  override def onStart(): Unit = {
     thread = new Thread(this)
     thread.start()
  }
  override def onStop(): Unit = {
     thread.interrupt()
  }
  override def run(): Unit = {
    while (true){
     receive();
     Thread.sleep(60*1000);
    }
   }
  
  private def receive(): Unit = {
    val httpClient = new DefaultHttpClient();
    val getRequest = new HttpGet(tflUrl);
    getRequest.addHeader("accept", "application/json");

    val response = httpClient.execute(getRequest);
    if (response.getStatusLine().getStatusCode() != 200) {
      throw new RuntimeException("Failed : HTTP error code : "
         + response.getStatusLine().getStatusCode());
    }

    val br = new BufferedReader(
                         new InputStreamReader((response.getEntity().getContent())));

    var output=br.readLine();
    while(output!=null){          
          println(output)
          output=br.readLine()
        }    
  }
} 

package com.wixpress.build.codota

import java.net.SocketTimeoutException

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wixpress.build.codota.CodotaThinClient.{PathAttemptResult, RetriesLoop}
import org.slf4j.LoggerFactory

import scala.util.{Failure, Success, Try}
import scalaj.http.{Http, HttpOptions, HttpResponse}

class CodotaThinClient(token: String,
                       codePack: String,
                       baseURL: String = CodotaThinClient.DefaultBaseURL,
                       maxRetries: Int = 5) {

  private val logger = LoggerFactory.getLogger(getClass)
  private val timeout: Int = 1000
  private val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
    .configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true)

  private val requestURLTemplate = s"$baseURL/api/codenav/artifact"

  def pathFor(artifactName: String): Option[String] = {
    retriesLoop(artifactName).collectFirst {
      case Success(p) => p
    }.flatten
  }

  private def retriesLoop(artifactName: String): RetriesLoop = {
    (1 to maxRetries).toStream.map(pathForArtifact(_, artifactName))
  }

  private def pathForArtifact(retry: Int, artifactName: String): PathAttemptResult = {
    pathForArtifact(artifactName) match {
      case Success(p) => Success(p)
      case Failure(e: SocketTimeoutException) if retry < maxRetries =>
        handleSocketTimeout(retry, artifactName, e)
      case Failure(e) => throw e
    }
  }

  private def handleSocketTimeout(retry: Int, artifactName: String, e: SocketTimeoutException) = {
    logger.warn(s"($retry/$maxRetries) Timeout when trying to get path for $artifactName")
    Thread.sleep(3000 / ((maxRetries - retry) + 1))
    Failure(e)
  }

  private def pathForArtifact(artifactName: String): PathAttemptResult = {
    for {
      response <- requestFor(artifactName)
      body <- responseBody(response)
      path <- extractPath(body, artifactName)
    } yield path
  }

  private def requestFor(artifactName: String) = {
    Try {
      Http(s"$requestURLTemplate/$artifactName/metadata")
        .param("codePack", codePack)
        .header("Authorization", s"bearer $token")
        .option(HttpOptions.readTimeout(timeout))
        .asString
    }
  }

  private def responseBody(resp: HttpResponse[String]) = {
    Try {
      val body = resp.body
      val code = resp.code

      code match {
        case 200 => body
        case 401 => throw NotAuthorizedException(body)
        case 404 if body.contains(codePack) => throw CodePackNotFoundException(body)
        case 403 => throw MissingCodePackException()
        case 404 => throw MetaDataOrArtifactNotFound(body)
      }
    }
  }

  private def extractPath(body: String, artifactName: String) = {
    Try {
      val metaData = mapper.readValue(body, classOf[ArtifactMetadata])
      Option(metaData.path)
    }
  }
}

object CodotaThinClient {
  val DefaultBaseURL = "https://gateway.codota.com"
  type PathAttemptResult = Try[Option[String]]
  type RetriesLoop = Stream[PathAttemptResult]
}

case class ArtifactMetadata(path: String) 

package com.wixpress.build.sync

import com.wixpress.build.maven.{Coordinates, DependencyNode}
import com.wixpress.build.sync.ArtifactoryRemoteStorage.{Sha256, _}
import org.apache.commons.codec.digest.DigestUtils
import org.slf4j.LoggerFactory
import scalaj.http.{Http, HttpResponse}

import scala.collection.parallel.CompositeThrowable
import scala.util.{Failure, Success, Try}

class MavenRepoRemoteStorage(baseUrls: List[String]) extends DependenciesRemoteStorage {

  private val log = LoggerFactory.getLogger(getClass)

  override def checksumFor(node: DependencyNode): Option[String] = {
    val artifact = node.baseDependency.coordinates
    getChecksum(artifact).orElse({
      log.info("Fallback to calculating checksum by downloading the bytes...")
      calculateChecksum(artifact)
    }).toOption
  }

  private def getChecksum(coordinates: Coordinates): Try[Sha256] = getWithFallback(getArtifactSha256, coordinates)

  private def calculateChecksum(coordinates: Coordinates): Try[Sha256] =
    for {
      res <- getWithFallback(getArtifactBytes, coordinates)
      sha256 <- calculateSha256(res)
    } yield sha256

  private def getWithFallback[T](getter: (Coordinates, String) => Try[T],
                                 coordinates: Coordinates): Try[T] = {
    val attempts = baseUrls.par.map(url => getter(coordinates, url)).toList
    Try {
      attempts.collectFirst {
        case Success(e) => e
      }.getOrElse {
        val failures = attempts.collect { case Failure(t) => t }
        log.warn(s"Could not fetch resource $coordinates: ${failures.map(_.getMessage)}")
        throw CompositeThrowable(failures.toSet)
      }
    }
  }

  private def getArtifactSha256(artifact: Coordinates, baseUrl: String): Try[Sha256] = {
    val url = s"$baseUrl/${artifact.toSha256Path}"
    extract(response = Http(url).asString, forArtifact = artifact, inUrl = url)
  }

  private def getArtifactBytes(artifact: Coordinates, baseUrl: String): Try[Array[Byte]] = {
    val url = s"$baseUrl/${artifact.toArtifactPath}"
    extract(response = Http(url).asBytes, forArtifact = artifact, inUrl = url)
  }

  private def extract[T](response: HttpResponse[T], forArtifact: Coordinates, inUrl: String): Try[T] =
    response match {
      case r if r.isSuccess => Success(r.body)
      case r if r.is4xx => Failure(artifactNotFoundException(forArtifact, inUrl))
      // TODO: should probably sleep and retry in case of response code 5xx
      case r => Failure(errorFetchingArtifactException(forArtifact, inUrl, r))
    }

  private def calculateSha256(jar: Array[Byte]) = {
    Try {
      DigestUtils.sha256Hex(jar)
    }
  }

  private def artifactNotFoundException(artifact: Coordinates, url: String) =
    ArtifactNotFoundException(s"Got 'NotFound' from $url")

  private def errorFetchingArtifactException[T](artifact: Coordinates, url: String, r: HttpResponse[T]) =
    ErrorFetchingArtifactException(s"Got Error [${r.code} : ${r.statusLine}] from $url")

}