java.net.URLConnection Scala Examples

package org.archive.archivespark.sparkling.http

import java.io.{BufferedInputStream, InputStream}
import java.net.{HttpURLConnection, URL, URLConnection}

import org.archive.archivespark.sparkling.logging.LogContext
import org.archive.archivespark.sparkling.util.Common

import scala.collection.JavaConverters._
import scala.util.Try

object HttpClient {
  val DefaultRetries: Int = 30
  val DefaultSleepMillis: Int = 1000
  val DefaultTimeoutMillis: Int = -1

  implicit val logContext: LogContext = LogContext(this)

  def request[R](url: String, headers: Map[String, String] = Map.empty, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: InputStream => R): R = rangeRequest(url, headers, retries = retries, sleepMillis = sleepMillis, timeoutMillis = timeoutMillis)(action)

  def rangeRequest[R](url: String, headers: Map[String, String] = Map.empty, offset: Long = 0, length: Long = -1, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: InputStream => R): R = {
    rangeRequestConnection(url, headers, offset, length, retries, sleepMillis, timeoutMillis) { case connection: HttpURLConnection =>
      val in = new BufferedInputStream(connection.getInputStream)
      val r = action(in)
      Try(in.close())
      r
    }
  }

  def requestMessage[R](url: String, headers: Map[String, String] = Map.empty, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: HttpMessage => R): R = rangeRequestMessage(url, headers, retries = retries, sleepMillis = sleepMillis, timeoutMillis = timeoutMillis)(action)

  def rangeRequestMessage[R](url: String, headers: Map[String, String] = Map.empty, offset: Long = 0, length: Long = -1, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: HttpMessage => R): R = {
    rangeRequestConnection(url, headers, offset, length, retries, sleepMillis, timeoutMillis) { case connection: HttpURLConnection =>
      val in = new BufferedInputStream(connection.getInputStream)
      val responseHeaders = connection.getHeaderFields.asScala.toMap.flatMap{case (k, v) => v.asScala.headOption.map((if (k == null) "" else k) -> _)}
      val message = new HttpMessage(connection.getResponseMessage, responseHeaders, in)
      val r = action(message)
      Try(in.close())
      r
    }
  }

  def requestConnection[R](url: String, headers: Map[String, String] = Map.empty, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: URLConnection => R): R = rangeRequestConnection(url, headers, retries = retries, sleepMillis = sleepMillis, timeoutMillis = timeoutMillis)(action)

  def rangeRequestConnection[R](url: String, headers: Map[String, String] = Map.empty, offset: Long = 0, length: Long = -1, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: URLConnection => R): R = {
    Common.timeoutWithReporter(timeoutMillis) { reporter =>
      val connection = Common.retry(retries, sleepMillis, (retry, e) => {
        "Request failed (" + retry + "/" + retries + "): " + url + " (" + offset + "-" + (if (length >= 0) length else "") + ") - " + e.getMessage
      }) { _ =>
        reporter.alive()
        val connection = new URL(url).openConnection()
        for ((key, value) <- headers) connection.addRequestProperty(key, value)
        if (offset > 0 || length >= 0) connection.addRequestProperty("Range", "bytes=" + offset + "-" + (if (length >= 0) offset + length - 1 else ""))
        connection.asInstanceOf[HttpURLConnection]
      }
      val r = action(connection)
      Try(connection.disconnect())
      r
    }
  }
} 

package slide

import java.io.{File, FileOutputStream}
import java.net.{URL, URLConnection}
import java.nio.channels.{Channels, ReadableByteChannel}

class FileManager {

    var currentFile: String = ""
    var numberOfDownloads: Int = 0

    def downloadFile(dlsite: String, path: String): Unit = {
        val url: URL = new URL(dlsite)
        val file: File = new File(path)

        if (isConnected(url)) {
            currentFile = path
            onDownloadStart()

            new Thread(new Runnable {
                override def run(): Unit = {
                    try {
                        val rbc: ReadableByteChannel = Channels.newChannel(url.openStream())
                        val fos: FileOutputStream = new FileOutputStream(file)

                        fos.getChannel.transferFrom(rbc, 0, java.lang.Long.MAX_VALUE)
                        fos.close()

                        numberOfDownloads += 1
                        onDownloadFinished()
                    } catch {
                        case e: Exception =>
                            println("Error: Could not download ADB, please run as Administrator")
                    }
                }
            }).start()
        }
    }

    def isConnected(site: URL): Boolean = {
        try {
            // test connection
            val conn: URLConnection = site.openConnection()
            conn.setConnectTimeout(5000)
            conn.getContent

            true
        } catch {
            case e: Exception => false
        }
    }

    def onDownloadStart(): Unit = {}

    def onDownloadFinished(): Unit = {}

    // var onDownloadStart: () => Unit = null
    // var onDownloadFinished: () => Unit = null
} 

package cn.piflow.bundle.http

import java.io.{BufferedReader, InputStreamReader, PrintWriter}
import java.net.{HttpURLConnection, InetAddress, URL, URLConnection}

import cn.piflow.Runner
import cn.piflow.conf.bean.FlowBean
import cn.piflow.conf.util.{FileUtil, OptionUtil}
import cn.piflow.util.{PropertyUtil, ServerIpUtil}
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.sql.SparkSession
import org.h2.tools.Server
import org.junit.Test

import scala.util.parsing.json.JSON

class GetUrlTest {

  @Test
  def testFlow(): Unit ={

    //parse flow json
    val file = "src/main/resources/flow/http/getUrl.json"
    val flowJsonStr = FileUtil.fileReader(file)
    val map = OptionUtil.getAny(JSON.parseFull(flowJsonStr)).asInstanceOf[Map[String, Any]]
    println(map)

    //create flow
    val flowBean = FlowBean(map)
    val flow = flowBean.constructFlow()


    val ip = InetAddress.getLocalHost.getHostAddress
    cn.piflow.util.FileUtil.writeFile("server.ip=" + ip, ServerIpUtil.getServerIpFile())
    val h2Server = Server.createTcpServer("-tcp", "-tcpAllowOthers", "-tcpPort","50001").start()
    //execute flow
    val spark = SparkSession.builder()
      .master("local[12]")
      .appName("hive")
      .config("spark.driver.memory", "4g")
      .config("spark.executor.memory", "8g")
      .config("spark.cores.max", "8")
      .config("hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
      .enableHiveSupport()
      .getOrCreate()

    val process = Runner.create()
      .bind(classOf[SparkSession].getName, spark)
      .bind("checkpoint.path", "")
      .bind("debug.path","")
      .start(flow);

    process.awaitTermination();
    val pid = process.pid();
    println(pid + "!!!!!!!!!!!!!!!!!!!!!")
    spark.close();
  }

} 

package com.webtrends.harness.http

import java.io.{BufferedReader, InputStreamReader}
import java.net.{HttpURLConnection, URLConnection}
import java.util.zip.{GZIPInputStream, InflaterInputStream}


  case class HttpResponseData(
                               statusLine: String,
                               content: String,
                               headers: collection.mutable.Map[String, String]) {

    private val startIndex = statusLine.indexOf(" ") + 1
    val status = statusLine.substring(startIndex, startIndex + 3)

    override def toString: String = {
      val sb = new StringBuilder
      sb.append(statusLine + "\n")
      headers.foreach(m => sb.append(m._1 + "=" + m._2 + "\n"))
      sb.append("\n" + content + "\n")
      sb.toString()
    }

  }
} 

package coursier.cache.protocol

import java.io.File
import java.net.{URL, URLConnection, URLStreamHandler, URLStreamHandlerFactory}

import coursier.test.HandmadeMetadata

class TestprotocolHandler extends URLStreamHandlerFactory {

  def createURLStreamHandler(protocol: String): URLStreamHandler = new URLStreamHandler {
    protected def openConnection(url: URL): URLConnection = {
      val f = new File(HandmadeMetadata.repoBase, "http/abc.com" + url.getPath)
      val resURLOpt = Option(f.toURI.toURL)

      resURLOpt match {
        case None =>
          new URLConnection(url) {
            def connect() = throw new NoSuchElementException(f.getAbsolutePath)
          }
        case Some(resURL) =>
          resURL.openConnection()
      }
    }
  }
}

object TestprotocolHandler {
  val protocol = "testprotocol"

  // get this namespace via a macro?
  val expectedClassName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
  assert(classOf[TestprotocolHandler].getName == expectedClassName)
} 

package almond.display

import java.io.{ByteArrayOutputStream, InputStream}
import java.net.{HttpURLConnection, URL, URLConnection}
import java.nio.charset.{Charset, StandardCharsets}

import scala.util.Try

abstract class TextDisplay extends UpdatableDisplay {

  def contentOrUrl: Either[URL, String]

  def content: Option[String] = contentOrUrl.right.toOption
  def url: Option[URL] = contentOrUrl.left.toOption

  def finalContent: String =
    contentOrUrl match {
      case Left(url) =>
        TextDisplay.urlContent(url)
      case Right(c) => c
    }

  def withContent(code: String): UpdatableDisplay
  def withUrl(url: String): UpdatableDisplay

}

object TextDisplay {

  type Builder[T] = Display.Builder[String, T]

  private[almond] def readFully(is: InputStream): Array[Byte] = {

    val buffer = new ByteArrayOutputStream
    val data = Array.ofDim[Byte](16384)

    var nRead = 0
    while ( {
      nRead = is.read(data, 0, data.length)
      nRead != -1
    })
      buffer.write(data, 0, nRead)

    buffer.flush()
    buffer.toByteArray
  }

  def urlContent(url: URL): String = {

    var conn: URLConnection = null
    val (rawContent, charsetOpt) = try {
      conn = url.openConnection()
      conn.setConnectTimeout(5000) // allow users to tweak that?
      val b = readFully(conn.getInputStream)
      val charsetOpt0 = conn match {
        case conn0: HttpURLConnection =>
          conn0
            .getContentType
            .split(';')
            .map(_.trim)
            .find(_.startsWith("charset="))
            .map(_.stripPrefix("charset="))
            .filter(Charset.isSupported)
            .map(Charset.forName)
        case _ =>
          None
      }
      (b, charsetOpt0)
    } finally {
      if (conn != null) {
        Try(conn.getInputStream.close())
        conn match {
          case conn0: HttpURLConnection =>
            Try(conn0.getErrorStream.close())
            Try(conn0.disconnect())
          case _ =>
        }
      }
    }

    new String(rawContent, charsetOpt.getOrElse(StandardCharsets.UTF_8))
  }
} 

package org.weave.deps

import java.io.InputStream
import java.net.URL
import java.net.URLConnection

import org.mule.weave.v2.parser.MessageCollector
import org.mule.weave.v2.parser.ast.AstNode
import org.mule.weave.v2.parser.ast.annotation.AnnotationNode
import org.mule.weave.v2.parser.ast.annotation.AnnotationNodeHelper
import org.mule.weave.v2.parser.ast.variables.NameIdentifier
import org.mule.weave.v2.parser.exception.WeaveRuntimeException
import org.mule.weave.v2.parser.phase.AnnotationProcessor
import org.mule.weave.v2.scope.AstNavigator
import org.mule.weave.v2.scope.ScopesNavigator

class ResourceDependencyAnnotationProcessor(weavePathUpdater: (InputStream, Boolean, String) => Unit) extends AnnotationProcessor {

  
  override def run(annotatedNode: AstNode, astNavigator: AstNavigator, scopeNavigator: ScopesNavigator, messageCollector: MessageCollector, annotation: AnnotationNode): Unit = {
    val url = AnnotationNodeHelper.argString("url", annotation).getOrElse(throw new WeaveRuntimeException("Missing argument `url`", annotation.location()))
    val unzip = AnnotationNodeHelper.argString("unzip", annotation).getOrElse(throw new WeaveRuntimeException("Missing argument `unzip`", annotation.location())).toBoolean
    val resourceUrl = new URL(url)
    val connection: URLConnection = resourceUrl.openConnection()
    val stream: InputStream = connection.getInputStream
    try {
      weavePathUpdater(stream, unzip, url)
    } finally {
      stream.close()
    }
  }

}


object ResourceDependencyAnnotationProcessor {

  def apply(weavePathUpdater: (InputStream, Boolean, String) => Unit): ResourceDependencyAnnotationProcessor = new ResourceDependencyAnnotationProcessor(weavePathUpdater)

  val ANNOTATION_NAME: NameIdentifier = NameIdentifier("dw::deps::Deps::ResourceDependency")
}