java.io.FileOutputStream Scala Examples

package org.apache.spark.deploy.worker

import java.io.{File, FileOutputStream, InputStream, IOException}

import scala.collection.JavaConverters._
import scala.collection.Map

import org.apache.spark.SecurityManager
import org.apache.spark.deploy.Command
import org.apache.spark.internal.Logging
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils


  def redirectStream(in: InputStream, file: File) {
    val out = new FileOutputStream(file, true)
    // TODO: It would be nice to add a shutdown hook here that explains why the output is
    //       terminating. Otherwise if the worker dies the executor logs will silently stop.
    new Thread("redirect output to " + file) {
      override def run() {
        try {
          Utils.copyStream(in, out, true)
        } catch {
          case e: IOException =>
            logInfo("Redirection to " + file + " closed: " + e.getMessage)
        }
      }
    }.start()
  }
} 

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.ledger.participant.state.kvutils

import java.io.{DataOutputStream, FileOutputStream}

import com.daml.ledger.participant.state.kvutils.DamlKvutils._
import org.slf4j.LoggerFactory

import scala.collection.JavaConverters._


  def dumpLedgerEntry(
      submission: DamlSubmission,
      participantId: String,
      entryId: DamlLogEntryId,
      logEntry: DamlLogEntry,
      outputState: Map[DamlStateKey, DamlStateValue]): Unit =
    optLedgerDumpStream.foreach { outs =>
      val dumpEntry = DamlKvutils.LedgerDumpEntry.newBuilder
        .setSubmission(Envelope.enclose(submission))
        .setEntryId(entryId)
        .setParticipantId(participantId)
        .setLogEntry(Envelope.enclose(logEntry))
        .addAllOutputState(
          outputState.map {
            case (k, v) =>
              DamlKvutils.LedgerDumpEntry.StatePair.newBuilder
                .setStateKey(k)
                .setStateValue(Envelope.enclose(v))
                .build
          }.asJava
        )
        .build

      // Messages are delimited by a header containing the message size as int32
      outs.writeInt(dumpEntry.getSerializedSize)
      dumpEntry.writeTo(outs)
      outs.flush()
    }

} 

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.ledger.participant.state.kvutils.export

import java.io.{DataOutputStream, FileOutputStream}
import java.time.Instant

import com.daml.ledger.participant.state.v1.ParticipantId
import com.daml.ledger.validator.LedgerStateOperations.{Key, Value}
import com.google.protobuf.ByteString
import org.slf4j.LoggerFactory

trait LedgerDataExporter {

  
  def finishedProcessing(correlationId: String): Unit
}

object LedgerDataExporter {
  val EnvironmentVariableName = "KVUTILS_LEDGER_EXPORT"

  private val logger = LoggerFactory.getLogger(this.getClass)

  private lazy val outputStreamMaybe: Option[DataOutputStream] = {
    Option(System.getenv(EnvironmentVariableName))
      .map { filename =>
        logger.info(s"Enabled writing ledger entries to $filename")
        new DataOutputStream(new FileOutputStream(filename))
      }
  }

  private lazy val instance = outputStreamMaybe
    .map(new FileBasedLedgerDataExporter(_))
    .getOrElse(NoopLedgerDataExporter)

  def apply(): LedgerDataExporter = instance
} 

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.jwt

import java.io.{File, FileNotFoundException, FileOutputStream}

import com.daml.lf.data.TryOps.Bracket.bracket
import scalaz.std.option._
import scalaz.syntax.applicative._

import scala.util.{Failure, Success, Try}

object RsaKeysGenerator {

  private val keySize: Int = 2048

  def generate(destination: domain.KeyPair[File]): Try[domain.KeyPair[File]] =
    for {
      keyPair <- generate_(): Try[domain.KeyPair[Array[Byte]]]
      publicKeyFile <- writeKey(keyPair.publicKey, destination.publicKey)
      privateKeyFile <- writeKey(keyPair.privateKey, destination.privateKey)
    } yield domain.KeyPair(publicKey = publicKeyFile, privateKey = privateKeyFile)

  def generate(): Try[domain.KeyPair[Seq[Byte]]] =
    generate_().map(k => k.map(as => as.toSeq))

  private def generate_(): Try[domain.KeyPair[Array[Byte]]] =
    Try {
      val kpg = java.security.KeyPairGenerator.getInstance("RSA")
      kpg.initialize(keySize)
      Option(kpg.generateKeyPair()).flatMap(domainKeyPair)
    } flatMap {
      case Some(x) => Success(x)
      case None => Failure(new IllegalStateException("Cannot generate RSA key pair, null returned"))
    }

  private def domainKeyPair(k: java.security.KeyPair): Option[domain.KeyPair[Array[Byte]]] =
    ^(Option(k.getPublic), Option(k.getPrivate)) { (pub, pvt) =>
      domain.KeyPair(publicKey = pub.getEncoded, privateKey = pvt.getEncoded)
    }

  private def writeKey(key: Array[Byte], file: File): Try[File] =
    bracket(Try(new FileOutputStream(file)))(close).flatMap { ostream =>
      for {
        encoder <- Try(java.util.Base64.getEncoder)
        _ <- Try(ostream.write(encoder.encode(key)))
        _ <- exists(file)
      } yield file
    }

  private def close(a: FileOutputStream): Try[Unit] = Try(a.close())

  private def exists(f: File): Try[File] =
    for {
      b <- Try(f.exists())
      x <- if (b) Success(f) else Failure(new FileNotFoundException(f.getAbsolutePath))
    } yield x
} 

// scalastyle:off println
package org.apache.spark.examples.graphx

import java.io.{FileOutputStream, PrintWriter}

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.graphx.{GraphXUtils, PartitionStrategy}
import org.apache.spark.graphx.util.GraphGenerators


  def main(args: Array[String]) {
    val options = args.map {
      arg =>
        arg.dropWhile(_ == '-').split('=') match {
          case Array(opt, v) => (opt -> v)
          case _ => throw new IllegalArgumentException("Invalid argument: " + arg)
        }
    }

    var app = "pagerank"
    var niter = 10
    var numVertices = 100000
    var numEPart: Option[Int] = None
    var partitionStrategy: Option[PartitionStrategy] = None
    var mu: Double = 4.0
    var sigma: Double = 1.3
    var degFile: String = ""
    var seed: Int = -1

    options.foreach {
      case ("app", v) => app = v
      case ("niters", v) => niter = v.toInt
      case ("nverts", v) => numVertices = v.toInt
      case ("numEPart", v) => numEPart = Some(v.toInt)
      case ("partStrategy", v) => partitionStrategy = Some(PartitionStrategy.fromString(v))
      case ("mu", v) => mu = v.toDouble
      case ("sigma", v) => sigma = v.toDouble
      case ("degFile", v) => degFile = v
      case ("seed", v) => seed = v.toInt
      case (opt, _) => throw new IllegalArgumentException("Invalid option: " + opt)
    }

    val conf = new SparkConf()
      .setAppName(s"GraphX Synth Benchmark (nverts = $numVertices, app = $app)")
    GraphXUtils.registerKryoClasses(conf)

    val sc = new SparkContext(conf)

    // Create the graph
    println(s"Creating graph...")
    val unpartitionedGraph = GraphGenerators.logNormalGraph(sc, numVertices,
      numEPart.getOrElse(sc.defaultParallelism), mu, sigma, seed)
    // Repartition the graph
    val graph = partitionStrategy.foldLeft(unpartitionedGraph)(_.partitionBy(_)).cache()

    var startTime = System.currentTimeMillis()
    val numEdges = graph.edges.count()
    println(s"Done creating graph. Num Vertices = $numVertices, Num Edges = $numEdges")
    val loadTime = System.currentTimeMillis() - startTime

    // Collect the degree distribution (if desired)
    if (!degFile.isEmpty) {
      val fos = new FileOutputStream(degFile)
      val pos = new PrintWriter(fos)
      val hist = graph.vertices.leftJoin(graph.degrees)((id, _, optDeg) => optDeg.getOrElse(0))
        .map(p => p._2).countByValue()
      hist.foreach {
        case (deg, count) => pos.println(s"$deg \t $count")
      }
    }

    // Run PageRank
    startTime = System.currentTimeMillis()
    if (app == "pagerank") {
      println("Running PageRank")
      val totalPR = graph.staticPageRank(niter).vertices.map(_._2).sum()
      println(s"Total PageRank = $totalPR")
    } else if (app == "cc") {
      println("Running Connected Components")
      val numComponents = graph.connectedComponents.vertices.map(_._2).distinct().count()
      println(s"Number of components = $numComponents")
    }
    val runTime = System.currentTimeMillis() - startTime

    println(s"Num Vertices = $numVertices")
    println(s"Num Edges = $numEdges")
    println(s"Creation time = ${loadTime/1000.0} seconds")
    println(s"Run time = ${runTime/1000.0} seconds")

    sc.stop()
  }
}
// scalastyle:on println 

package org.apache.spark.graphx

import java.io.File
import java.io.FileOutputStream
import java.io.OutputStreamWriter
import java.nio.charset.StandardCharsets

import org.apache.spark.SparkFunSuite
import org.apache.spark.util.Utils

class GraphLoaderSuite extends SparkFunSuite with LocalSparkContext {

  test("GraphLoader.edgeListFile") {
    withSpark { sc =>
      val tmpDir = Utils.createTempDir()
      val graphFile = new File(tmpDir.getAbsolutePath, "graph.txt")
      val writer = new OutputStreamWriter(new FileOutputStream(graphFile), StandardCharsets.UTF_8)
      for (i <- (1 until 101)) writer.write(s"$i 0\n")
      writer.close()
      try {
        val graph = GraphLoader.edgeListFile(sc, tmpDir.getAbsolutePath)
        val neighborAttrSums = graph.aggregateMessages[Int](
          ctx => ctx.sendToDst(ctx.srcAttr),
          _ + _)
        assert(neighborAttrSums.collect.toSet === Set((0: VertexId, 100)))
      } finally {
        Utils.deleteRecursively(tmpDir)
      }
    }
  }
} 

package org.apache.spark.storage

import java.io.{FileOutputStream, IOException, RandomAccessFile}
import java.nio.ByteBuffer
import java.nio.channels.FileChannel.MapMode

import com.google.common.io.Closeables

import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
import org.apache.spark.util.io.ChunkedByteBuffer


  def put(blockId: BlockId)(writeFunc: FileOutputStream => Unit): Unit = {
    if (contains(blockId)) {
      throw new IllegalStateException(s"Block $blockId is already present in the disk store")
    }
    logDebug(s"Attempting to put block $blockId")
    val startTime = System.currentTimeMillis
    val file = diskManager.getFile(blockId)
    val fileOutputStream = new FileOutputStream(file)
    var threwException: Boolean = true
    try {
      writeFunc(fileOutputStream)
      threwException = false
    } finally {
      try {
        Closeables.close(fileOutputStream, threwException)
      } finally {
         if (threwException) {
          remove(blockId)
        }
      }
    }
    val finishTime = System.currentTimeMillis
    logDebug("Block %s stored as %s file on disk in %d ms".format(
      file.getName,
      Utils.bytesToString(file.length()),
      finishTime - startTime))
  }

  def putBytes(blockId: BlockId, bytes: ChunkedByteBuffer): Unit = {
    put(blockId) { fileOutputStream =>
      val channel = fileOutputStream.getChannel
      Utils.tryWithSafeFinally {
        bytes.writeFully(channel)
      } {
        channel.close()
      }
    }
  }

  def getBytes(blockId: BlockId): ChunkedByteBuffer = {
    val file = diskManager.getFile(blockId.name)
    val channel = new RandomAccessFile(file, "r").getChannel
    Utils.tryWithSafeFinally {
      // For small files, directly read rather than memory map
      if (file.length < minMemoryMapBytes) {
        val buf = ByteBuffer.allocate(file.length.toInt)
        channel.position(0)
        while (buf.remaining() != 0) {
          if (channel.read(buf) == -1) {
            throw new IOException("Reached EOF before filling buffer\n" +
              s"offset=0\nfile=${file.getAbsolutePath}\nbuf.remaining=${buf.remaining}")
          }
        }
        buf.flip()
        new ChunkedByteBuffer(buf)
      } else {
        new ChunkedByteBuffer(channel.map(MapMode.READ_ONLY, 0, file.length))
      }
    } {
      channel.close()
    }
  }

  def remove(blockId: BlockId): Boolean = {
    val file = diskManager.getFile(blockId.name)
    if (file.exists()) {
      val ret = file.delete()
      if (!ret) {
        logWarning(s"Error deleting ${file.getPath()}")
      }
      ret
    } else {
      false
    }
  }

  def contains(blockId: BlockId): Boolean = {
    val file = diskManager.getFile(blockId.name)
    file.exists()
  }
} 

package org.apache.spark.storage

import java.io.{File, FileOutputStream}

import org.scalatest.{BeforeAndAfter, Matchers}

import org.apache.spark._
import org.apache.spark.util.Utils

class TopologyMapperSuite  extends SparkFunSuite
  with Matchers
  with BeforeAndAfter
  with LocalSparkContext {

  test("File based Topology Mapper") {
    val numHosts = 100
    val numRacks = 4
    val props = (1 to numHosts).map{i => s"host-$i" -> s"rack-${i % numRacks}"}.toMap
    val propsFile = createPropertiesFile(props)

    val sparkConf = (new SparkConf(false))
    sparkConf.set("spark.storage.replication.topologyFile", propsFile.getAbsolutePath)
    val topologyMapper = new FileBasedTopologyMapper(sparkConf)

    props.foreach {case (host, topology) =>
      val obtainedTopology = topologyMapper.getTopologyForHost(host)
      assert(obtainedTopology.isDefined)
      assert(obtainedTopology.get === topology)
    }

    // we get None for hosts not in the file
    assert(topologyMapper.getTopologyForHost("host").isEmpty)

    cleanup(propsFile)
  }

  def createPropertiesFile(props: Map[String, String]): File = {
    val testFile = new File(Utils.createTempDir(), "TopologyMapperSuite-test").getAbsoluteFile
    val fileOS = new FileOutputStream(testFile)
    props.foreach{case (k, v) => fileOS.write(s"$k=$v\n".getBytes)}
    fileOS.close
    testFile
  }

  def cleanup(testFile: File): Unit = {
    testFile.getParentFile.listFiles.filter { file =>
      file.getName.startsWith(testFile.getName)
    }.foreach { _.delete() }
  }

} 

package de.valtech.foss

import scala.io.Source
import scala.collection.mutable.Map
import scala.collection.mutable.ListBuffer
import scala.collection.JavaConverters._
import Console.{GREEN, RED, RESET}
import scala.language.reflectiveCalls

import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.nio.channels.FileChannel.MapMode._
import java.nio.ByteOrder._
import java.nio.ByteBuffer

import de.valtech.foss.proto.RosbagIdxOuterClass.RosbagIdx

object Main extends App {
  def help() = {
    Console.err.printf(s"""
${RESET}${GREEN}Usage:
	--file <ros.bag> file to process
	--version print Rosbag version and exit
	--offset <offset> --number <records> Seek at offset < 1073741824 and read the specified number of records
${RESET}By default will just create the protobuf idx file needed for configuration.\n\n""")
    sys.exit(0)
  }

  val pargs = Map[String,AnyRef]()
  def process_cli(args: List[String]) :Boolean = args match {
    case Nil => true // parse success
    case "-v" :: rest => pargs += ("version" -> Some(true)); process_cli(rest)
    case "--version" :: rest => pargs += ("version" -> Some(true)); process_cli(rest)
    case "-f" :: x :: rest => pargs += ("file" -> x); process_cli(rest)
    case "--file" :: x :: rest => pargs += ("file" -> x); process_cli(rest)
    case "-n" :: x :: rest => pargs += ("number" -> Some(x.toInt)); process_cli(rest)
    case "--number" :: x :: rest => pargs += ("number" -> Some(x.toInt)); process_cli(rest)
    case "-o" :: x :: rest => pargs += ("offset" -> Some(x.toInt)); process_cli(rest)
    case "--offset" :: x :: rest => pargs += ("offset" -> Some(x.toInt)); process_cli(rest)
    case "-h" :: rest => help(); false
    case "--help" :: rest => help(); false
    case _ => Console.err.printf(s"${RESET}${RED}Unknown argument " + args.head); false
  }
  process_cli(args.toList)

  def use[T <: { def close() }]
    (resource: T)
    (code: T ⇒ Unit) =
    try
      code(resource)
    finally
      resource.close()

  pargs("file") match {
    case f:String => process()
    case _ => help()
  }

  def process(): Unit = {
    val fin = new File(pargs("file").asInstanceOf[String])
    use(new FileInputStream(fin)) { stream => {
      //printf("min: %s\n", Math.min(1073741824, fin.length) )
      val buffer = stream.getChannel.map(READ_ONLY, 0, Math.min(1073741824, fin.length)).order(LITTLE_ENDIAN)
      val p:RosbagParser = new RosbagParser(buffer)
      val version = p.read_version()
      val h = p.read_record().get
      if(pargs contains "version") {
        printf("%s\n%s\n\n", version, h)
        return
      }
      if(pargs contains "number"){
        buffer position pargs.getOrElse("offset",None).asInstanceOf[Option[Int]].getOrElse(0)
        for(i <- List.range(0,pargs("number").asInstanceOf[Option[Int]].getOrElse(0)))
          println(p.read_record)
        return
      }
      val idxpos = h.header.fields("index_pos").asInstanceOf[Long]
      //printf("idxpos: %s %s\n", idxpos, Math.min(1073741824, fin.length) )
      val b = stream.getChannel.map(READ_ONLY, idxpos, Math.min(1073741824, fin.length - idxpos)).order(LITTLE_ENDIAN)
      val pp:RosbagParser = new RosbagParser(b)
      val c = pp.read_connections(h.header, Nil)
      val chunk_idx = pp.read_chunk_infos(c)
      Console.err.printf(s"""${RESET}${GREEN}Found: """
          + chunk_idx.size
          +s""" chunks\n${RESET}It should be the same number reported by rosbag tool.\nIf you encounter any issues try reindexing your file and submit an issue.
          ${RESET}\n""")
      val fout = new FileOutputStream(pargs("file").asInstanceOf[String] + ".idx.bin")
      val builder = RosbagIdx.newBuilder
      for(i <- chunk_idx) builder.addArray(i)
      builder.build().writeTo(fout)
      fout.close()
      //printf("[%s]\n",chunk_idx.toArray.mkString(","))
    }}
  }
} 

package com.intel.analytics.bigdl.visualization.tensorboard

import java.io.{File, FileOutputStream}

import com.google.common.primitives.{Ints, Longs}
import com.intel.analytics.bigdl.utils.Crc32
import netty.Crc32c
import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}
import org.tensorflow.util.Event


private[bigdl] class RecordWriter(file: Path, fs: FileSystem) {
  val outputStream = if (file.toString.startsWith("hdfs://")) {
    // FSDataOutputStream couldn't flush data to localFileSystem in time. So reading summaries
    // will throw exception.
    fs.create(file, true, 1024)
  } else {
    // Using FileOutputStream when write to local.
    new FileOutputStream(new File(file.toString))
  }
  val crc32 = new Crc32c()
  def write(event: Event): Unit = {
    val eventString = event.toByteArray
    val header = Longs.toByteArray(eventString.length.toLong).reverse
    outputStream.write(header)
    outputStream.write(Ints.toByteArray(Crc32.maskedCRC32(crc32, header).toInt).reverse)
    outputStream.write(eventString)
    outputStream.write(Ints.toByteArray(Crc32.maskedCRC32(crc32, eventString).toInt).reverse)
    if (outputStream.isInstanceOf[FSDataOutputStream]) {
      // Flush data to HDFS.
      outputStream.asInstanceOf[FSDataOutputStream].hflush()
    }
  }

  def close(): Unit = {
    outputStream.close()
  }
} 

package org.archive.archivespark.sparkling.io

import java.io.{FileInputStream, FileOutputStream, OutputStream}

import org.apache.hadoop.fs.Path
import org.archive.archivespark.sparkling.logging.{Log, LogContext}

import scala.util.Try

class HdfsFileWriter private(filename: String, append: Boolean, replication: Short) extends OutputStream {
  implicit val logContext: LogContext = LogContext(this)

  private val file = IOUtil.tmpFile

  Log.info("Writing to temporary local file " + file.getCanonicalPath + " (" + filename + ")...")

  val out = new FileOutputStream(file)

  override def close(): Unit = {
    Try { out.close() }
    Log.info("Copying from temporary file " + file.getCanonicalPath + " to " + filename + "...")
    if (append) {
      val in = new FileInputStream(file)
      val appendOut = HdfsIO.fs.append(new Path(filename))
      IOUtil.copy(in, appendOut)
      appendOut.close()
      in.close()
      file.delete()
    } else HdfsIO.copyFromLocal(file.getCanonicalPath, filename, move = true, overwrite = true, replication)
    Log.info("Done. (" + filename + ")")
  }

  override def write(b: Int): Unit = out.write(b)
  override def write(b: Array[Byte]): Unit = out.write(b)
  override def write(b: Array[Byte], off: Int, len: Int): Unit = out.write(b, off, len)
  override def flush(): Unit = out.flush()
}

object HdfsFileWriter {
  def apply(filename: String, overwrite: Boolean = false, append: Boolean = false, replication: Short = 0): HdfsFileWriter = {
    if (!overwrite && !append) HdfsIO.ensureNewFile(filename)
    new HdfsFileWriter(filename, append, replication)
  }
} 

package com.versioneye

import java.io.{File, FileOutputStream}
import java.util.Properties

import scala.io.Source


object PropertiesUtil {

  protected val propertiesFile: String = "versioneye.properties"

  def writeProperties(response: ProjectJsonResponse, propertiesFile: File, baseUrl: String): Unit = {
    var properties: Properties = null

    if (!propertiesFile.exists()) {
      createPropertiesFile(propertiesFile)
      properties = new Properties()
    }
    else {
      properties = loadProperties(propertiesFile)
    }

    if (response.getId != null) {
      properties.setProperty("project_id", response.getId)
    }

    val fos = new FileOutputStream(propertiesFile)
    properties.store(fos, s" Properties for $baseUrl")
    fos.close()
  }

  def getProperties(propertiesFile: File): Properties = {
    return loadProperties(propertiesFile)
  }

  private def loadProperties(file: File): Properties = {
    if (!file.exists) {
      return null
    }

    val properties = new Properties()
    val reader = Source.fromFile(file).reader()
    properties.load(reader)
    reader.close()
    return properties
  }

  private def createPropertiesFile(file: File) {
    val parent: File = file.getParentFile
    if (!parent.exists) {
      parent.mkdirs
    }
    file.createNewFile
  }

  def getPropertiesFile(properties: String, projectDirectory: File, withHome: Boolean): File = {
    val candidates = getPropertyFileCandidates(properties, projectDirectory, false)
    val firstFile = candidates.find(_.exists())
    return firstFile.orElse(candidates.find(!_.exists())).get
  }

  def containing(key: String, file: File): Boolean = {
    if (!file.exists()) {
      return false
    }

    return loadProperties(file).containsKey(key)
  }

  def getPropertiesFileContainingProperty(key: String, properties: String, projectDirectory: File): Option[File] = {
    val candidates = getPropertyFileCandidates(properties, projectDirectory, true)
    val firstFile = candidates.find(containing(key, _))
    return firstFile
  }

  def getPropertyFileCandidates(properties: String, projectDirectory: File, withHome: Boolean): Seq[File] = {
    if (!properties.isEmpty) {
      return Seq(new File(properties));
    }

    var qaResources = new File(projectDirectory, "src/qa/resources/" + propertiesFile)
    var mainResources = new File(projectDirectory, "src/main/resources/" + propertiesFile)
    var userHome = new File(System.getProperty("user.home") + "/.m2/" + propertiesFile)

    if (withHome)
      return Seq(qaResources, mainResources, userHome)
    else
      return Seq(qaResources, mainResources)
  }

} 

package slide

import java.io.{File, FileOutputStream}
import java.net.{URL, URLConnection}
import java.nio.channels.{Channels, ReadableByteChannel}

class FileManager {

    var currentFile: String = ""
    var numberOfDownloads: Int = 0

    def downloadFile(dlsite: String, path: String): Unit = {
        val url: URL = new URL(dlsite)
        val file: File = new File(path)

        if (isConnected(url)) {
            currentFile = path
            onDownloadStart()

            new Thread(new Runnable {
                override def run(): Unit = {
                    try {
                        val rbc: ReadableByteChannel = Channels.newChannel(url.openStream())
                        val fos: FileOutputStream = new FileOutputStream(file)

                        fos.getChannel.transferFrom(rbc, 0, java.lang.Long.MAX_VALUE)
                        fos.close()

                        numberOfDownloads += 1
                        onDownloadFinished()
                    } catch {
                        case e: Exception =>
                            println("Error: Could not download ADB, please run as Administrator")
                    }
                }
            }).start()
        }
    }

    def isConnected(site: URL): Boolean = {
        try {
            // test connection
            val conn: URLConnection = site.openConnection()
            conn.setConnectTimeout(5000)
            conn.getContent

            true
        } catch {
            case e: Exception => false
        }
    }

    def onDownloadStart(): Unit = {}

    def onDownloadFinished(): Unit = {}

    // var onDownloadStart: () => Unit = null
    // var onDownloadFinished: () => Unit = null
} 

package com.wavesplatform.dex.db

import java.io.{File, FileInputStream, FileOutputStream}
import java.nio.file.Files
import java.util.Base64

import cats.syntax.either._
import com.google.common.primitives.{Bytes, Ints}
import com.wavesplatform.dex.crypto.Enigma
import com.wavesplatform.dex.db.AccountStorage.Settings.EncryptedFile
import com.wavesplatform.dex.domain.account.KeyPair
import com.wavesplatform.dex.domain.bytes.ByteStr
import com.wavesplatform.dex.domain.crypto
import net.ceedubs.ficus.readers.ValueReader

import scala.collection.mutable.ArrayBuffer

case class AccountStorage(keyPair: KeyPair)

object AccountStorage {

  sealed trait Settings

  object Settings {

    case class InMem(seed: ByteStr)                        extends Settings
    case class EncryptedFile(path: File, password: String) extends Settings

    implicit val valueReader: ValueReader[Settings] = ValueReader.relative[Settings] { config =>
      config.getString("type") match {
        case "in-mem" => InMem(Base64.getDecoder.decode(config.getString("in-mem.seed-in-base64")))
        case "encrypted-file" =>
          EncryptedFile(
            path = new File(config.getString("encrypted-file.path")),
            password = config.getString("encrypted-file.password")
          )
        case x => throw new IllegalArgumentException(s"The type of account storage '$x' is unknown. Please update your settings.")
      }
    }
  }

  def load(settings: Settings): Either[String, AccountStorage] = settings match {
    case Settings.InMem(seed) => Right(AccountStorage(KeyPair(seed)))
    case Settings.EncryptedFile(file, password) =>
      if (file.isFile) {
        val encryptedSeedBytes = readFile(file)
        val key                = Enigma.prepareDefaultKey(password)
        val decryptedBytes     = Enigma.decrypt(key, encryptedSeedBytes)
        AccountStorage(KeyPair(decryptedBytes)).asRight
      } else s"A file '${file.getAbsolutePath}' doesn't exist".asLeft
  }

  def save(seed: ByteStr, to: EncryptedFile): Unit = {
    Files.createDirectories(to.path.getParentFile.toPath)
    val key                = Enigma.prepareDefaultKey(to.password)
    val encryptedSeedBytes = Enigma.encrypt(key, seed.arr)
    writeFile(to.path, encryptedSeedBytes)
  }

  def getAccountSeed(baseSeed: ByteStr, nonce: Int): ByteStr = ByteStr(crypto.secureHash(Bytes.concat(Ints.toByteArray(nonce), baseSeed)))

  def readFile(file: File): Array[Byte] = {
    val reader = new FileInputStream(file)
    try {
      val buff = new Array[Byte](1024)
      val r    = new ArrayBuffer[Byte]
      while (reader.available() > 0) {
        val read = reader.read(buff)
        if (read > 0) {
          r.appendAll(buff.iterator.take(read))
        }
      }
      r.toArray
    } finally {
      reader.close()
    }
  }

  def writeFile(file: File, bytes: Array[Byte]): Unit = {
    val writer = new FileOutputStream(file, false)
    try writer.write(bytes)
    finally writer.close()
  }
} 

package com.bigchange.util

import java.io.{File, FileOutputStream}

import jxl.Workbook
import jxl.write.{Label, WritableSheet, WritableWorkbook}
import org.apache.poi.xssf.usermodel.{XSSFSheet, XSSFWorkbook}
import org.apache.spark.rdd.RDD


  def createXLS(data: RDD[(String,Iterable[((String, String),Int)])]) = {

    var sheetNumber = 0

    data.sortBy(_._1).foreach { x =>

      val path = x._1.split("-").slice(0, 2).mkString("-")  // 按月

      val file = new File(rootDir + "/" + path + ".xls")

      if(!file.exists()) {

        file.createNewFile()
        writableBook = Workbook.createWorkbook(new File(rootDir + "/" + fileName))
        sheetNumber = 0

      } else {

        writableBook = Workbook.createWorkbook(file, Workbook.getWorkbook(file))
        val  numberOfSheet = writableBook.getNumberOfSheets
        sheetNumber = numberOfSheet

      }

      val writerData = x._2
      println("writerData:" + writerData.size)

      var sheet = writableBook.createSheet(x._1, sheetNumber)

      // 初始化新sheet的模板
      sheetInit(sheet, 0, Array("stock","time","value"))

      var i = 1
      var k = 0 // 超出行限制另建sheet
      writerData.foreach { key =>

        if(i > 60000) {
          sheetNumber += 1
          k = k + 1
          writableBook.createSheet(x._1 + "-" + k, sheetNumber)
          sheet = writableBook.getSheet(sheetNumber)
          // 初始化新sheet的模板
          sheetInit(sheet, 0, Array("stock","time","value"))
          i = 0
        }
        val stockName = new Label(0, i, key._1._1)
        val time = new Label(1, i, key._1._2)
        val number = new Label(2, i, key._2.toString)
        sheet.addCell(stockName)
        sheet.addCell(time)
        sheet.addCell(number)
        i += 1
      }
      writableBook.write()
      writableBook.close()
      sheetNumber += 1

    }
  }
} 

package com.owlandrews.nescala.helpers

import com.owlandrews.nescala.Console

object File {
  import java.io.File
  import java.net.URL
  import java.io.{FileFilter, FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream}
  import javax.imageio.ImageIO

  import scala.util.Try
  import scala.xml.XML
  import scala.language.postfixOps

  import sys.process._

  import com.typesafe.config.ConfigFactory

  def Download(url: String, filename: String) = (for{
    url <- Try(new URL(url))
    conn <- Try(url.openConnection().connect())
    file <- Try(new File(filename))
  } yield Try(url  #> file !!)) map {x => new File(filename)}

  def Writer(filename: String)(op: java.io.PrintWriter => Unit) = {
    val p = new java.io.PrintWriter(new File(filename))
    try op(p)
    finally p.close()
  }

  def Write(filename: String, content: String) = {
    val res = new java.io.PrintWriter(new File(filename))
    res.write(content)
    res.close()
  }

  def Filter = new FileFilter {
    override def accept(pathname: File): Boolean = pathname.getName.toLowerCase.endsWith(".nes")
  }

  def Image(file:Try[File]) = file.map(ImageIO.read)

  def Image(filename:String) = Try(ImageIO.read(resource(filename)))

  def Xml(filename:String) = XML.load(resource("/database.xml"))

  def Config(filename:String) = {
    val file = new File(filename)
    file.exists() match {
      case true => ConfigFactory.parseFile(file)
      case false => ConfigFactory.empty()
    }
  }

  def SaveState(console:Console) = {
    val fos = new FileOutputStream(s"$ApplicationFolder/${console.cartridge.CRC}.save")
    val oos = new ObjectOutputStream(fos)

    oos.writeObject(console)
    oos.close()
  }

  def LoadState(crc:String):Try[Console] = Try {
    val fis = new FileInputStream(s"$ApplicationFolder/$crc.save")
    val ois = new ObjectInputStreamWithCustomClassLoader(fis)

    val console = ois.readObject.asInstanceOf[Console]
    ois.close()
    console
  }

  // Taken from: https://gist.github.com/ramn/5566596
  private class ObjectInputStreamWithCustomClassLoader(fileInputStream: FileInputStream) extends ObjectInputStream(fileInputStream) {
    override def resolveClass(desc: java.io.ObjectStreamClass): Class[_] = {
      try { Class.forName(desc.getName, false, getClass.getClassLoader) }
      catch { case ex: ClassNotFoundException => super.resolveClass(desc) }
    }
  }

  lazy val ApplicationFolder: File = {
    val settingDirectory = System.getProperty("user.home") + "/.nescala"
    val settings = new java.io.File(settingDirectory)
    if (!settings.exists()) settings.mkdir()
    settings
  }

  private def resource(filename:String) = getClass.getResourceAsStream(filename)
} 

package com.rossabaker
package benchmarks

import org.openjdk.jmh.annotations._

@State(Scope.Thread)
@Fork(2)
@Measurement(iterations = 10)
@Warmup(iterations = 10)
@Threads(1)
class Cp extends BenchmarkUtils {
  @Benchmark
  def fs2Sync(): Unit = {
    import _root_.fs2._, Stream._
    import java.nio.file.Paths
    io.file.readAll[Task](Paths.get("testdata/lorem-ipsum.txt"), 4096)
      .to(io.file.writeAll[Task](Paths.get("out/lorem-ipsum.txt")))
      .run
      .unsafeRun
  }

  @Benchmark
  def fs2Async(): Unit = {
    import _root_.fs2._, Stream._
    import java.nio.file.Paths
    io.file.readAllAsync[Task](Paths.get("testdata/lorem-ipsum.txt"), 4096)
      .to(io.file.writeAllAsync[Task](Paths.get("out/lorem-ipsum.txt")))
      .run
      .unsafeRun
  }

  @Benchmark
  def scalazStreamIo(): Unit = {
    import _root_.scalaz.stream._, Process._
    constant(4096)
      .through(io.fileChunkR("testdata/lorem-ipsum.txt"))
      .to(io.fileChunkW("out/lorem-ipsum.txt"))
      .run
      .unsafePerformSync
  }

  @Benchmark
  def scalazStreamNio(): Unit = {
    import _root_.scalaz.stream._, Process._
    constant(4096)
      .through(nio.file.chunkR("testdata/lorem-ipsum.txt"))
      .to(nio.file.chunkW("out/lorem-ipsum.txt"))
      .run
      .unsafePerformSync
  }

   }
            callback.onError(ex)
          }

          def onComplete(): Unit = {
            try {
              out.close()
              callback.onSuccess(())
            } catch {
              case NonFatal(ex) =>
                callback.onError(ex)
            }
          }
        }
      }

    Await.result(
      copyFile(new File("testdata/lorem-ipsum.txt"), new File("out/lorem-ipsum.txt"), 4096)
        .runAsync(monixScheduler),
      Duration.Inf
    )
  }
} 

package com.example.streaming

import java.io.{ InputStream, OutputStream, FileInputStream, FileOutputStream }

import cats.implicits._
import cats.effect.{ IO, IOApp, Blocker, ExitCode }

import fs2.{ Stream, Chunk, Pure }

import dev.tauri.seals.scodec.StreamCodecs._

object Main extends IOApp {

  sealed trait Color
  final case object Brown extends Color
  final case object Grey extends Color

  sealed trait Animal
  final case class Elephant(name: String, tuskLength: Float) extends Animal
  final case class Quokka(name: String, color: Color = Brown) extends Animal
  final case class Quagga(name: String, speed: Double) extends Animal

  def transform(from: InputStream, to: OutputStream)(f: Animal => Stream[Pure, Animal]): IO[Unit] = {
    Blocker[IO].use { blocker =>
      val input = fs2.io.readInputStream(
        IO.pure(from),
        chunkSize = 1024,
        blocker = blocker
      )
      val sIn: Stream[IO, Animal] = input.through(streamDecoderFromReified[Animal].toPipeByte[IO]).flatMap(f)
      val sOut: Stream[IO, Unit] = streamEncoderFromReified[Animal].encode(sIn).flatMap { bv =>
        Stream.chunk(Chunk.bytes(bv.bytes.toArray))
      }.through(fs2.io.writeOutputStream(
        IO.pure(to),
        blocker = blocker,
        closeAfterUse = true
      ))
      sOut.compile.drain
    }
  }

  val transformer: Animal => Stream[Pure, Animal] = {
    case Elephant(n, tl) => Stream(Elephant(n, tl + 17))
    case Quokka(n, Brown) => Stream(Quokka(n, Grey))
    case q @ Quokka(_, _) => Stream(q)
    case Quagga(_, _) => Stream.empty
  }

  override def run(args: List[String]): IO[ExitCode] = {
    val (from, to) = args match {
      case List(from, to, _*) =>
        (from, to)
      case List(from) =>
        (from, "out.bin")
      case _ =>
        ("in.bin", "out.bin")
    }

    val task = transform(new FileInputStream(from), new FileOutputStream(to))(transformer)
    task.as(ExitCode.Success)
  }
} 

package org.apache.iota.fey

import java.io.{File, FileOutputStream}
import java.net.URL
import java.nio.file.{Files, Paths}
import com.eclipsesource.schema._
import akka.actor.ActorRef
import com.eclipsesource.schema.SchemaValidator
import org.apache.commons.io.IOUtils
import play.api.libs.json._
import scala.concurrent.duration._
import scala.util.Properties._

abstract class FeyGenericActorReceiver(override val params: Map[String,String] = Map.empty,
                                       override val backoff: FiniteDuration = 1.minutes,
                                       override val connectTo: Map[String,ActorRef] = Map.empty,
                                       override val schedulerTimeInterval: FiniteDuration = 2.seconds,
                                       override val orchestrationName: String = "",
                                       override val orchestrationID: String = "",
                                       override val autoScale: Boolean = false) extends FeyGenericActor{

  private[fey] val feyCore = FEY_CORE_ACTOR.actorRef

  override final def processMessage[T](message: T, sender: ActorRef): Unit = {
    try {
      val jsonString = getJSONString(message)
      if(jsonString != "{}") {
        processJson(jsonString)
      }
      startBackoff()
    }catch{
      case e: Exception => log.error(e, s"Could not process message $message")
    }
  }

  private[fey] def processJson(jsonString: String) = {
    var orchID:String = "None"
    try{
      val orchestrationJSON = Json.parse(jsonString)
      orchID = (orchestrationJSON \ JSON_PATH.GUID).as[String]
      val valid = validJson(orchestrationJSON)
      if(valid && (orchestrationJSON \ JSON_PATH.COMMAND).as[String].toUpperCase != "DELETE"){
        checkForLocation(orchestrationJSON)
      }
      if(valid) {
        feyCore ! FeyCore.ORCHESTRATION_RECEIVED(orchestrationJSON, None)
      }else{
        log.warning(s"Could not forward Orchestration $orchID. Invalid JSON schema")
      }
    } catch {
      case e: Exception =>
        log.error(e, s"Orchestration $orchID could not be forwarded")
    }
  }

  
  def resolveCredentials(credentials: Option[JsObject]):Option[(String, String)] = {
    credentials match {
      case None => None
      case Some(cred) =>
        val user = (cred \ JSON_PATH.JAR_CRED_USER).as[String]
        val password = (cred \ JSON_PATH.JAR_CRED_PASSWORD).as[String]
        Option(envOrElse(user,user), envOrElse(password,password))
    }
  }

} 

package org.apache.iota.fey

import java.io.FileOutputStream
import java.net.URL
import java.io.File

import com.eclipsesource.schema._
import org.slf4j.LoggerFactory
import play.api.libs.json._
import JSON_PATH._
import java.nio.file.{Files, Paths}

import org.apache.commons.io.IOUtils
import org.apache.commons.codec.binary.Base64
import scala.util.Properties._


  def exceptionOnRun(e: Exception): Unit
}

object HttpBasicAuth {
  val BASIC = "Basic"
  val AUTHORIZATION = "Authorization"

  def encodeCredentials(username: String, password: String): String = {
    new String(Base64.encodeBase64((username + ":" + password).getBytes))
  }

  def getHeader(username: String, password: String): String =
    BASIC + " " + encodeCredentials(username, password)
} 

package higherkindness.skeuomorph

import java.io.{File, FileOutputStream, InputStream}
import java.nio.file.{Files, Paths, StandardOpenOption}

import cats.effect.{Resource, Sync}

object FileUtils {
  def fileHandle[F[_]: Sync](name: String): Resource[F, File] =
    Resource.make(
      Sync[F].delay(new File(name))
    )(file => Sync[F].delay(file.deleteOnExit()))

  def fileOutputStream[F[_]: Sync](file: File): Resource[F, FileOutputStream] =
    Resource.make(
      Sync[F].delay(new FileOutputStream(file))
    )(fos => Sync[F].delay(fos.close()))

  def fileInputStream[F[_]: Sync](name: String): Resource[F, InputStream] =
    Resource.make(
      Sync[F].delay(Files.newInputStream(Paths.get(name), StandardOpenOption.DELETE_ON_CLOSE))
    )(is => Sync[F].delay(is.close()))
} 

package org.apache.openwhisk.standalone

import java.io.{File, FileOutputStream, InputStream}
import java.util.zip.ZipInputStream

object Unzip {

  def apply(is: InputStream, dir: File): Unit = {
    //Based on https://stackoverflow.com/a/40547896/1035417
    val zis = new ZipInputStream((is))
    val dest = dir.toPath
    Stream.continually(zis.getNextEntry).takeWhile(_ != null).foreach { zipEntry =>
      if (!zipEntry.isDirectory) {
        val outPath = dest.resolve(zipEntry.getName)
        val outPathParent = outPath.getParent
        if (!outPathParent.toFile.exists()) {
          outPathParent.toFile.mkdirs()
        }

        val outFile = outPath.toFile
        val out = new FileOutputStream(outFile)
        val buffer = new Array[Byte](4096)
        Stream.continually(zis.read(buffer)).takeWhile(_ != -1).foreach(out.write(buffer, 0, _))
        out.close()
      }
    }
    zis.close()
  }

} 

package ml.combust.mleap.runtime.serialization

import java.io.{File, FileOutputStream, OutputStream}
import java.nio.charset.Charset

import ml.combust.mleap.ClassLoaderUtil
import ml.combust.mleap.runtime.frame.LeapFrame
import resource._

import scala.reflect.ClassTag
import scala.util.Try


object FrameWriter {
  def apply[LF <: LeapFrame[LF]](frame: LF,
                                 format: String = BuiltinFormats.json,
                                 clOption: Option[ClassLoader] = None)
                                (implicit ct: ClassTag[LF]): FrameWriter = {
    val cl = clOption.getOrElse(ClassLoaderUtil.findClassLoader(classOf[FrameWriter].getCanonicalName))
    cl.loadClass(s"$format.DefaultFrameWriter").
      getConstructor(classOf[LeapFrame[_]]).
      newInstance(frame).
      asInstanceOf[FrameWriter]
  }
}

trait FrameWriter {
  def toBytes(charset: Charset = BuiltinFormats.charset): Try[Array[Byte]]

  def save(file: File): Try[Any] = save(file, BuiltinFormats.charset)
  def save(file: File, charset: Charset = BuiltinFormats.charset): Try[Any] = {
    (for(out <- managed(new FileOutputStream(file))) yield {
      save(out, charset)
    }).tried.flatMap(identity)
  }

  def save(out: OutputStream): Try[Any] = save(out, BuiltinFormats.charset)
  def save(out: OutputStream, charset: Charset): Try[Any] = {
    toBytes(charset).map(out.write)
  }
} 

package ml.combust.bundle.serializer

import java.io.{File, FileInputStream, FileOutputStream}
import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}

import resource._


case class FileUtil() {
  def rmRF(path: File): Array[(String, Boolean)] = {
    Option(path.listFiles).map(_.flatMap(f => rmRF(f))).getOrElse(Array()) :+ (path.getPath -> path.delete)
  }

  def extract(source: File, dest: File): Unit = {
    dest.mkdirs()
    for(in <- managed(new ZipInputStream(new FileInputStream(source)))) {
      extract(in, dest)
    }
  }

  def extract(in: ZipInputStream, dest: File): Unit = {
    dest.mkdirs()
    val buffer = new Array[Byte](1024 * 1024)

    var entry = in.getNextEntry
    while(entry != null) {
      if(entry.isDirectory) {
        new File(dest, entry.getName).mkdirs()
      } else {
        val filePath = new File(dest, entry.getName)
        for(out <- managed(new FileOutputStream(filePath))) {
          var len = in.read(buffer)
          while(len > 0) {
            out.write(buffer, 0, len)
            len = in.read(buffer)
          }
        }
      }
      entry = in.getNextEntry
    }
  }

  def zip(source: File, dest: File): Unit = {
    for(out <- managed(new ZipOutputStream(new FileOutputStream(dest)))) {
      zip(source, out)
    }
  }

  def zip(source: File, dest: ZipOutputStream): Unit = zip(source, source, dest)

  def zip(base: File, source: File, dest: ZipOutputStream): Unit = {
    val buffer = new Array[Byte](1024 * 1024)

    for(files <- Option(source.listFiles);
        file <- files) {
      val name = file.toString.substring(base.toString.length + 1)

      if(file.isDirectory) {
        dest.putNextEntry(new ZipEntry(s"$name/"))
        zip(base, file, dest)
      } else {
        dest.putNextEntry(new ZipEntry(name))

        for (in <- managed(new FileInputStream(file))) {
          var read = in.read(buffer)
          while (read > 0) {
            dest.write(buffer, 0, read)
            read = in.read(buffer)
          }
        }
      }
    }
  }
} 

package com.amazon.milan

import java.io.{File, FileOutputStream}
import java.net.URL
import java.nio.file.{Files, Path}

import com.typesafe.scalalogging.Logger
import org.slf4j.LoggerFactory


package object tools {
  private lazy val logger = Logger(LoggerFactory.getLogger("milan"))

  def addToSbtClasspath(paths: Seq[Path]): Unit = {
    val urls = paths.map(_.toUri.toURL).toList

    urls.foreach(url => logger.info(s"Adding {$url} to classpath."))

    val classLoader = this.getClass.getClassLoader
    val addMethod = classLoader.getClass.getDeclaredMethod("add", classOf[Seq[URL]])
    addMethod.invoke(classLoader, urls)
  }

  
  def compileApplicationInstance(providerClassName: String,
                                 providerParameters: List[(String, String)],
                                 compilerClassName: String,
                                 compilerParameters: List[(String, String)],
                                 outputFile: Path): File = {
    val providerClass = ClassHelper.loadClass(providerClassName)

    val provider =
      providerClass.getConstructors.find(_.getParameterCount == 0) match {
        case None =>
          throw new Exception(s"Provider class $providerClassName does not have a default constructor.")

        case Some(constructor) =>
          constructor.newInstance().asInstanceOf[ApplicationInstanceProvider]
      }

    val instance = provider.getApplicationInstance(providerParameters)

    val actualCompilerClassName = KnownCompilers.convertFromKnownCompiler(compilerClassName)
    val compilerClass = ClassHelper.loadClass(actualCompilerClassName)

    val compiler = compilerClass.getConstructors.find(_.getParameterCount == 0) match {
      case None =>
        throw new Exception(s"Compiler class $actualCompilerClassName does not have a default constructor.")

      case Some(constructor) =>
        constructor.newInstance().asInstanceOf[ApplicationInstanceCompiler]
    }

    println(s"Writing generated code to output file '$outputFile'.")

    Files.createDirectories(outputFile.getParent)

    val outputStream = new FileOutputStream(outputFile.toFile)

    try {
      compiler.compile(instance, compilerParameters, outputStream)
      outputFile.toFile
    }
    finally {
      outputStream.close()
    }
  }
} 

package flaky.history

import java.io.{File, FileInputStream, FileOutputStream}
import java.util.zip.{ZipEntry, ZipOutputStream}

object Zip {


  def compressFolder(zipFilePath: File, folder: File): Unit = {
    import java.io.File
    def recursiveListFiles(f: File): Array[File] = {
      val these = f.listFiles
      these.filter(_.isFile) ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
    }

    val toList = recursiveListFiles(folder).toList
    compress(zipFilePath, folder, toList)
  }

  def compress(zipFilePath: File, root: File, files: List[File]): Unit = {
    val zip = new ZipOutputStream(new FileOutputStream(zipFilePath))
    val rootPath = root.getAbsolutePath
    try {
      for (file <- files) {
        zip.putNextEntry(new ZipEntry(file.getAbsolutePath.substring(rootPath.length)))
        val in = new FileInputStream(file)
        try {
          Iterator
            .continually(in.read())
            .takeWhile(_ > -1)
            .foreach(zip.write)
        } finally {
          in.close()
        }
        zip.closeEntry()
      }
    }
    finally {
      zip.close()
    }
  }
} 

package flaky

import java.io.File

trait Unzip {

  def unzip(zipped: File, unzipDir: File, deleteOnExit: Boolean = true): Unit = {
    import java.io.{FileInputStream, FileOutputStream}
    import java.util.zip.ZipInputStream
    val fis = new FileInputStream(zipped)
    val zis = new ZipInputStream(fis)

    unzipDir.mkdirs()
    Stream
      .continually(zis.getNextEntry)
      .takeWhile(_ != null)
      .foreach { file =>
        if (file.isDirectory) {
          val dir = new File(unzipDir, file.getName)
          dir.mkdirs()
          if (deleteOnExit){
            dir.deleteOnExit()
          }
        } else {
          val file1 = new File(unzipDir, file.getName)
          if (deleteOnExit){
            file1.deleteOnExit()
          }
          val fout = new FileOutputStream(file1)
          val buffer = new Array[Byte](1024)
          Stream.continually(zis.read(buffer)).takeWhile(_ != -1).foreach(fout.write(buffer, 0, _))
        }
      }
  }
} 

package org.datafy.aws.app.matt.extras

import java.io.{File, FileOutputStream, IOException, InputStream}
import java.util

import scala.collection.JavaConverters._
import org.xml.sax.{ContentHandler, SAXException}
import org.apache.tika.metadata.Metadata
import org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE
import org.apache.tika.mime.MediaType
import org.apache.tika.parser.{AbstractParser, ParseContext}
import org.apache.commons.io.IOUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.parquet.hadoop.ParquetFileReader
import org.apache.parquet.hadoop.ParquetReader
import org.apache.parquet.format.converter.ParquetMetadataConverter
import org.apache.parquet.hadoop.util.HadoopInputFile
import org.apache.parquet.tools.json.JsonRecordFormatter
import org.apache.parquet.tools.read.{SimpleReadSupport, SimpleRecord}
import org.apache.tika.exception.TikaException
import org.apache.tika.sax.XHTMLContentHandler

import scala.util.Random


class TikaParquetParser extends AbstractParser {
  // make some stuff here
  final val PARQUET_RAW = MediaType.application("x-parquet")

  private val SUPPORTED_TYPES: Set[MediaType] = Set(PARQUET_RAW)

  def getSupportedTypes(context: ParseContext): util.Set[MediaType] = {
    SUPPORTED_TYPES.asJava
  }

  @throws(classOf[IOException])
  @throws(classOf[SAXException])
  @throws(classOf[TikaException])
  def parse(stream: InputStream, handler: ContentHandler,
            metadata: Metadata, context: ParseContext): Unit = {
    // create temp file from stream
    val fileNamePrefix = Random.alphanumeric.take(5).mkString
    val tempFile = File.createTempFile(s"parquet-${fileNamePrefix}", ".parquet")
    IOUtils.copy(stream, new FileOutputStream(tempFile))

    val conf = new Configuration()
    val path = new Path(tempFile.getAbsolutePath)
    val parquetMetadata = ParquetFileReader.readFooter(conf, path, ParquetMetadataConverter.NO_FILTER)
    var defaultReader: ParquetReader[SimpleRecord] = null

    val columns = parquetMetadata.getFileMetaData.getSchema.getFields
    metadata.set(CONTENT_TYPE, PARQUET_RAW.toString)
    metadata.set("Total Number of Columns", columns.size.toString)
    metadata.set("Parquet Column Names", columns.toString)

    val xhtml = new XHTMLContentHandler(handler, metadata)
    xhtml.startDocument()
    xhtml.startElement("p")

    // ::TODO:: ensure parquet reader reads all files not only file row
    try {
      defaultReader = ParquetReader.builder(new SimpleReadSupport(), new Path(tempFile.getAbsolutePath)).build()
      if(defaultReader.read() != null) {
        val values: SimpleRecord = defaultReader.read()
        val jsonFormatter = JsonRecordFormatter.fromSchema(parquetMetadata.getFileMetaData.getSchema)

        val textContent: String = jsonFormatter.formatRecord(values)
        xhtml.characters(textContent)
        xhtml.endElement("p")
        xhtml.endDocument()
      }

    } catch {
        case e: Throwable => e.printStackTrace()
          if (defaultReader != null) {
          try {
            defaultReader.close()
          } catch{
            case _: Throwable =>
          }
        }
    } finally {
      if (tempFile != null) tempFile.delete()
    }
  }

} 

package org.datafy.aws.app.matt.extras

import java.io.{File, FileOutputStream, IOException, InputStream}
import java.util

import org.apache.commons.io.IOUtils
import org.apache.hadoop.conf.Configuration

import scala.collection.JavaConverters._
import org.apache.hadoop.fs.Path
import org.apache.hadoop.hive.serde2.objectinspector.StructField
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector
import org.apache.orc.OrcFile
import org.apache.orc.OrcFile.ReaderOptions
import org.apache.orc.Reader
import org.apache.orc.RecordReader
import org.apache.tika.exception.TikaException
import org.apache.tika.metadata.Metadata
import org.apache.tika.mime.MediaType
import org.apache.tika.parser.{AbstractParser, ParseContext}
import org.xml.sax.{ContentHandler, SAXException}

import scala.util.Random


class TikaHadoopOrcParser extends AbstractParser  {
  final val ORC_RAW = MediaType.application("x-orc")

  private val SUPPORTED_TYPES: Set[MediaType] = Set(ORC_RAW)

  def getSupportedTypes(context: ParseContext): util.Set[MediaType] = {
    SUPPORTED_TYPES.asJava
  }

  @throws(classOf[IOException])
  @throws(classOf[SAXException])
  @throws(classOf[TikaException])
  def parse(stream: InputStream, handler: ContentHandler,
            metadata: Metadata, context: ParseContext): Unit = {
    // create temp file from stream
    try {
      val fileNamePrefix = Random.alphanumeric.take(5).mkString
      val tempFile = File.createTempFile(s"orc-${fileNamePrefix}", ".orc")
      IOUtils.copy(stream, new FileOutputStream(tempFile))

      val path = new Path(tempFile.getAbsolutePath)
      val conf = new Configuration()
      val orcReader = OrcFile.createReader(path, new ReaderOptions(conf))
      val records: RecordReader = orcReader.rows()

      val storeRecord = null
      val firstBlockKey = null

    } catch {
      case e: Throwable => e.printStackTrace()
    }



//    val fields =

  }
} 

package com.stratio.sparta.serving.api.utils

import java.io.{BufferedOutputStream, File, FileOutputStream}
import java.net.InetAddress
import java.text.DecimalFormat
import java.util.function.Predicate

import akka.event.slf4j.SLF4JLogging
import com.stratio.sparta.serving.api.constants.HttpConstant
import com.stratio.sparta.serving.core.config.SpartaConfig
import com.stratio.sparta.serving.core.models.files.SpartaFile
import spray.http.BodyPart

import scala.util.{Failure, Success, Try}

trait FileActorUtils extends SLF4JLogging {

  //The dir where the files will be saved
  val targetDir: String
  val apiPath: String

  //Regexp for name validation
  val patternFileName: Option[Predicate[String]] = None

  def deleteFiles(): Try[_] =
    Try {
      val directory = new File(targetDir)
      if (directory.exists && directory.isDirectory)
        directory.listFiles.filter(_.isFile).toList.foreach { file =>
          if (patternFileName.isEmpty || (patternFileName.isDefined && patternFileName.get.test(file.getName)))
            file.delete()
        }
    }

  def deleteFile(fileName: String): Try[_] =
    Try {
      val plugin = new File(s"$targetDir/$fileName")
      if (plugin.exists && !plugin.isDirectory)
        plugin.delete()
    }

  def browseDirectory(): Try[Seq[SpartaFile]] =
    Try {
      val directory = new File(targetDir)
      if (directory.exists && directory.isDirectory) {
        directory.listFiles.filter(_.isFile).toList.flatMap { file =>
          if (patternFileName.isEmpty || (patternFileName.isDefined && patternFileName.get.test(file.getName)))
            Option(SpartaFile(file.getName, s"$url/${file.getName}", file.getAbsolutePath,
              sizeToMbFormat(file.length())))
          else None
        }
      } else Seq.empty[SpartaFile]
    }

  def uploadFiles(files: Seq[BodyPart]): Try[Seq[SpartaFile]] =
    Try {
      files.flatMap { file =>
        val fileNameOption = file.filename.orElse(file.name.orElse {
          log.warn(s"Is necessary one file name to upload files")
          None
        })
        fileNameOption.flatMap { fileName =>
          if (patternFileName.isEmpty || (patternFileName.isDefined && patternFileName.get.test(fileName))) {
            val localMachineDir = s"$targetDir/$fileName"

            Try(saveFile(file.entity.data.toByteArray, localMachineDir)) match {
              case Success(newFile) =>
                Option(SpartaFile(fileName, s"$url/$fileName", localMachineDir, sizeToMbFormat(newFile.length())))
              case Failure(e) =>
                log.error(s"Error saving file in path $localMachineDir", e)
                None
            }
          } else {
            log.warn(s"$fileName is Not a valid file name")
            None
          }
        }
      }
    }

  private def sizeToMbFormat(size: Long): String = {
    val formatter = new DecimalFormat("####.##")
    s"${formatter.format(size.toDouble / (1024 * 1024))} MB"
  }

  private def saveFile(array: Array[Byte], fileName: String): File = {
    log.info(s"Saving file to: $fileName")
    new File(fileName).getParentFile.mkdirs
    val bos = new BufferedOutputStream(new FileOutputStream(fileName))
    bos.write(array)
    bos.close()
    new File(fileName)
  }

  private def url: String = {
    val host = Try(InetAddress.getLocalHost.getHostName).getOrElse(SpartaConfig.apiConfig.get.getString("host"))
    val port = SpartaConfig.apiConfig.get.getInt("port")

    s"http://$host:$port/${HttpConstant.SpartaRootPath}/$apiPath"
  }
} 

package org.apache.spark.sql.execution.datasources.oap.utils

import java.io.{ByteArrayOutputStream, DataOutputStream, FileOutputStream}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FSDataInputStream, Path}
import org.roaringbitmap.RoaringBitmap

import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.execution.datasources.OapException
import org.apache.spark.sql.execution.datasources.oap.filecache.{BitmapFiberId, FiberCache}
import org.apache.spark.sql.oap.OapRuntime
import org.apache.spark.sql.test.oap.SharedOapContext
import org.apache.spark.util.Utils

// Below are used to test the functionality of OapBitmapWrappedFiberCache class.
class OapBitmapWrappedFiberCacheSuite
  extends QueryTest with SharedOapContext {

  private def loadRbFile(fin: FSDataInputStream, offset: Long, size: Int): FiberCache =
    OapRuntime.getOrCreate.fiberCacheManager.toIndexFiberCache(fin, offset, size)

  test("test the functionality of OapBitmapWrappedFiberCache class") {
    val CHUNK_SIZE = 1 << 16
    val dataForRunChunk = (1 to 9).toSeq
    val dataForArrayChunk = Seq(1, 3, 5, 7, 9)
    val dataForBitmapChunk = (1 to 10000).filter(_ % 2 == 1)
    val dataCombination =
      dataForBitmapChunk ++ dataForArrayChunk ++ dataForRunChunk
    val dataArray =
      Array(dataForRunChunk, dataForArrayChunk, dataForBitmapChunk, dataCombination)
    dataArray.foreach(dataIdx => {
      val dir = Utils.createTempDir()
      val rb = new RoaringBitmap()
      dataIdx.foreach(rb.add)
      val rbFile = dir.getAbsolutePath + "rb.bin"
      rb.runOptimize()
      val rbFos = new FileOutputStream(rbFile)
      val rbBos = new ByteArrayOutputStream()
      val rbDos = new DataOutputStream(rbBos)
      rb.serialize(rbDos)
      rbBos.writeTo(rbFos)
      rbBos.close()
      rbDos.close()
      rbFos.close()
      val rbPath = new Path(rbFile.toString)
      val conf = new Configuration()
      val fin = rbPath.getFileSystem(conf).open(rbPath)
      val rbFileSize = rbPath.getFileSystem(conf).getFileStatus(rbPath).getLen
      val rbFiber = BitmapFiberId(
        () => loadRbFile(fin, 0L, rbFileSize.toInt), rbPath.toString, 0, 0)
      val rbWfc = new OapBitmapWrappedFiberCache(
        OapRuntime.getOrCreate.fiberCacheManager.get(rbFiber))
      rbWfc.init
      val chunkLength = rbWfc.getTotalChunkLength
      val length = dataIdx.size / CHUNK_SIZE
      assert(chunkLength == (length + 1))
      val chunkKeys = rbWfc.getChunkKeys
      assert(chunkKeys(0).toInt == 0)
      rbWfc.setOffset(0)
      val chunk = rbWfc.getIteratorForChunk(0)
      chunk match {
        case RunChunkIterator(rbWfc) => assert(chunk == RunChunkIterator(rbWfc))
        case ArrayChunkIterator(rbWfc, 0) => assert(chunk == ArrayChunkIterator(rbWfc, 0))
        case BitmapChunkIterator(rbWfc) => assert(chunk == BitmapChunkIterator(rbWfc))
        case _ => throw new OapException("unexpected chunk in OapBitmapWrappedFiberCache.")
      }
      rbWfc.release
      fin.close
      dir.delete
    })
  }
} 

package com.hortonworks.spark.atlas.sql

import java.io.{FileOutputStream, PrintWriter}
import java.nio.file.Files

import scala.util.Random
import org.apache.spark.sql.execution.LeafExecNode
import org.apache.spark.sql.execution.command.{ExecutedCommandExec, LoadDataCommand}
import com.hortonworks.spark.atlas.types.external
import com.hortonworks.spark.atlas._
import com.hortonworks.spark.atlas.sql.testhelper.BaseHarvesterSuite
import org.apache.spark.sql.SparkSession

abstract class BaseLoadDataHarvesterSuite
  extends BaseHarvesterSuite {

  protected val sourceTblName = "source_" + Random.nextInt(100000)

  protected override def initializeTestEnvironment(): Unit = {
    prepareDatabase()

    _spark.sql(s"CREATE TABLE $sourceTblName (name string)")
  }

  override protected def cleanupTestEnvironment(): Unit = {
    cleanupDatabase()
  }

  test("LOAD DATA [LOCAL] INPATH path source") {
    val file = Files.createTempFile("input", ".txt").toFile
    val out = new PrintWriter(new FileOutputStream(file))
    out.write("a\nb\nc\nd\n")
    out.close()

    val qe = _spark.sql(s"LOAD DATA LOCAL INPATH '${file.getAbsolutePath}' " +
      s"OVERWRITE INTO  TABLE $sourceTblName").queryExecution
    val qd = QueryDetail(qe, 0L)
    val node = qe.sparkPlan.collect { case p: LeafExecNode => p }
    assert(node.size == 1)
    val execNode = node.head.asInstanceOf[ExecutedCommandExec]

    val entities = CommandsHarvester.LoadDataHarvester.harvest(
      execNode.cmd.asInstanceOf[LoadDataCommand], qd)
    validateProcessEntity(entities.head, _ => {}, inputs => {
      inputs.size should be (1)
      val inputEntity = inputs.head.asInstanceOf[SACAtlasEntityWithDependencies].entity
      inputEntity.getTypeName should be (external.FS_PATH_TYPE_STRING)
      inputEntity.getAttribute("name") should be (file.getAbsolutePath.toLowerCase)
    }, outputs => {
      outputs.size should be (1)
      assertTable(outputs.head, _dbName, sourceTblName, _clusterName, _useSparkTable)
    })
  }
}

class LoadDataHarvesterSuite
  extends BaseLoadDataHarvesterSuite
  with WithHiveSupport {

  override def beforeAll(): Unit = {
    super.beforeAll()
    initializeTestEnvironment()
  }

  override def afterAll(): Unit = {
    cleanupTestEnvironment()
    super.afterAll()
  }

  override def getSparkSession: SparkSession = sparkSession

  override def getDbName: String = "sac"

  override def expectSparkTableModels: Boolean = true
}

class LoadDataHarvesterWithRemoteHMSSuite
  extends BaseLoadDataHarvesterSuite
  with WithRemoteHiveMetastoreServiceSupport {

  override def beforeAll(): Unit = {
    super.beforeAll()
    initializeTestEnvironment()
  }

  override def afterAll(): Unit = {
    cleanupTestEnvironment()
    super.afterAll()
  }

  override def getSparkSession: SparkSession = sparkSession

  override def expectSparkTableModels: Boolean = false

  override def getDbName: String = dbName
} 

package com.intel.ssg.bdt.nlp

import java.io.{DataOutputStream, DataInputStream, FileInputStream, FileOutputStream}

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object CRFFromParsedFile {

  def main(args: Array[String]) {
    val templateFile = "src/test/resources/chunking/template"
    val trainFile = "src/test/resources/chunking/serialized/train.data"
    val testFile = "src/test/resources/chunking/serialized/test.data"

    val conf = new SparkConf().setAppName(s"${this.getClass.getSimpleName}")
    val sc = new SparkContext(conf)

    val templates: Array[String] = scala.io.Source.fromFile(templateFile).getLines().filter(_.nonEmpty).toArray
    val trainRDD: RDD[Sequence] = sc.textFile(trainFile).filter(_.nonEmpty).map(Sequence.deSerializer)

    val model: CRFModel = CRF.train(templates, trainRDD, 0.25, 1, 100, 1E-3, "L1")

    val testRDD: RDD[Sequence] = sc.textFile(testFile).filter(_.nonEmpty).map(Sequence.deSerializer)

    
    val results: RDD[Sequence] = model.setNBest(10)
      .setVerboseMode(VerboseLevel1)
      .predict(testRDD)

    val score = results
      .zipWithIndex()
      .map(_.swap)
      .join(testRDD.zipWithIndex().map(_.swap))
      .map(_._2)
      .map(x => x._1.compare(x._2))
      .reduce(_ + _)
    val total = testRDD.map(_.toArray.length).reduce(_ + _)
    println(s"Prediction Accuracy: $score / $total")

    sc.stop()
  }
} 

package org.oedura.scavro.plugin

import java.io.{FileOutputStream, InputStream}

import sbt._

import scala.io.Source
import scala.util.Random

class TestUtils(workingDir: File) {
  (workingDir / "in").mkdir
  (workingDir / "out").mkdir

  def tmpDir = workingDir
  def tmpPath = workingDir.getAbsolutePath

  private def extractResource(resourceName: String): File = {
    val is: InputStream = getClass.getResourceAsStream(s"/$resourceName")
    val text = Source.fromInputStream(is).mkString
    val os: FileOutputStream = new FileOutputStream(workingDir / "in" / resourceName)
    os.write(text.getBytes)
    os.close()
    is.close()

    workingDir / "in" / resourceName
  }

  lazy val schemaFile: File = extractResource("Number.avsc")
  lazy val protocolFile: File = {
    schemaFile
    extractResource("NumberSystem.avdl")
  }

  def cleanup() = {
    def getRecursively(f: File): Seq[File] = f.listFiles.filter(_.isDirectory).flatMap(getRecursively) ++ f.listFiles

    getRecursively(workingDir).foreach { f =>
      if (!f.delete()) throw new RuntimeException("Failed to delete " + f.getAbsolutePath)
    }
    tmpDir.delete()
  }
}

object TestUtils {
  private val alphabet = ('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')

  def randomFile(dir: File, prefix: String = "", suffix: String = "", maxTries: Int = 100, nameSize: Int = 10): File = {
    def randomFileImpl(triesLeft: Int): String = {
      val testName: String = (1 to nameSize).map(_ => alphabet(Random.nextInt(alphabet.size))).mkString
      if (!(dir / (prefix + testName + suffix)).exists) prefix + testName + suffix
      else if (triesLeft < 0) throw new Exception("Unable to find empty random file path.")
      else randomFileImpl(triesLeft - 1)
    }

    dir / randomFileImpl(maxTries)
  }

  def randomFileName(prefix: String, suffix: String = "", maxTries: Int = 100, nameSize: Int = 10): String = {
    def randomFileNameImpl(triesLeft: Int): String = {
      val testName: String = (1 to nameSize).map(_ => alphabet(Random.nextInt(alphabet.size))).mkString
      if (!file(prefix + testName + suffix).exists) prefix + testName + suffix
      else if (triesLeft < 0) throw new Exception("Unable to find empty random file path.")
      else randomFileNameImpl(triesLeft - 1)
    }

    randomFileNameImpl(maxTries)
  }

  def apply(workingDir: File) = {
    if (workingDir.exists && workingDir.isDirectory) new TestUtils(workingDir)
    else if (!workingDir.exists) {
      val success = workingDir.mkdirs
      if (success) new TestUtils(workingDir)
      else throw new Exception("Cannot initialize working directory")
    } else throw new Exception("Requested directory is occupied by ordinary file")
  }

} 

package akka.persistence.kafka.journal

import java.io.FileOutputStream

import akka.persistence.PersistentRepr
import akka.persistence.serialization.Snapshot
import cats.effect.{IO, Sync}
import com.evolutiongaming.kafka.journal.FromBytes.implicits._
import com.evolutiongaming.kafka.journal.IOSuite._
import com.evolutiongaming.kafka.journal._
import com.evolutiongaming.kafka.journal.util.CatsHelper._
import org.scalatest.funsuite.AsyncFunSuite
import org.scalatest.matchers.should.Matchers
import play.api.libs.json.JsString
import scodec.bits.ByteVector
import TestJsonCodec.instance
import cats.implicits._

import scala.util.Try

class EventSerializerSpec extends AsyncFunSuite with ActorSuite with Matchers {

  for {
    (name, payloadType, payload) <- List(
      ("PersistentRepr.bin",       PayloadType.Binary, Snapshot("binary")),
      ("PersistentRepr.text.json", PayloadType.Json, "text"),
      ("PersistentRepr.json",      PayloadType.Json, JsString("json")))
  } {

    test(s"toEvent & toPersistentRepr, payload: $payload") {
      val persistenceId = "persistenceId"
      val persistentRepr = PersistentRepr(
        payload = payload,
        sequenceNr = 1,
        persistenceId = persistenceId,
        manifest = "manifest",
        writerUuid = "writerUuid")

      val fa = for {
        serializer <- EventSerializer.of[IO](actorSystem)
        event      <- serializer.toEvent(persistentRepr)
        actual     <- serializer.toPersistentRepr(persistenceId, event)
        _          <- Sync[IO].delay { actual shouldEqual persistentRepr }
        payload    <- event.payload.getOrError[IO]("Event.payload is not defined")
        _           = payload.payloadType shouldEqual payloadType
        
        bytes      <- ByteVectorOf[IO](getClass, name)
      } yield {
        payload match {
          case payload: Payload.Binary => payload.value shouldEqual bytes
          case payload: Payload.Text   => payload.value shouldEqual bytes.fromBytes[Try, String].get
          case payload: Payload.Json   => payload.value shouldEqual JsonCodec.summon[Try].decode.fromBytes(bytes).get
        }
      }

      fa.run()
    }
  }

  def writeToFile[F[_] : Sync](bytes: ByteVector, path: String): F[Unit] = {
    Sync[F].delay {
      val os = new FileOutputStream(path)
      os.write(bytes.toArray)
      os.close()
    }
  }
} 

package org.openkoreantext.processor.tools

import java.io.FileOutputStream

import org.openkoreantext.processor.tokenizer.KoreanTokenizer.KoreanToken
import org.openkoreantext.processor.util.KoreanDictionaryProvider._
import org.openkoreantext.processor.OpenKoreanTextProcessor._


object CreateParsingExamples extends Runnable  {
  case class ParsingExample(text: String, parse: Seq[KoreanToken])

  def run {
    System.err.println("Reading the goldenset..")

    val parsedPairs = readFileByLineFromResources("example_chunks.txt").flatMap {
      case line if line.length > 0 =>
        val chunk = line.trim
        val parsed = tokenize(normalize(chunk))
        Some(ParsingExample(chunk, parsed))
      case line => None
    }.toSet


    val outputFile: String = "src/test/resources/org/openkoreantext/processor/util/current_parsing.txt"

    System.err.println("Writing the new goldenset to " + outputFile)

    val out = new FileOutputStream(outputFile)
    parsedPairs.toSeq.sortBy(_.text).foreach {
      p =>
        out.write(p.text.getBytes)
        out.write("\t".getBytes)
        out.write(p.parse.mkString("/").getBytes)
        out.write("\n".getBytes)
    }
    out.close()

    System.err.println("Testing the new goldenset " + outputFile)
  }
} 

package org.openkoreantext.processor.tools

import java.io.FileOutputStream

import org.openkoreantext.processor.normalizer.KoreanNormalizer
import org.openkoreantext.processor.phrase_extractor.KoreanPhraseExtractor.KoreanPhrase
import org.openkoreantext.processor.util.KoreanDictionaryProvider._
import org.openkoreantext.processor.OpenKoreanTextProcessor._


object CreatePhraseExtractionExamples extends Runnable {

  case class PhraseExample(text: String, phrases: Seq[KoreanPhrase])

  def run {
    System.err.println("Reading the sample tweets..")

    val phrasePairs = readFileByLineFromResources("example_tweets.txt").flatMap {
      case line if line.length > 0 =>
        val chunk = line.trim
        val normalized = KoreanNormalizer.normalize(chunk)
        val tokens = tokenize(normalized)
        val phrases = extractPhrases(tokens)
        Some(PhraseExample(chunk, phrases))
      case line => None
    }.toSet


    val outputFile: String = "src/test/resources/org/openkoreantext/processor/util/current_phrases.txt"

    System.err.println("Writing the new phrases to " + outputFile)

    val out = new FileOutputStream(outputFile)
    phrasePairs.toSeq.sortBy(_.text).foreach {
      p =>
        out.write(p.text.getBytes)
        out.write("\t".getBytes)
        out.write(p.phrases.mkString("/").getBytes)
        out.write("\n".getBytes)
    }
    out.close()

    System.err.println("Testing the new phrases " + outputFile)
  }
} 

package org.openkoreantext.processor.tools

import java.io.FileOutputStream

import scala.io.Source


object DeduplicateAndSortDictionaries extends Runnable  {

  private[this] def readWords(filename: String): Set[String] = {
    Source.fromFile(filename)(io.Codec("UTF-8"))
        .getLines()
        .map(_.trim)
        .filter(_.length > 0)
        .toSet
  }

  private val RESOURCES_TO_CLEANUP = Seq(
    "noun/nouns.txt", "noun/entities.txt", "noun/spam.txt",
    "noun/names.txt", "noun/twitter.txt", "noun/lol.txt",
    "noun/slangs.txt", "noun/company_names.txt",
    "noun/foreign.txt", "noun/geolocations.txt", "noun/profane.txt",
    "noun/kpop.txt", "noun/bible.txt",
    "noun/wikipedia_title_nouns.txt", "noun/pokemon.txt", "noun/congress.txt",
    "noun/brand.txt", "noun/fashion.txt", "noun/neologism.txt",

    "substantives/modifier.txt", "substantives/suffix.txt",
    "substantives/family_names.txt", "substantives/given_names.txt",

    "adjective/adjective.txt", "adverb/adverb.txt",

    "auxiliary/determiner.txt", "auxiliary/exclamation.txt", "auxiliary/conjunctions.txt",

    "josa/josa.txt", "typos/typos.txt",

    "verb/eomi.txt", "verb/pre_eomi.txt", "verb/verb.txt", "verb/verb_prefix.txt"
  )

  def run {
    RESOURCES_TO_CLEANUP.foreach {
      f: String =>
        val outputFolder = "src/main/resources/org/openkoreantext/processor/util/"
        System.err.println("Processing %s.".format(f))
        val words = readWords(outputFolder + f).toList.sorted

        val out = new FileOutputStream(outputFolder + f)

        words.foreach {
          word: String => out.write((word + "\n").getBytes)
        }
        out.close()
    }
  }
} 

package org.openkoreantext.processor.tools

import java.io.FileOutputStream

import org.openkoreantext.processor.util.KoreanConjugation._
import org.openkoreantext.processor.util.KoreanDictionaryProvider._


object CreateConjugationExamples extends Runnable  {
  case class ConjugationExample(word: String, conjugations: Seq[String])

  def run {
    System.err.println("Reading the verbs and adjectives..")

    def updateConjugateExamples(file: String, isAdj: Boolean, outputFileName: String) {
      System.err.println("Writing the expansion goldenset in " + outputFileName)

      val outputPath = "src/test/resources/org/openkoreantext/processor/util/" + outputFileName
      val out = new FileOutputStream(outputPath)

      val words = readWordsAsSeq(file)
      val goldenset = words.map(word =>
        ConjugationExample(word, conjugatePredicated(Set(word), isAdj).toSeq.sorted)
      )

      goldenset.foreach {
        c => out.write(
          "%s\t%s\n".format(c.word, c.conjugations.mkString(", ")).getBytes
        )
      }

      out.close()
    }

    updateConjugateExamples("adjective/adjective.txt", isAdj = true, "adj_conjugate.txt")
    updateConjugateExamples("verb/verb.txt", isAdj = false, "verb_conjugate.txt")
  }
} 

package org.apache.spark.graphx

import java.io.File
import java.io.FileOutputStream
import java.io.OutputStreamWriter
import java.nio.charset.StandardCharsets

import org.apache.spark.SparkFunSuite
import org.apache.spark.util.Utils

class GraphLoaderSuite extends SparkFunSuite with LocalSparkContext {

  test("GraphLoader.edgeListFile") {
    withSpark { sc =>
      val tmpDir = Utils.createTempDir()
      val graphFile = new File(tmpDir.getAbsolutePath, "graph.txt")
      val writer = new OutputStreamWriter(new FileOutputStream(graphFile), StandardCharsets.UTF_8)
      for (i <- (1 until 101)) writer.write(s"$i 0\n")
      writer.close()
      try {
        val graph = GraphLoader.edgeListFile(sc, tmpDir.getAbsolutePath)
        val neighborAttrSums = graph.aggregateMessages[Int](
          ctx => ctx.sendToDst(ctx.srcAttr),
          _ + _)
        assert(neighborAttrSums.collect.toSet === Set((0: VertexId, 100)))
      } finally {
        Utils.deleteRecursively(tmpDir)
      }
    }
  }
} 

package org.apache.spark.deploy.worker

import java.io.{File, FileOutputStream, InputStream, IOException}

import scala.collection.JavaConverters._
import scala.collection.Map

import org.apache.spark.SecurityManager
import org.apache.spark.deploy.Command
import org.apache.spark.internal.Logging
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils


  def redirectStream(in: InputStream, file: File) {
    val out = new FileOutputStream(file, true)
    // TODO: It would be nice to add a shutdown hook here that explains why the output is
    //       terminating. Otherwise if the worker dies the executor logs will silently stop.
    new Thread("redirect output to " + file) {
      override def run() {
        try {
          Utils.copyStream(in, out, true)
        } catch {
          case e: IOException =>
            logInfo("Redirection to " + file + " closed: " + e.getMessage)
        }
      }
    }.start()
  }
} 

package org.apache.spark.storage

import java.io.{FileOutputStream, IOException, RandomAccessFile}
import java.nio.ByteBuffer
import java.nio.channels.FileChannel.MapMode

import com.google.common.io.Closeables

import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
import org.apache.spark.util.io.ChunkedByteBuffer


  def put(blockId: BlockId)(writeFunc: FileOutputStream => Unit): Unit = {
    if (contains(blockId)) {
      throw new IllegalStateException(s"Block $blockId is already present in the disk store")
    }
    logDebug(s"Attempting to put block $blockId")
    val startTime = System.currentTimeMillis
    val file = diskManager.getFile(blockId)
    val fileOutputStream = new FileOutputStream(file)
    var threwException: Boolean = true
    try {
      writeFunc(fileOutputStream)
      threwException = false
    } finally {
      try {
        Closeables.close(fileOutputStream, threwException)
      } finally {
         if (threwException) {
          remove(blockId)
        }
      }
    }
    val finishTime = System.currentTimeMillis
    logDebug("Block %s stored as %s file on disk in %d ms".format(
      file.getName,
      Utils.bytesToString(file.length()),
      finishTime - startTime))
  }

  def putBytes(blockId: BlockId, bytes: ChunkedByteBuffer): Unit = {
    put(blockId) { fileOutputStream =>
      val channel = fileOutputStream.getChannel
      Utils.tryWithSafeFinally {
        bytes.writeFully(channel)
      } {
        channel.close()
      }
    }
  }

  def getBytes(blockId: BlockId): ChunkedByteBuffer = {
    val file = diskManager.getFile(blockId.name)
    val channel = new RandomAccessFile(file, "r").getChannel
    Utils.tryWithSafeFinally {
      // For small files, directly read rather than memory map
      if (file.length < minMemoryMapBytes) {
        val buf = ByteBuffer.allocate(file.length.toInt)
        channel.position(0)
        while (buf.remaining() != 0) {
          if (channel.read(buf) == -1) {
            throw new IOException("Reached EOF before filling buffer\n" +
              s"offset=0\nfile=${file.getAbsolutePath}\nbuf.remaining=${buf.remaining}")
          }
        }
        buf.flip()
        new ChunkedByteBuffer(buf)
      } else {
        new ChunkedByteBuffer(channel.map(MapMode.READ_ONLY, 0, file.length))
      }
    } {
      channel.close()
    }
  }

  def remove(blockId: BlockId): Boolean = {
    val file = diskManager.getFile(blockId.name)
    if (file.exists()) {
      val ret = file.delete()
      if (!ret) {
        logWarning(s"Error deleting ${file.getPath()}")
      }
      ret
    } else {
      false
    }
  }

  def contains(blockId: BlockId): Boolean = {
    val file = diskManager.getFile(blockId.name)
    file.exists()
  }
} 

package org.apache.spark.storage

import java.io.{File, FileOutputStream}

import org.scalatest.{BeforeAndAfter, Matchers}

import org.apache.spark._
import org.apache.spark.util.Utils

class TopologyMapperSuite  extends SparkFunSuite
  with Matchers
  with BeforeAndAfter
  with LocalSparkContext {

  test("File based Topology Mapper") {
    val numHosts = 100
    val numRacks = 4
    val props = (1 to numHosts).map{i => s"host-$i" -> s"rack-${i % numRacks}"}.toMap
    val propsFile = createPropertiesFile(props)

    val sparkConf = (new SparkConf(false))
    sparkConf.set("spark.storage.replication.topologyFile", propsFile.getAbsolutePath)
    val topologyMapper = new FileBasedTopologyMapper(sparkConf)

    props.foreach {case (host, topology) =>
      val obtainedTopology = topologyMapper.getTopologyForHost(host)
      assert(obtainedTopology.isDefined)
      assert(obtainedTopology.get === topology)
    }

    // we get None for hosts not in the file
    assert(topologyMapper.getTopologyForHost("host").isEmpty)

    cleanup(propsFile)
  }

  def createPropertiesFile(props: Map[String, String]): File = {
    val testFile = new File(Utils.createTempDir(), "TopologyMapperSuite-test").getAbsoluteFile
    val fileOS = new FileOutputStream(testFile)
    props.foreach{case (k, v) => fileOS.write(s"$k=$v\n".getBytes)}
    fileOS.close
    testFile
  }

  def cleanup(testFile: File): Unit = {
    testFile.getParentFile.listFiles.filter { file =>
      file.getName.startsWith(testFile.getName)
    }.foreach { _.delete() }
  }

} 

import collection.mutable.ArrayBuffer
import java.io.{ObjectInputStream, FileOutputStream, FileInputStream, ObjectOutputStream}

class Person(var name:String) extends Serializable{

  val friends = new ArrayBuffer[Person]()

  def addFriend(friend : Person){
    friends += friend
  }

  override def toString() = {
    var str = "My name is " + name + " and my friends name is "
    friends.foreach(str += _.name + ",")
    str
  }
}


object Test extends App{
  val p1 = new Person("Ivan")
  val p2 = new Person("F2")
  val p3 = new Person("F3")

  p1.addFriend(p2)
  p1.addFriend(p3)
  println(p1)

  val out = new ObjectOutputStream(new FileOutputStream("person.obj"))
  out.writeObject(p1)
  out.close()

  val in =  new ObjectInputStream(new FileInputStream("person.obj"))
  val p = in.readObject().asInstanceOf[Person]
  println(p)
} 

package org.clulab.wm.wmexchanger.utils

import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.FilenameFilter
import java.io.ObjectInputStream
import java.io.ObjectOutputStream
import java.io.PrintWriter

import org.clulab.wm.wmexchanger.utils.Closer.AutoCloser

import scala.io.Source

object FileUtils {

  def appendingPrintWriterFromFile(file: File): PrintWriter = Sinker.printWriterFromFile(file, append = true)

  def appendingPrintWriterFromFile(path: String): PrintWriter = Sinker.printWriterFromFile(path, append = true)

  def printWriterFromFile(file: File): PrintWriter = Sinker.printWriterFromFile(file, append = false)

  def printWriterFromFile(path: String): PrintWriter = Sinker.printWriterFromFile(path, append = false)

  // Output
  def newBufferedOutputStream(file: File): BufferedOutputStream =
    new BufferedOutputStream(new FileOutputStream(file))

  def newBufferedOutputStream(filename: String): BufferedOutputStream =
    newBufferedOutputStream(new File(filename))

  def newAppendingBufferedOutputStream(file: File): BufferedOutputStream =
    new BufferedOutputStream(new FileOutputStream(file, true))

  def newAppendingBufferedOutputStream(filename: String): BufferedOutputStream =
    newAppendingBufferedOutputStream(new File(filename))

  def newObjectOutputStream(filename: String): ObjectOutputStream =
    new ObjectOutputStream(newBufferedOutputStream(filename))

  // Input
  def newBufferedInputStream(file: File): BufferedInputStream =
    new BufferedInputStream(new FileInputStream(file))

  def newBufferedInputStream(filename: String): BufferedInputStream =
    newBufferedInputStream(new File(filename))

  def newObjectInputStream(filename: String): ObjectInputStream =
    new ObjectInputStream(newBufferedInputStream(filename))

  def findFiles(collectionDir: String, extension: String): Seq[File] = {
    val dir = new File(collectionDir)
    val filter = new FilenameFilter {
      def accept(dir: File, name: String): Boolean = name.endsWith(extension)
    }

    val result = Option(dir.listFiles(filter))
        .getOrElse(throw Sourcer.newFileNotFoundException(collectionDir))
    result
  }

  protected def getTextFromSource(source: Source): String = source.mkString

  def getTextFromFile(file: File): String =
    Sourcer.sourceFromFile(file).autoClose { source =>
      getTextFromSource(source)
    }
} 

package org.clulab.wm.eidos.utils

import java.io.BufferedOutputStream
import java.io.FileOutputStream
import java.io.IOException
import java.io.OutputStreamWriter
import java.io.PrintWriter
import java.io.SyncFailedException
import java.nio.charset.StandardCharsets

import org.clulab.wm.eidos.test.TestUtils._
import org.clulab.wm.eidos.utils.Closer.AutoCloser

class TestDiskFull extends Test {

  def test1 = {
    val file = "/E:/full.dat"
    var i = 0

    try {
      val text1 = "The quick brown fox jumped over the lazy dog."
      val text = text1 + text1

      for (limit <- 1 until 400) {
        val fos = new FileOutputStream(file)
        val osw = new OutputStreamWriter(new BufferedOutputStream(fos), StandardCharsets.UTF_8.toString)
        i = 0

        new PrintWriter(osw).autoClose { pw =>
          while (i < limit) {
            pw.print(text)
            i += 1
            //          pw.flush()
            //          osw.flush()
            //          fos.flush()
            fos.getFD.sync()
          }
        }
      }
    }
    catch {
      case exception: SyncFailedException =>
        println(s"Synchronization failed for file $file at $i")
        println("Exiting with code -2 on assumption that the disk is full")
        System.exit(-2)
      case exception: IOException =>
        println(s"IO failed for file $file at $i")
        println("Exiting with code -2 on assumption that the disk is full")
        System.exit(-2)
      case exception: Exception =>
        println(s"Exception for file $file at $i")
        exception.printStackTrace()
      case throwable: Throwable =>
        println(s"Throwable for file $file at $i")
        throwable.printStackTrace()
    }
  }

//  test1
} 

package pl.edu.pw.ii.zsibio.dwh.benchmark.utils

import java.io.{File, FileOutputStream, PrintWriter}
import java.util.Calendar

import pl.edu.pw.ii.zsibio.dwh.benchmark.dao.{ConnectDriver, EngineConnection, QueryResult}
import net.jcazevedo.moultingyaml._
import net.jcazevedo.moultingyaml.DefaultYamlProtocol
import net.jcazevedo.moultingyaml.DefaultYamlProtocol._
import org.apache.log4j.Logger
import pl.edu.pw.ii.zsibio.dwh.benchmark.dao.ConnectDriver.Value
import pl.edu.pw.ii.zsibio.dwh.benchmark.utils.QueryType.QueryType


case class Query(queryId:String, queryType:String, queryEngine:String, storageFormat:String,queryDesc:String,
                 statement:String)

object QueryType extends Enumeration {

  type QueryType = Value
  val SELECT, CREATE, UPDATE  = Value


}
object QueryExecutorWithLogging {
  val log = Logger.getLogger("pl.edu.pw.ii.zsibio.dwh.benchmark.utils.QueryExecutorWithLogging")
  object QueryYamlProtocol extends DefaultYamlProtocol {
    implicit val queryFormat = yamlFormat6(Query)
  }


  def runStatement(query: Query, conn:EngineConnection, logFile:String, dryRun: Boolean) = {
    log.info(s"Running ${query.queryId} ... using ${query.queryEngine} engine")
    log.debug(s"Executing query: ${query.statement}")
    query.queryType.toLowerCase() match {
      case "select" => logQuery(conn, query, logFile, dryRun)
      case _ => conn.executeUpdate(query.statement.toLowerCase)
    }


  }

  def parseQueryYAML(file:String,storageType:String,connString:String, kuduMaster:String, dbName:String, ifExplain:Boolean = false)  : Query ={
    log.info(s"Parsing ${file}")
    val lines = scala.io.Source.fromFile(file).mkString
    val yml = lines.stripMargin.parseYaml
    import QueryYamlProtocol._
    queryPreprocess(yml.convertTo[Query], storageType, connString, kuduMaster, dbName, ifExplain)

  }

  private def logQuery(conn:EngineConnection, query: Query, logFile:String, dryRun:Boolean) ={
    val rs = conn.executeQuery(query.statement.toLowerCase,true)
    //rs.rs.next()
    val result = s"${Calendar.getInstance().getTime().toString},${query.queryId}," +
      s"${query.queryEngine},${query.storageFormat},${rs.timing.get.getTiming()},${dryRun.toString}\n"
    log.info(s"Result: ${result}")
    val writer = new PrintWriter(new FileOutputStream(new File(logFile),true))
    writer.write(result)
    writer.flush()
    writer.close()
  }

  private def queryPreprocess(query: Query, storageType: String, connString: String, kuduMaster: String, dbName: String, ifExplain: Boolean) = {
    def replaceVars(property:String) ={
      property
        .replaceAll("\\{\\{DATA_FORMAT\\}\\}",storageType.toLowerCase)
        .replaceAll("\\{\\{DB_NAME\\}\\}",dbName.toLowerCase)
        .replaceAll("\\{\\{KUDU_MASTER\\}\\}",kuduMaster )
        .replaceAll("\\{\\{IF_EXPLAIN\\}\\}", if(ifExplain) "EXPLAIN " else "")
        .replaceAll("\\{\\{PERCENTILE_APPROX\\}\\}", if(query.queryEngine.toLowerCase=="presto") "approx_percentile" else "percentile_approx")

    }
    query.copy(
      queryId = replaceVars(query.queryId),
      queryDesc = replaceVars(query.queryDesc),
      storageFormat = replaceVars(query.storageFormat),
      statement = replaceVars(query.statement.replaceAll(",",",\n").replaceAll("\\(","\\(  "))
    )
  }

} 

package io.github.mandar2812.dynaml.utils.sumac

import collection._
import java.util.Properties
import java.io.{FileOutputStream, File, FileInputStream, BufferedInputStream}

import collection.JavaConverters._


trait PropertiesConfig extends ExternalConfig {
  self: Args =>

  var propertyFile: File = _

  abstract override def readArgs(originalArgs: Map[String,String]): Map[String,String] = {
    parse(originalArgs, false)

    val props = new Properties()
    if (propertyFile != null) {
      val in = new BufferedInputStream(new FileInputStream(propertyFile))
      props.load(in)
      in.close()
    }
    //append args we read from the property file to the args from the command line, and pass to next trait
    super.readArgs(ExternalConfigUtil.mapWithDefaults(originalArgs,props.asScala))
  }

  abstract override def saveConfig() {
    PropertiesConfig.saveConfig(this, propertyFile)
    super.saveConfig()
  }

}

object PropertiesConfig {
  def saveConfig(args: Args, propertyFile: File) {
    val props = new Properties()
    args.getStringValues.foreach{case(k,v) => props.put(k,v)}
    val out = new FileOutputStream(propertyFile)
    props.store(out, "")
    out.close()
  }
} 

package de.zalando.swagger

import java.io.{File, FileOutputStream}

import scala.io.Source


trait ExpectedResults {

  val resourcesPath = "swagger-parser/src/test/resources/"

  def expectationsFolder: String = "/expected_results/"

  def dump(result: String, file: File, suffix: String): Unit = {
    if (result.nonEmpty) {
      val newFile = target(file, suffix)
      newFile.getParentFile.mkdirs()
      newFile.delete()
      newFile.createNewFile()
      val out = new FileOutputStream(newFile)
      out.write(result.getBytes)
      out.close()
    }
  }

  def asInFile(file: File, suffix: String): String = {
    val expectedFile = target(file, suffix)
    if (expectedFile.canRead) {
      val src = Source.fromFile(expectedFile)
      val result = src.getLines().mkString("\n")
      src.close()
      result
    } else
      ""
  }

  def target(file: File, suffix: String): File =
    new File(file.getParentFile.getParent + expectationsFolder + file.getName + "." + suffix)

  def clean(str: String): String = str.split("\n").map(_.trim).mkString("\n")
} 

package de.zalando

import java.io.{File, FileOutputStream}

import de.zalando.apifirst.util.ScalaPrinter
import de.zalando.model._

import scala.io.Source


trait ExpectedResults {

  val model = Seq[WithModel](
    additional_properties_yaml,
    basic_polymorphism_yaml,
    nested_arrays_yaml,
    nested_options_yaml,
    basic_extension_yaml,
    expanded_polymorphism_yaml,
    nested_objects_yaml,
    options_yaml
  )
  val examples = Seq[WithModel](
    basic_auth_api_yaml,
    cross_spec_references_yaml,
    echo_api_yaml,
    error_in_array_yaml,
    form_data_yaml,
    full_petstore_api_yaml,
    hackweek_yaml,
    heroku_petstore_api_yaml,
    instagram_api_yaml,
    minimal_api_yaml,
    nakadi_yaml,
    security_api_yaml,
    simple_petstore_api_yaml,
    split_petstore_api_yaml,
    string_formats_yaml,
    type_deduplication_yaml,
    uber_api_yaml
  )
  val validations = Seq[WithModel](
    nested_arrays_validation_yaml,
    nested_objects_validation_yaml,
    nested_options_validation_yaml,
    numbers_validation_yaml,
    string_formats_validation_yaml
  )

  val resourcesPath = "play-scala-generator/src/test/resources/"

  def expectationsFolder: String = "/expected_results/"

  def dump(result: String, name: String, suffix: String): Unit = {
    if (result.nonEmpty) {
      val newFile = target(name, suffix)
      newFile.getParentFile.mkdirs()
      newFile.delete()
      newFile.createNewFile()
      val out = new FileOutputStream(newFile)
      out.write(result.getBytes)
      out.close()
    }
  }

  def asInFile(name: String, suffix: String): String = {
    val expectedFile = target(name, suffix)
    if (expectedFile.canRead) {
      val src = Source.fromFile(expectedFile)
      val result = src.getLines().mkString("\n")
      src.close()
      result
    } else
      ""
  }

  def target(name: String, suffix: String): File =
    new File(resourcesPath + expectationsFolder + name + "." + suffix)

  def clean(str: String): String = str.split("\n").map(_.trim).filter(_.nonEmpty).mkString("\n")

  def nameFromModel(ast: WithModel): String = ScalaPrinter.nameFromModel(ast)

} 

package ru.stachek66.tools

import java.io.{IOException, File, FileOutputStream}

import org.apache.commons.compress.archivers.ArchiveInputStream
import org.apache.commons.io.IOUtils
import ru.stachek66.nlp.mystem.Properties


trait Decompressor {

  def traditionalExtension: String

  def unpack(src: File, dst: File): File

  @throws(classOf[IOException])
  private[tools] def copyUncompressedAndClose(stream: ArchiveInputStream, dest: File): File = {
    // must be read
    val entry = stream.getNextEntry
    if (entry.isDirectory)
      throw new IOException("Decompressed entry is a directory (unexpectedly)")

    val os = new FileOutputStream(dest)

    try {
      IOUtils.copy(stream, os)
    } finally {
      os.close()
      stream.close()
    }
    dest
  }
}

object Decompressor {
  def select: Decompressor =
    if (Properties.CurrentOs.contains("win")) Zip else TarGz
} 

package io.bullet.borer.output

import java.io.{BufferedOutputStream, File, FileOutputStream}

import io.bullet.borer.Output.ToValueProvider

trait ToFileOutput { this: ToOutputStreamOutput =>

  implicit object ToFileProvider extends ToValueProvider[File] {
    type Out = ToFile
    def apply(file: File, bufferSize: Int, allowBufferCaching: Boolean) = new ToFile(file, bufferSize)
  }

  
  final class ToFile(file: File, bufferSize: Int)
      extends ToOutputStreamBase(new BufferedOutputStream(new FileOutputStream(file), bufferSize), bufferSize) {
    type Self   = ToFile
    type Result = File

    def result(): File = {
      outputStream.close()
      file
    }
  }
} 

package org.grapheco.server.util

import java.io.{File, FileOutputStream, StringWriter}
import java.util.Properties

import cn.pidb.blob.Blob
import cn.pidb.engine.blob.{BlobIO, InlineBlob, RemoteBlob}
import org.apache.velocity.app.VelocityEngine
import org.apache.velocity.tools.ToolManager
import org.apache.velocity.tools.config.DefaultKey
import org.neo4j.values.storable.{BlobValue, ValueWriter}
import org.springframework.util.ClassUtils
import scala.collection.JavaConversions
import java.io.FileOutputStream
import java.io.IOException


object VelocityUtils {
  val pro = new Properties();
  val toolManager = new ToolManager();
  toolManager.configure("tools.xml");

  pro.setProperty("input.encoding", "UTF-8");
  pro.setProperty("output.encoding", "UTF-8");
  val ve = new VelocityEngine(pro);
  val props = new Properties()
  props.put("runtime.log.logsystem.class", "org.apache.velocity.runtime.log.SimpleLog4JLogSystem")
  props.put("runtime.log.logsystem.log4j.category", "velocity")
  props.put("runtime.log.logsystem.log4j.logger", "velocity")
  ve.init(props)
  def parse(expr: String, context: Map[String, Any]): Any = {
    val vc = toolManager.createContext();
    val writer = new StringWriter();

    context.foreach(kv => vc.put(kv._1,
      //is a scala Map?
      if (kv._2.isInstanceOf[Map[_, _]]) {
        JavaConversions.mapAsJavaMap(kv._2.asInstanceOf[Map[_, _]])
      }
      else {
        kv._2
      }));

    try {
      if (expr.startsWith("=")) {
        val expr1 = expr.substring(1);
        ve.evaluate(vc, writer, "", s"#set($$__VAR=$expr1)");
        var value = vc.get("__VAR");
        //if is a blob
        if(value.isInstanceOf[Blob]){
          //get blob
          var result:String = ""
          try {
            val data = value.asInstanceOf[Blob].toBytes()
            val path = ClassUtils.getDefaultClassLoader.getResource("").getPath.replace("/WEB-INF/classes","") + "static/"
            val tool = new FileSystemTool()
            result = tool.filesave(data,path, System.currentTimeMillis.toString+".jpg")
          }
          catch{
            case e:Throwable =>
              print(e.toString)
          }
          //TODO url
          return "http://localhost:9999/graphserver/static/"+result
        }
        return value
      }
      else {
        ve.evaluate(vc, writer, "", expr);
        writer.getBuffer.toString.trim
      }
    }
    catch {
      case e: Throwable =>
        throw new WrongExpressionException(expr, e);
    }
  }
}

class WrongExpressionException(msg: String, e: Throwable) extends RuntimeException(msg, e) {

}

@DefaultKey("fileTool")
class FileSystemTool {
  def exists(path: String) = new File(path).exists();



  @throws[IOException]
  def filesave(file: Array[Byte], filePath: String, fileName: String): String = { //目标目录
    val targetfile = new File(filePath)
    if (!targetfile.exists) targetfile.mkdirs
    //二进制流写入
    val out = new FileOutputStream(filePath + fileName)
    out.write(file)
    out.flush()
    out.close()
    return  fileName
  }
} 

package com.kakao.cuesheet.deps

import java.io.{BufferedOutputStream, File, FileOutputStream, IOException}
import java.net.{URL, URLDecoder}
import java.nio.file.{Files, Paths}
import java.util.zip.{ZipEntry, ZipOutputStream}

import com.kakao.mango.io.FileSystems
import com.kakao.mango.logging.Logging
import com.kakao.shaded.guava.io.Files.createTempDir

sealed trait DependencyNode {
  def path: String
}

case class ManagedDependency(group: String, artifact: String, classifier: String = "jar")

case class ManagedDependencyNode(
  path: String,
  group: String,
  artifact: String,
  classifier: String,
  version: String,
  children: Seq[ManagedDependency]
) extends DependencyNode {
  def key = ManagedDependency(group, artifact, classifier)
}

case class DirectoryDependencyNode(path: String) extends DependencyNode with Logging {
  lazy val compressed: UnmanagedDependencyNode = {
    val tmpdir = createTempDir()
    val jar = new File(s"${tmpdir.getAbsolutePath}/local-${tmpdir.getName}.jar")
    val root = Paths.get(path)

    val output = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(jar)))
    var count = 0

    FileSystems.entries(root).foreach { path =>
      if (resourceExtensions.exists(path.toString.endsWith)) {
        val entry = new ZipEntry(root.relativize(path).toString)
        output.putNextEntry(entry)
        try {
          Files.copy(path, output)
          count += 1
        } catch {
          case e: IOException => logger.warn(s"skipping $path due to an IOException: ${e.getMessage}")
        }
        output.closeEntry()
      }
    }

    output.close()

    logger.debug(s"Successfully zipped $count files in $path into $jar")

    UnmanagedDependencyNode(jar.getAbsolutePath)
  }
}

case class JavaRuntimeDependencyNode(path: String) extends DependencyNode
case class UnmanagedDependencyNode(path: String) extends DependencyNode

object DependencyNode {

  val resolver = new ChainedArtifactResolver(
    new IvyPathArtifactResolver,
    new IvyOriginalPathArtifactResolver,
    new MavenPathArtifactResolver,
    new GradlePathArtifactResolver,
    new JavaRuntimeResolver,
    new MavenMetadataArtifactResolver,
    new UnmanagedJarResolver
  )

  def resolve(url: URL): DependencyNode = {
    if (url.getProtocol != "file") {
      throw new IllegalArgumentException("non-file dependency is not supported")
    }

    val path = URLDecoder.decode(url.getFile, "UTF-8")
    val file = new File(path)
    if (file.isDirectory) {
      return DirectoryDependencyNode(file.getAbsolutePath)
    }

    if (!file.isFile || !file.canRead) {
      throw new IllegalArgumentException(s"$path is not a file or readable")
    }

    DependencyNode.resolver.resolve(file.getAbsolutePath) match {
      case Some(node) => node
      case None => throw new IllegalArgumentException(s"Could not determine the dependency of $path")
    }
  }
} 

package com.kakao.cuesheet.examples.util

import java.io.FileOutputStream

import com.google.common.io.{ByteStreams, Files}

import scala.util.control.NonFatal

object ExampleData {
  lazy val path: String = {
    try {
      val resource = "data.tsv"
      val tmpfile = Files.createTempDir().getAbsolutePath + resource
      val input = getClass.getResourceAsStream(resource)
      val output = new FileOutputStream(tmpfile)
      ByteStreams.copy(input, output)
      input.close()
      output.close()
      tmpfile
    } catch {
      case NonFatal(e) =>
        throw new RuntimeException("Could not copy example data file to temp directory", e)
    }
  }
} 

package com.aol.sbt.sonar

import java.io.{File, FileOutputStream}
import java.util.Properties

import org.sonar.runner.Main
import sbt.Keys._
import sbt._

import scala.collection.JavaConversions
;

object SonarRunnerPlugin extends AutoPlugin {

  object autoImport {
    val sonarProperties = settingKey[Map[String, String]]("SonarRunner configuration properties. See http://docs.codehaus.org/display/SONAR/Analysis+Parameters.")
    val sonar = taskKey[Unit]("Runs Sonar agent")
    val generateSonarConfiguration = taskKey[File]("Generates Sonar configuration")
    val sonarRunnerOptions = settingKey[Seq[String]]("Extra options for sonar runner")
  }

  import com.aol.sbt.sonar.SonarRunnerPlugin.autoImport._

  override def projectSettings: Seq[Setting[_]] = Seq(
    generateSonarConfiguration := makeConfiguration(target.value + "/sonar-project.properties", sonarProperties.value),
    sonarProperties := Map(
      "sonar.projectName" -> name.value,
      "sonar.projectVersion" -> version.value,
      "sonar.projectKey" -> "%s:%s".format(organization.value, name.value),
      "sonar.binaries" -> filePathsToString(Seq((classDirectory in Compile).value)),
      "sonar.sources" -> filePathsToString((unmanagedSourceDirectories in Compile).value),
      "sonar.tests" -> filePathsToString((unmanagedSourceDirectories in Test).value),
      "sonar.projectBaseDir" -> file(".").absolutePath,
      "sonar.sourceEncoding" -> "UTF-8",
      "sonar.host.url" -> "http://localhost:9000",
      "sonar.jdbc.url" -> "jdbc:mysql://localhost:3306/sonar",
      "sonar.jdbc.username" -> "sonar",
      "sonar.jdbc.password" -> "sonar"
    ),
    sonarRunnerOptions := Seq.empty,
    sonar := {
      lazy val logger: TaskStreams = streams.value
      runSonarAgent(generateSonarConfiguration.value, logger, sonarRunnerOptions.value)
    }
  )

  def runSonarAgent(configFile: File, logger: TaskStreams, sonarRunnerOptions: Seq[String]) = {
    logger.log.info("**********************************")
    logger.log.info("Publishing reports to SonarQube...")
    logger.log.info("**********************************")
    Main.main(Array[String]("-D", "project.settings=" + configFile.getCanonicalPath, "-D", "project.home=" + file(".").absolutePath) ++ sonarRunnerOptions)
  }

  private[this] def filePathsToString(files: Seq[File]) = files.filter(_.exists).map(_.getAbsolutePath).toSet.mkString(",")

  private[this] def makeConfiguration(configPath: String, props: Map[String, String]): File = {
    val p = new Properties()
    p.putAll(JavaConversions.mapAsJavaMap(props))
    p.store(new FileOutputStream(configPath), null)
    file(configPath)
  }
} 

package org.apache.daffodil.tdml

import java.io.File
import org.junit.Assert.assertEquals
import org.junit.Test
import java.io.FileOutputStream
import org.apache.daffodil.api.URISchemaSource
import org.junit.Before
import scala.Right

class TestSchemaCache {

  object SCache extends SchemaCache[Null, Null]

  var compileCount = 0
  var originalUSS: URISchemaSource = null
  var newUSS: URISchemaSource = null
  var tempFile: File = null

  @Before def setup: Unit = {
    compileCount = 0
    SCache.resetCache
    tempFile = java.io.File.createTempFile("tdml", "tdml")
    tempFile.deleteOnExit()
    touchFile()
    val originalURI = tempFile.toURI
    originalUSS = URISchemaSource(originalURI)
    val newURI = tempFile.toURI
    newUSS = URISchemaSource(newURI)
  }

  
  def touchFile(): Unit = {
    val startingModTime = tempFile.lastModified()
    var iters = 0
    while (tempFile.lastModified() <= startingModTime) {
      iters += 1
      Thread.sleep(100)
      val os = new FileOutputStream(tempFile)
      os.write(0)
      os.flush()
      os.close()
    }
    // println("iters = " + iters)
  }

  def compileTheSchema(uss: URISchemaSource): Unit = {
    SCache.compileAndCache(uss, false, false, null, null) {
      compileCount += 1
      uss.newInputSource().getByteStream().close()
      Right(null)
    }
  }

  @Test def testReset: Unit = {
    compileTheSchema(originalUSS)
    SCache.resetCache
  }

  @Test def testSameFileCompiledOnce: Unit = {
    compileTheSchema(originalUSS)
    assertEquals(1, compileCount)
    compileTheSchema(newUSS) // file has not been touched, so this should hit the cache.
    assertEquals(1, compileCount)
  }

  @Test def testSameFileCompiledTwice: Unit = {
    compileTheSchema(originalUSS)
    assertEquals(1, compileCount)

    touchFile()

    compileTheSchema(newUSS) // file has changed
    assertEquals(2, compileCount)

  }

} 

package applications.analysis

import java.io.{BufferedWriter, FileOutputStream, OutputStreamWriter}

import functions.segment.Segmenter
import org.apache.log4j.{Level, Logger}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Row, SparkSession}


object StarsAnalysisDemo {
  def main(args: Array[String]) {
    Logger.getLogger("org").setLevel(Level.WARN)

    val spark = SparkSession
      .builder
      .master("local[2]")
      .appName("Stars Analysis Demo")
      .getOrCreate()

    val filePath = "E:/data/chinaNews/entertainment.txt"


    // 加载数据，并保留年份和内容字段，并对内容字段进行过滤
    import spark.implicits._
    val data = spark.sparkContext.textFile(filePath).flatMap { line =>
      val tokens: Array[String] = line.split("\u00ef")
      if (tokens.length > 3) {
        var year: String = tokens(2).split("-")(0)
        if (tokens(2).contains("年")) year = tokens(2).split("年")(0)

        var content = tokens(3)
        if (content.length > 22 && content.substring(0, 20).contains("日电")) {
          content = content.substring(content.indexOf("日电") + 2, content.length).trim
        }

        if (content.startsWith("(")) content = content.substring(content.indexOf(")") + 1, content.length)
        if (content.length > 20 && content.substring(content.length - 20, content.length).contains("记者")) {
          content = content.substring(0, content.lastIndexOf("记者")).trim
        }

        Some(year, content)
      } else None
    }.toDF("year", "content")

    // 分词，去除长度为1的词，每个词保留词性
    val segmenter = new Segmenter()
      .isAddNature(true)
      .isDelEn(true)
      .isDelNum(true)
      .setMinTermLen(2)
      .setMinTermNum(5)
      .setSegType("StandardSegment")
      .setInputCol("content")
      .setOutputCol("segmented")
    val segDF: DataFrame = segmenter.transform(data)
    segDF.cache()

    val segRDD: RDD[(Int, Seq[String])] = segDF.select("year", "segmented").rdd.map { case Row(year: String, terms: Seq[String]) =>
      (Integer.parseInt(year), terms)
    }

    val result: Array[String] = segRDD.map(line => line._1.toString + "\u00ef" + line._2.mkString(",")).collect()
    val writer: BufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream("E:/entertainment_seg.txt")))
    result.foreach(line => writer.write(line + "\n"))
    writer.close()

    // 统计2016出现在新闻中最多的明星
    val stars2016 = segRDD.filter(_._1 == 2016)
      .flatMap { case (year: Int, termStr: Seq[String]) =>
        val person = termStr
          .map(term => (term.split("/")(0), term.split("/")(1)))
          .filter(_._2.equalsIgnoreCase("nr"))
          .map(term => (term._1, 1L))

        person
      }
      .reduceByKey(_ + _)
      .sortBy(_._2, ascending = false)

    segDF.unpersist()

    stars2016.take(100).foreach(println)

    spark.stop()
  }
} 

import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
import java.io.{ByteArrayOutputStream, File, FileInputStream, FileOutputStream, InputStream}

object ZipUtil {

  def addToZip(sourceZip: File, destZip: File, extra: Seq[(String, Array[Byte])]): Unit = {
    
    val is = new FileInputStream(sourceZip)
    val os = new FileOutputStream(destZip)
    val bootstrapZip = new ZipInputStream(is)
    val outputZip = new ZipOutputStream(os)

    def readFullySync(is: InputStream) = {
      val buffer = new ByteArrayOutputStream
      val data = Array.ofDim[Byte](16384)

      var nRead = is.read(data, 0, data.length)
      while (nRead != -1) {
        buffer.write(data, 0, nRead)
        nRead = is.read(data, 0, data.length)
      }

      buffer.flush()
      buffer.toByteArray
    }

    def zipEntries(zipStream: ZipInputStream): Iterator[(ZipEntry, Array[Byte])] =
      new Iterator[(ZipEntry, Array[Byte])] {
        private var nextEntry = Option.empty[ZipEntry]
        private def update() =
          nextEntry = Option(zipStream.getNextEntry)

        update()

        def hasNext = nextEntry.nonEmpty
        def next() = {
          val ent = nextEntry.get
          val data = readFullySync(zipStream)

          update()

          (ent, data)
        }
      }

    val extraNames = extra.map(_._1).toSet

    for ((ent, data) <- zipEntries(bootstrapZip) if !extraNames(ent.getName)) {

      // Same workaround as https://github.com/spring-projects/spring-boot/issues/13720
      // (https://github.com/spring-projects/spring-boot/commit/a50646b7cc3ad941e748dfb450077e3a73706205#diff-2ff64cd06c0b25857e3e0dfdb6733174R144)
      ent.setCompressedSize(-1L)

      outputZip.putNextEntry(ent)
      outputZip.write(data)
      outputZip.closeEntry()
    }

    for ((dest, data) <- extra) {
      outputZip.putNextEntry(new ZipEntry(dest))
      outputZip.write(data)
      outputZip.closeEntry()
    }

    outputZip.close()

    is.close()
    os.close()

  }

} 

package com.johnsnowlabs.nlp.annotators.ner.dl

import java.io.{BufferedOutputStream, File, FileOutputStream}
import java.nio.file.Paths

import com.johnsnowlabs.nlp.util.io.ResourceHelper
import org.apache.commons.lang.SystemUtils
import org.apache.spark.SparkFiles
import org.apache.spark.sql.SparkSession
import org.tensorflow.TensorFlow

object LoadsContrib {
  @transient var loadedToCluster = false
  @transient var loadedToTensorflow = false

  private lazy val lib1 = "_sparse_feature_cross_op.so"
  private lazy val lib2 = "_lstm_ops.so"

  private def resourcePath(os: String, lib: String) = "ner-dl/"+os+"/"+lib

  
    if (!LoadsContrib.loadedToCluster && contribPaths.isDefined) {
      LoadsContrib.loadedToCluster = true
      spark.sparkContext.addFile(copyResourceToTmp(contribPaths.get._1).getPath)
      spark.sparkContext.addFile(copyResourceToTmp(contribPaths.get._2).getPath)
    }
  }

  def loadContribToTensorflow(): Unit = {
    if (!LoadsContrib.loadedToTensorflow && contribPaths.isDefined) {
      LoadsContrib.loadedToTensorflow = true
      val fp1 = SparkFiles.get(getFileName(contribPaths.get._1))
      val fp2 = SparkFiles.get(getFileName(contribPaths.get._2))
      if (new File(fp1).exists() && new File(fp2).exists()) {
        TensorFlow.loadLibrary(fp1)
        TensorFlow.loadLibrary(fp2)
      }
    }
  }

} 

package faces.utils

import java.io.{File, FileInputStream, FileOutputStream}

import scalismo.faces.color.{ColorSpaceOperations, RGBA}
import scalismo.faces.image.BufferedImageConverter
import scalismo.faces.io.{MeshIO, PixelImageIO}
import scalismo.faces.mesh.{ColorNormalMesh3D, TextureMappedProperty}
import scalismo.geometry.{Point, _2D}
import scalismo.mesh.{MeshSurfaceProperty, TriangleCell, TriangleList}
import spray.json.JsObject

import scala.reflect.ClassTag
import scala.util.Try
import spray.json._

object TextureMappedPropertyIO extends App {

  import scalismo.faces.io.renderparameters.RenderParameterJSONFormatV2._

  import scalismo.faces.io.RenderParameterIO._
  def read[A: ClassTag](directory: String, stem: String)(implicit converter: BufferedImageConverter[A], ops: ColorSpaceOperations[A]): TextureMappedProperty[A] = read[A](new File(directory+"/"+stem+".json"),new File(directory+"/"+stem+".png"))

  def read[A: ClassTag](mappingFile: File, imageFile: File)(implicit converter: BufferedImageConverter[A],  ops: ColorSpaceOperations[A]) : TextureMappedProperty[A] = {

    import scalismo.faces.io.RenderParameterIO.readASTFromStream

    val fields = readASTFromStream(new FileInputStream(mappingFile)).asJsObject.fields
    val triangles = fields("triangles").convertTo[IndexedSeq[TriangleCell]]
    val triangulation = TriangleList(triangles)

    val textureMapping = fields("textureMapping").convertTo[MeshSurfaceProperty[Point[_2D]]]

    val texture = PixelImageIO.read[A](imageFile).get

    TextureMappedProperty[A](triangulation, textureMapping, texture)
  }

  def write[A:ClassTag](textureMappedProperty: TextureMappedProperty[A], directory: String, stem: String)(implicit converter: BufferedImageConverter[A]): Try[Unit] = Try {
    val writeImage = PixelImageIO.write(
      textureMappedProperty.texture,
      new File(directory+"/"+stem+".png")
    ).get

    val mapping = JsObject(
      "triangles" -> textureMappedProperty.triangulation.triangles.toJson,
      "textureMapping" -> textureMappedProperty.textureMapping.toJson,
      "@type" -> "TextureMappedProperty".toJson
    )

    val os = new FileOutputStream(new File(directory+"/"+stem+".json"))
    writeASTToStream(mapping, os)
  }

} 

// Copyright (C) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in project root for information.

package com.microsoft.ml.spark.core.env

import java.io.File
import java.nio.file.{Files, StandardCopyOption}

import scala.io.{BufferedSource, Source}

object FileUtilities {

  def join(folders: String*): File = {
    folders.tail.foldLeft(new File(folders.head)) { case (f, s) => new File(f, s) }
  }

  def join(base: File, folders: String*): File = {
    folders.foldLeft(base) { case (f, s) => new File(f, s) }
  }

  // Same for StandardOpenOption
  type StandardOpenOption = java.nio.file.StandardOpenOption
  object StandardOpenOption {
    import java.nio.file.{StandardOpenOption => S}
    val APPEND = S.APPEND
    val CREATE = S.CREATE
  }

  def allFiles(dir: File, pred: (File => Boolean) = null): Array[File] = {
    def loop(dir: File): Array[File] = {
      val (dirs, files) = dir.listFiles.sorted.partition(_.isDirectory)
      (if (pred == null) files else files.filter(pred)) ++ dirs.flatMap(loop)
    }
    loop(dir)
  }

  // readFile takes a file name or a File, and function to extract a value from
  // BufferedSource which defaults to _.mkString; performs the read, closes the
  // source, and returns the result
  def readFile[T](file: File, read: BufferedSource => T): T = {
    val i = Source.fromFile(file)
    try read(i) finally i.close
  }
  def readFile(file: File): String = readFile(file, _.mkString)

  def writeFile(file: File, stuff: Any, flags: StandardOpenOption*): Unit = {
    Files.write(file.toPath, stuff.toString.getBytes(), flags: _*)
    ()
  }

  def copyFile(from: File, toDir: File, overwrite: Boolean = false): Unit = {
    Files.copy(from.toPath, (new File(toDir, from.getName)).toPath,
               (if (overwrite) Seq(StandardCopyOption.REPLACE_EXISTING)
                else Seq()): _*)
    ()
  }

  // Perhaps this should move into a more specific place, not a generic file utils thing
  def zipFolder(dir: File, out: File): Unit = {
    import java.io.{BufferedInputStream, FileInputStream, FileOutputStream}
    import java.util.zip.{ZipEntry, ZipOutputStream}
    val bufferSize = 2 * 1024
    val data = new Array[Byte](bufferSize)
    val zip = new ZipOutputStream(new FileOutputStream(out))
    val prefixLen = dir.getParentFile.toString.length + 1
    allFiles(dir).foreach { file =>
      zip.putNextEntry(new ZipEntry(file.toString.substring(prefixLen).replace(java.io.File.separator, "/")))
      val in = new BufferedInputStream(new FileInputStream(file), bufferSize)
      var b = 0
      while (b >= 0) { zip.write(data, 0, b); b = in.read(data, 0, bufferSize) }
      in.close()
      zip.closeEntry()
    }
    zip.close()
  }

} 

package org.argus.jawa.core.util

import java.io.{File, FileOutputStream, InputStream, OutputStream}
import java.util.zip.{ZipEntry, ZipFile}

import scala.collection.JavaConverters._


object ZipUtil {
  val BUFSIZE = 4096
  val buffer = new Array[Byte](BUFSIZE)

  def unZip(source: String, targetFolder: String): Boolean = {
    val zipFile = new ZipFile(source)

    unzipAllFile(zipFile.entries.asScala.toList, getZipEntryInputStream(zipFile), new File(targetFolder))
  }

  def getZipEntryInputStream(zipFile: ZipFile)(entry: ZipEntry): InputStream = zipFile.getInputStream(entry)

  def unzipAllFile(entryList: List[ZipEntry], inputGetter: ZipEntry => InputStream, targetFolder: File): Boolean = {

    entryList match {
      case entry :: entries =>

        if (entry.isDirectory)
          new File(targetFolder, entry.getName).mkdirs
        else
          saveFile(inputGetter(entry), new FileOutputStream(new File(targetFolder, entry.getName)))

        unzipAllFile(entries, inputGetter, targetFolder)
      case _ =>
        true
    }

  }

  def saveFile(fis: InputStream, fos: OutputStream): Unit = {
    writeToFile(bufferReader(fis), fos)
    fis.close()
    fos.close()
  }

  def bufferReader(fis: InputStream)(buffer: Array[Byte]): (Int, Array[Byte]) = (fis.read(buffer), buffer)

  def writeToFile(reader: Array[Byte] => (Int, Array[Byte]), fos: OutputStream): Boolean = {
    val (length, data) = reader(buffer)
    if (length >= 0) {
      fos.write(data, 0, length)
      writeToFile(reader, fos)
    } else
      true
  }
} 

package stagedparsec

import lms._
import lms.testutil.FileDiffSpec

import scala.lms.common._
import scala.lms.internal.Effects

import java.io.PrintWriter
import java.io.StringWriter
import java.io.FileOutputStream

trait RecParsersProg
    extends CharParsers
    with Functions {

  import Parser._

   with SeqOpsExp
          with MyScalaCompile { self =>

        val codegen = new ScalaGenCharParsers
            with ScalaGenStruct
            with ScalaGenIfThenElse {
          val IR: self.type = self
        }

        codegen.emitSource(recNumber _, "recNumber", new java.io.PrintWriter(System.out))
        codegen.reset

        val testcRecNumber = compile(recNumber)
        scala.Console.println(testcRecNumber("12345".toArray))
        codegen.reset
      }
      assertFileEqualsCheck(prefix + "rec-parser")
    }
  }
} 

package stagedparsec

import lms._
import lms.testutil.FileDiffSpec

import scala.lms.common._
import scala.lms.internal.Effects

import java.io.PrintWriter
import java.io.StringWriter
import java.io.FileOutputStream

trait StringReaderProg extends StringReaderOps with MiscOps {

  //print reader.first
  def testFirst(in: Rep[Array[Char]]): Rep[Char] = {
    val rdr: Rep[StringReader] = StringReader(in)
    rdr.first
  }

  def testAtEnd(in: Rep[Array[Char]], offset: Rep[Int]): Rep[Boolean] = {
    val rdr: Rep[StringReader] = StringReader(in, offset)
    rdr.atEnd
  }

  //compute rdr.rest and print first
  def testRest(in: Rep[Array[Char]]): Rep[Char] = {
    val rdr: Rep[StringReader] = StringReader(in)
    val rst = rdr.rest

    rst.first
  }

  def testIteration(in: Rep[Array[Char]]): Rep[Unit] = {
    val rdr = StringReader(in)
    rdr.foreach { c => println(c) }
  }
}

class StringReaderSuite extends FileDiffSpec {

  val prefix = "test-out/"

  def `StringReader generate code with no diff` = {
    withOutFile(prefix + "stringreader") {
      new StringReaderProg
        with StringReaderOpsExpOpt
        with MiscOpsExp
         with SeqOpsExp
        with MyScalaCompile { self =>

        val codegen = new ScalaGenStringReaderOps with ScalaGenMiscOps { val IR: self.type = self }

        codegen.emitSource(testFirst _, "testFirst", new java.io.PrintWriter(System.out))
        codegen.reset

        val testcFirst = compile(testFirst)
        scala.Console.println(testcFirst("hello".toArray))
        codegen.reset

        codegen.emitSource2(testAtEnd _, "testAtEnd", new java.io.PrintWriter(System.out))
        codegen.reset

        val testcAtEnd = compile2(testAtEnd)
        scala.Console.println(testcAtEnd("hello".toArray, 0))
        scala.Console.println(testcAtEnd("hello".toArray, 6))
        codegen.reset

        codegen.emitSource(testRest _, "testRest", new java.io.PrintWriter(System.out))
        codegen.reset

        val testcRest = compile(testRest)
        scala.Console.println(testcRest("hello".toArray))
        codegen.reset

        codegen.emitSource(testIteration _, "testIteration", new java.io.PrintWriter(System.out))
        codegen.reset

        val testcIteration = compile(testIteration)
        testcIteration("hello".toArray)
        codegen.reset

      }
    }

    assertFileEqualsCheck(prefix + "stringreader")
  }
} 

package jgo.tools.compiler

import lexer._
import parser._
import parser.combinatorExten._
import transl._
import interm.symbol.PackageSymbol

import java.io.{File, FileOutputStream}

object Main extends App {
  val fileName = args(0)
  
  //if fileName == "abc.go", "./abc.go" in current dir.
  //if "/abc.go", in root dir.
  //if "~/abc.go", in home dir.
  val file = new File(fileName)
  
  val scanner = Scanner(file)
  val pkg = PackageSymbol("package") //add processing of pkg name later
  val comp = new CompilationUnitCompiler(pkg, scanner) with ExceptionTracing
  val intermErr = comp.compile
  
  intermErr match {
    case Result(interm) =>
      val outputBytes = new PkgTranslator(interm).outputBytes
      //he should be closed properly, but we'll just hope the JVM takes care of it
      new FileOutputStream(args(1)).write(outputBytes)
      
      //Let the record show that this is the single piece of code my friend Aaron has
      //committed to the project, and not only does Aaron fail to close his file;
      //he also anthropomorphizes it! I'd fix this now, but it's not really worth
      //it given that I'm going to have to replace this whole thing anyway with
      //something that does multiple files. In the meantime, may the gods of IO,
      //low latency be upon them, have mercy on my neural configuration.
    
    case Problems(errs) =>
      errs foreach { err => println(err.longString); println() }
  }
} 

package org.apache.spark.graphx

import java.io.File
import java.io.FileOutputStream
import java.io.OutputStreamWriter
import java.nio.charset.StandardCharsets

import org.apache.spark.SparkFunSuite
import org.apache.spark.util.Utils

class GraphLoaderSuite extends SparkFunSuite with LocalSparkContext {

  test("GraphLoader.edgeListFile") {
    withSpark { sc =>
      val tmpDir = Utils.createTempDir()
      val graphFile = new File(tmpDir.getAbsolutePath, "graph.txt")
      val writer = new OutputStreamWriter(new FileOutputStream(graphFile), StandardCharsets.UTF_8)
      for (i <- (1 until 101)) writer.write(s"$i 0\n")
      writer.close()
      try {
        val graph = GraphLoader.edgeListFile(sc, tmpDir.getAbsolutePath)
        val neighborAttrSums = graph.aggregateMessages[Int](
          ctx => ctx.sendToDst(ctx.srcAttr),
          _ + _)
        assert(neighborAttrSums.collect.toSet === Set((0: VertexId, 100)))
      } finally {
        Utils.deleteRecursively(tmpDir)
      }
    }
  }
} 

package org.apache.spark.deploy.worker

import java.io.{File, FileOutputStream, InputStream, IOException}

import scala.collection.JavaConverters._
import scala.collection.Map

import org.apache.spark.SecurityManager
import org.apache.spark.deploy.Command
import org.apache.spark.internal.Logging
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils


  def redirectStream(in: InputStream, file: File) {
    val out = new FileOutputStream(file, true)
    // TODO: It would be nice to add a shutdown hook here that explains why the output is
    //       terminating. Otherwise if the worker dies the executor logs will silently stop.
    new Thread("redirect output to " + file) {
      override def run() {
        try {
          Utils.copyStream(in, out, true)
        } catch {
          case e: IOException =>
            logInfo("Redirection to " + file + " closed: " + e.getMessage)
        }
      }
    }.start()
  }
} 

package org.apache.spark.storage

import java.io.{FileOutputStream, IOException, RandomAccessFile}
import java.nio.ByteBuffer
import java.nio.channels.FileChannel.MapMode

import com.google.common.io.Closeables

import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
import org.apache.spark.util.io.ChunkedByteBuffer


  def put(blockId: BlockId)(writeFunc: FileOutputStream => Unit): Unit = {
    if (contains(blockId)) {
      throw new IllegalStateException(s"Block $blockId is already present in the disk store")
    }
    logDebug(s"Attempting to put block $blockId")
    val startTime = System.currentTimeMillis
    val file = diskManager.getFile(blockId)
    val fileOutputStream = new FileOutputStream(file)
    var threwException: Boolean = true
    try {
      writeFunc(fileOutputStream)
      threwException = false
    } finally {
      try {
        Closeables.close(fileOutputStream, threwException)
      } finally {
         if (threwException) {
          remove(blockId)
        }
      }
    }
    val finishTime = System.currentTimeMillis
    logDebug("Block %s stored as %s file on disk in %d ms".format(
      file.getName,
      Utils.bytesToString(file.length()),
      finishTime - startTime))
  }

  def putBytes(blockId: BlockId, bytes: ChunkedByteBuffer): Unit = {
    put(blockId) { fileOutputStream =>
      val channel = fileOutputStream.getChannel
      Utils.tryWithSafeFinally {
        bytes.writeFully(channel)
      } {
        channel.close()
      }
    }
  }

  def getBytes(blockId: BlockId): ChunkedByteBuffer = {
    val file = diskManager.getFile(blockId.name)
    val channel = new RandomAccessFile(file, "r").getChannel
    Utils.tryWithSafeFinally {
      // For small files, directly read rather than memory map
      if (file.length < minMemoryMapBytes) {
        val buf = ByteBuffer.allocate(file.length.toInt)
        channel.position(0)
        while (buf.remaining() != 0) {
          if (channel.read(buf) == -1) {
            throw new IOException("Reached EOF before filling buffer\n" +
              s"offset=0\nfile=${file.getAbsolutePath}\nbuf.remaining=${buf.remaining}")
          }
        }
        buf.flip()
        new ChunkedByteBuffer(buf)
      } else {
        new ChunkedByteBuffer(channel.map(MapMode.READ_ONLY, 0, file.length))
      }
    } {
      channel.close()
    }
  }

  def remove(blockId: BlockId): Boolean = {
    val file = diskManager.getFile(blockId.name)
    if (file.exists()) {
      val ret = file.delete()
      if (!ret) {
        logWarning(s"Error deleting ${file.getPath()}")
      }
      ret
    } else {
      false
    }
  }

  def contains(blockId: BlockId): Boolean = {
    val file = diskManager.getFile(blockId.name)
    file.exists()
  }
} 

package org.apache.spark.storage

import java.io.{File, FileOutputStream}

import org.scalatest.{BeforeAndAfter, Matchers}

import org.apache.spark._
import org.apache.spark.util.Utils

class TopologyMapperSuite  extends SparkFunSuite
  with Matchers
  with BeforeAndAfter
  with LocalSparkContext {

  test("File based Topology Mapper") {
    val numHosts = 100
    val numRacks = 4
    val props = (1 to numHosts).map{i => s"host-$i" -> s"rack-${i % numRacks}"}.toMap
    val propsFile = createPropertiesFile(props)

    val sparkConf = (new SparkConf(false))
    sparkConf.set("spark.storage.replication.topologyFile", propsFile.getAbsolutePath)
    val topologyMapper = new FileBasedTopologyMapper(sparkConf)

    props.foreach {case (host, topology) =>
      val obtainedTopology = topologyMapper.getTopologyForHost(host)
      assert(obtainedTopology.isDefined)
      assert(obtainedTopology.get === topology)
    }

    // we get None for hosts not in the file
    assert(topologyMapper.getTopologyForHost("host").isEmpty)

    cleanup(propsFile)
  }

  def createPropertiesFile(props: Map[String, String]): File = {
    val testFile = new File(Utils.createTempDir(), "TopologyMapperSuite-test").getAbsoluteFile
    val fileOS = new FileOutputStream(testFile)
    props.foreach{case (k, v) => fileOS.write(s"$k=$v\n".getBytes)}
    fileOS.close
    testFile
  }

  def cleanup(testFile: File): Unit = {
    testFile.getParentFile.listFiles.filter { file =>
      file.getName.startsWith(testFile.getName)
    }.foreach { _.delete() }
  }

} 

package org.apache.spark.deploy.worker

import java.io.{File, FileOutputStream, InputStream, IOException}
import java.lang.System._

import scala.collection.JavaConversions._
import scala.collection.Map

import org.apache.spark.Logging
import org.apache.spark.deploy.Command
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils


  def redirectStream(in: InputStream, file: File) {
    val out = new FileOutputStream(file, true)
    // TODO: It would be nice to add a shutdown hook here that explains why the output is
    //       terminating. Otherwise if the worker dies the executor logs will silently stop.
    new Thread("redirect output to " + file) {
      override def run() {
        try {
          Utils.copyStream(in, out, true)
        } catch {
          case e: IOException =>
            logInfo("Redirection to " + file + " closed: " + e.getMessage)
        }
      }
    }.start()
  }
} 

package com.karasiq.highcharts.generator

import java.io.{BufferedWriter, FileOutputStream, OutputStreamWriter, PrintWriter}
import java.nio.file._
import java.nio.file.attribute.BasicFileAttributes

import scala.util.control.Exception
import scalaj.http.{Http, HttpOptions}

import com.karasiq.highcharts.generator.writers.{ScalaClassWriter, ScalaJsClassBuilder}

case class HighchartsApiDoc(library: String) {
  private val defaultPackage = System.getProperty(s"highcharts-generator.$library.package", s"com.$library")

  private def httpGet(url: String): List[ConfigurationObject] = {
    val page = Http.get(url)
      .header("User-Agent", "Mozilla/5.0 (X11; OpenBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36")
      .header("Accept", "application/json")
      .options(HttpOptions.connTimeout(10000), HttpOptions.readTimeout(10000))

    val json = page.asString
    ConfigurationObject.fromJson(json)
  }

  private def writeFiles(pkg: String, configs: List[ConfigurationObject], rootObject: Option[String] = None): Unit = {
    val header =
      s"""
          |package $pkg
          |
          |import scalajs.js, js.`|`
          |import com.highcharts.CleanJsObject
          |import com.highcharts.HighchartsUtils._
          |
          |""".stripMargin

    val outputDir = Paths.get(System.getProperty("highcharts-generator.output", "src/main/scala"), pkg.split("\\."):_*)
    Files.createDirectories(outputDir)

    // Remove all files
    Files.walkFileTree(outputDir, new SimpleFileVisitor[Path] {
      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
        Files.delete(file)
        FileVisitResult.CONTINUE
      }
    })

    val classes = new ScalaJsClassBuilder().parse(configs, rootObject)
    val classWriter = new ScalaClassWriter
    classes.foreach { scalaJsClass ⇒
      val file = outputDir.resolve(scalaJsClass.scalaName + ".scala")
      println(s"Writing $file...")
      val writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file.toFile, true), "UTF-8")))
      Exception.allCatch.andFinally(writer.close()) {
        if (Files.size(file) == 0) {
          writer.print(header)
        }
        classWriter.writeClass(scalaJsClass) { line ⇒
          writer.println(line)
        }
        writer.flush()
      }
    }
  }

  def writeConfigs(): Unit = {
    val configs = httpGet(s"https://api.highcharts.com/$library/dump.json")
    writeFiles(s"$defaultPackage.config", configs, Some(s"${library.capitalize}Config"))
  }

  def writeApis(): Unit = {
    val configs = httpGet(s"https://api.highcharts.com/$library/object/dump.json")
    writeFiles(s"$defaultPackage.api", configs)
  }

  def writeAll(): Unit = {
    // TODO: https://github.com/highcharts/highcharts/issues/7227
    writeConfigs()
    // writeApis() // TODO: 404
  }
}

object Main extends App {
  HighchartsApiDoc("highcharts").writeAll()
  HighchartsApiDoc("highstock").writeAll()
  HighchartsApiDoc("highmaps").writeAll()
} 

package ai.deepsense.deeplang.doperations.readwritedataframe.filestorage

import java.io.{BufferedWriter, FileOutputStream, IOException, OutputStreamWriter}
import java.nio.file.{Files, Paths}
import java.util.UUID

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}

import ai.deepsense.deeplang.ExecutionContext
import ai.deepsense.deeplang.doperations.exceptions.DeepSenseIOException
import ai.deepsense.deeplang.doperations.readwritedataframe.FilePath

private[filestorage] object FileDownloader {

  def downloadFile(url: String)(implicit context: ExecutionContext): FilePath = {
    if (context.tempPath.startsWith("hdfs://")) {
      downloadFileToHdfs(url)
    } else {
      downloadFileToDriver(url)
    }
  }

  private def downloadFileToHdfs(url: String)(implicit context: ExecutionContext) = {
    val content = scala.io.Source.fromURL(url).getLines()
    val hdfsPath = s"${context.tempPath}/${UUID.randomUUID()}"

    val configuration = new Configuration()
    val hdfs = FileSystem.get(configuration)
    val file = new Path(hdfsPath)
    val hdfsStream = hdfs.create(file)
    val writer = new BufferedWriter(new OutputStreamWriter(hdfsStream))
    try {
      content.foreach {s =>
        writer.write(s)
        writer.newLine()
      }
    } finally {
      safeClose(writer)
      hdfs.close()
    }

    FilePath(hdfsPath)
  }

  private def downloadFileToDriver(url: String)
                                  (implicit context: ExecutionContext) = {
    val outputDirPath = Paths.get(context.tempPath)
    // We're checking if the output is a directory following symlinks.
    // The default behaviour of createDirectories is NOT to follow symlinks
    if (!Files.isDirectory(outputDirPath)) {
      Files.createDirectories(outputDirPath)
    }

    val outFilePath = Files.createTempFile(outputDirPath, "download", ".csv")
    // content is a stream. Do not invoke stuff like .toList() on it.
    val content = scala.io.Source.fromURL(url).getLines()
    val writer: BufferedWriter =
      new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFilePath.toFile)))
    try {
      content.foreach {s =>
        writer.write(s)
        writer.newLine()
      }
    } finally {
      safeClose(writer)
    }
    FilePath(s"file:///$outFilePath")
  }

  private def safeClose(bufferedWriter: BufferedWriter): Unit = {
    try {
      bufferedWriter.flush()
      bufferedWriter.close()
    } catch {
      case e: IOException => throw new DeepSenseIOException(e)
    }
  }

} 

package ai.deepsense.deeplang.doperations.readwritedataframe.googlestorage

import java.io.{ByteArrayInputStream, FileOutputStream}
import java.util

import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
import com.google.api.client.http.FileContent
import com.google.api.client.json.gson.GsonFactory
import com.google.api.services.drive.model.File
import com.google.api.services.drive.{Drive, DriveScopes}

import ai.deepsense.commons.resources.ManagedResource
import ai.deepsense.commons.utils.LoggerForCallerClass
import ai.deepsense.deeplang.doperations.inout.CsvParameters.ColumnSeparatorChoice

private[googlestorage] object GoogleDriveClient {

  val logger = LoggerForCallerClass()

  val googleSheetCsvSeparator = ColumnSeparatorChoice.Comma()

  private val ApplicationName = "Seahorse"

  private val Scopes = util.Arrays.asList(DriveScopes.DRIVE)

  def uploadCsvFileAsGoogleSheet(
      credentials: GoogleCretendialsJson,
      sheetId: GoogleSheetId,
      filePath: String
    ): Unit = {
    val fileMetadata = new File().setMimeType("application/vnd.google-apps.spreadsheet")
    val mediaContent = new FileContent("text/csv", new java.io.File(filePath))

    driveService(credentials).files.update(sheetId, fileMetadata, mediaContent).execute
  }

  def downloadGoogleSheetAsCsvFile(
      credentials: GoogleCretendialsJson,
      sheetId: GoogleSheetId,
      filePath: String
    ): Unit = {
    val file = new java.io.File(filePath)
    file.getParentFile.mkdirs()

    ManagedResource(new FileOutputStream(file)) { fos =>
      driveService(credentials).files().export(sheetId, "text/csv").executeMediaAndDownloadTo(fos)
      logger.info(s"Downloaded google sheet id=$sheetId to the file $filePath")
    }
  }

  private def driveService(serviceAccountCredentials: String): Drive = {
    val credential = {
      val in = new ByteArrayInputStream(serviceAccountCredentials.getBytes)
      GoogleCredential.fromStream(in).createScoped(Scopes)
    }
    new Drive.Builder(
      GoogleNetHttpTransport.newTrustedTransport(),
      jsonFactory,
      credential
    ).setApplicationName(ApplicationName).build
  }

  // Default choice is JacksonFactory. However spark depends on Jackson as well
  // and google/spark jackson versions are binary incompatible with each other.
  private val jsonFactory = GsonFactory.getDefaultInstance

} 

package org.platanios.tensorflow.data.utilities

import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.utils.IOUtils

import java.io.{File, FileOutputStream, InputStream}
import java.nio.file.{Files, Path}
import java.util.zip.GZIPInputStream


object CompressedFiles {
  def decompressTGZ(tgzFilePath: Path, destinationPath: Path, bufferSize: Int = 8192): Unit = {
    decompressTGZStream(Files.newInputStream(tgzFilePath), destinationPath, bufferSize)
  }

  def decompressTar(tarFilePath: Path, destinationPath: Path, bufferSize: Int = 8192): Unit = {
    decompressTarStream(Files.newInputStream(tarFilePath), destinationPath, bufferSize)
  }

  def decompressTGZStream(tgzStream: InputStream, destinationPath: Path, bufferSize: Int = 8192): Unit = {
    decompressTarStream(new GZIPInputStream(tgzStream), destinationPath, bufferSize)
  }

  def decompressTarStream(tarStream: InputStream, destinationPath: Path, bufferSize: Int = 8192): Unit = {
    val inputStream = new TarArchiveInputStream(tarStream)
    var entry = inputStream.getNextTarEntry
    while (entry != null) {
      if (!entry.isDirectory) {
        val currentFile = new File(destinationPath.toAbsolutePath.toString, entry.getName)
        val parentFile = currentFile.getParentFile
        if (!parentFile.exists)
          parentFile.mkdirs()
        IOUtils.copy(inputStream, new FileOutputStream(currentFile))
      }
      entry = inputStream.getNextTarEntry
    }
    inputStream.close()
  }
} 

package com.ebay.rtran.xml.util

import java.io.{File, FileOutputStream}
import javax.xml.stream.XMLOutputFactory

import com.sun.xml.internal.txw2.output.IndentingXMLStreamWriter
import org.apache.axiom.om.OMElement


object XmlUtil {

  def writeOMElement2File(file: File, root: OMElement, indenting: Boolean = false): Unit = {
    val writer = XMLOutputFactory.newInstance.createXMLStreamWriter(new FileOutputStream(file))
    val indentedWriter = if (indenting) new IndentingXMLStreamWriter(writer) else writer
    indentedWriter.writeStartDocument()
    indentedWriter.writeCharacters("\n")
    root.serialize(indentedWriter)
    indentedWriter.writeCharacters("\n")
    indentedWriter.flush()
    indentedWriter.close()
  }
} 

package com.ebay.rtran.report

import java.io.{File, FileOutputStream}

import ch.qos.logback.classic
import ch.qos.logback.classic.encoder.PatternLayoutEncoder
import ch.qos.logback.classic.filter.ThresholdFilter
import ch.qos.logback.classic.spi.ILoggingEvent
import ch.qos.logback.core.FileAppender
import org.reflections.Reflections
import com.ebay.rtran.report.api.IReportEventSubscriber
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.JavaConversions._
import scala.language.postfixOps
import scala.util.{Success, Try}


trait ReportAndLogSupport {

  val reportFilePrefix: String
  val warnLogPrefix: String
  val debugLogPrefix: String

  def createReportAndLogs[T](projectRoot: File,
                             taskId: Option[String], packages: String*)(fn: => T): T = {
    val appenders = prepareAppenders(projectRoot, taskId)
    val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME).asInstanceOf[classic.Logger]
    appenders foreach rootLogger.addAppender
    appenders foreach (_.start)
    val reportFile = new File(projectRoot, s"$reportFilePrefix${taskId.map("-" + _) getOrElse ""}.md")
    val result = Report.createReport(
      new FileOutputStream(reportFile),
      subscribers = allSubscribers(projectRoot, packages: _*)
    )(fn)
    appenders foreach (_.stop)
    appenders foreach rootLogger.detachAppender
    result
  }

  def allSubscribers(projectRoot: File, packages: String*) = {
    val subscribers = packages flatMap {prefix =>
      new Reflections(prefix).getSubTypesOf(classOf[IReportEventSubscriber[_]])
    } map {clazz =>
      Try(clazz.getDeclaredConstructor(classOf[File]).newInstance(projectRoot)) orElse Try(clazz.newInstance)
    } collect {
      case Success(subscriber) => subscriber
    } toList

    subscribers.sortBy(_.sequence)
  }

  private def prepareAppenders(projectRoot: File, taskId: Option[String]) = {
    val lc = LoggerFactory.getILoggerFactory.asInstanceOf[classic.LoggerContext]
    val encoders = Array(new PatternLayoutEncoder, new PatternLayoutEncoder)
    encoders foreach (_ setContext lc)
    encoders foreach (_ setPattern "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}.%M:%L - %m%n")
    encoders foreach (_.start)

    val warnFileAppender = new FileAppender[ILoggingEvent]
    warnFileAppender.setName("warnFileAppender")
    warnFileAppender.setFile(s"${projectRoot.getAbsolutePath}/$warnLogPrefix${taskId.map("-" + _) getOrElse ""}.log")
    warnFileAppender.addFilter(new SameThreadFilter)
    val warnFilter = new ThresholdFilter
    warnFilter.setLevel("WARN")
    warnFilter.start()
    warnFileAppender.addFilter(warnFilter)

    val debugFileAppender = new FileAppender[ILoggingEvent]
    debugFileAppender.setName("debugFileAppender")
    debugFileAppender.setFile(s"${projectRoot.getAbsolutePath}/$debugLogPrefix${taskId.map("-" + _) getOrElse ""}.log")
    debugFileAppender.addFilter(new SameThreadFilter)
    val debugFilter = new ThresholdFilter
    debugFilter.setLevel("DEBUG")
    debugFilter.start()
    debugFileAppender.addFilter(debugFilter)

    val result = List(warnFileAppender, debugFileAppender)
    result.foreach(_ setContext lc)
    result zip encoders foreach (entry => entry._1 setEncoder entry._2)
    result
  }
} 

package com.ebay.rtran.report

import java.io.{File, FileOutputStream, OutputStream}
import java.util.Optional
import com.typesafe.scalalogging.LazyLogging
import com.ebay.rtran.report.api.IReportEventSubscriber
import org.scalatest.{FlatSpecLike, Matchers}


class ReportTest extends FlatSpecLike with Matchers with LazyLogging {

  "Report" should "accept log event correctly" in {
    val subscriber = new TestSubscriber
    val outputStream = new FileOutputStream("test")
    Report.createReport(outputStream, subscribers = List(subscriber))(testFunc())
    subscriber.getCount should be (3)
    new File("test").delete
  }

  "Report" should "work in concurrent mode" in {
    val subscribers = (1 to 10) map (_ => new TestSubscriber)

    subscribers foreach {sub =>
      new Thread(new TestThread(sub)).start()
    }

    val waitPeriod = 1000

    Thread.sleep(waitPeriod)

    subscribers foreach (_.getCount should be (3))
  }

  class TestThread(subscriber: IReportEventSubscriber[_]) extends Runnable {
    override def run(): Unit = {
      val file = new File(s"test-${System.nanoTime}")
      val outputStream = new FileOutputStream(file)
      Report.createReport(outputStream, subscribers = List(subscriber))(testFunc())
      file.delete
    }
  }

  def testFunc(): Unit = {
    val str = "hello"
    val number = 2000
    logger.info("String {}", str)
    logger.info(s"number ${number + 2}")
    logger.info("String {} number {}", str, number.toString)
  }

  class TestSubscriber extends IReportEventSubscriber[Int] {
    import scala.compat.java8.OptionConverters._

    private var count = 0

    def getCount = count

    override def filter(event: scala.Any): Optional[Int] = Some(1).asJava

    override def dumpTo(outputStream: OutputStream): Unit = {}

    override def doAccept(event: Int): Unit = count += event
  }

} 

package yaooqinn.kyuubi.operation

import java.io.{BufferedReader, File, FileInputStream, FileNotFoundException, FileOutputStream, InputStreamReader, IOException, PrintStream}
import java.util.ArrayList

import scala.collection.JavaConverters._

import org.apache.commons.io.FileUtils
import org.apache.hadoop.io.IOUtils
import org.apache.kyuubi.Logging
import org.apache.spark.sql.Row

import yaooqinn.kyuubi.KyuubiSQLException

class LogFile private (
    file: File,
    private var reader: Option[BufferedReader],
    writer: PrintStream,
    @volatile private var isRemoved: Boolean = false) extends Logging {

  def this(file: File) = {
    this(file,
      LogFile.createReader(file, isRemoved = false),
      new PrintStream(new FileOutputStream(file)))
  }

  private def resetReader(): Unit = {
    reader.foreach(IOUtils.closeStream)
    reader = None
  }

  private def readResults(nLines: Long): Seq[Row] = {
    reader = reader.orElse(LogFile.createReader(file, isRemoved))

    val logs = new ArrayList[Row]()
    reader.foreach { r =>
      var i = 1
      try {
        var line: String = r.readLine()
        while ((i < nLines || nLines <= 0) && line != null) {
          logs.add(Row(line))
          line = r.readLine()
          i += 1
        }
      } catch {
        case e: FileNotFoundException =>
          val operationHandle = file.getName
          val path = file.getAbsolutePath
          val msg = if (isRemoved) {
            s"Operation[$operationHandle] has been closed and the log file $path has been removed"
          } else {
            s"Operation[$operationHandle] Log file $path is not found"
          }
          throw new KyuubiSQLException(msg, e)
      }
    }
    logs.asScala
  }

  
  def write(msg: String): Unit = {
    writer.print(msg)
  }


  def close(): Unit = synchronized {
    try {
      reader.foreach(_.close())
      writer.close()
      if (!isRemoved) {
        FileUtils.forceDelete(file)
        isRemoved = true
      }
    } catch {
      case e: IOException =>
        error(s"Failed to remove corresponding log file of operation: ${file.getName}", e)
    }
  }
}

object LogFile {

  def createReader(file: File, isRemoved: Boolean): Option[BufferedReader] = try {
    Option(new BufferedReader(new InputStreamReader(new FileInputStream(file))))
  } catch {
    case e: FileNotFoundException =>
      val operationHandle = file.getName
      val path = file.getAbsolutePath
      val msg = if (isRemoved) {
        s"Operation[$operationHandle] has been closed and the log file $path has been removed"
      } else {
        s"Operation[$operationHandle] Log file $path is not found"
      }
      throw new KyuubiSQLException(msg, e)
  }
} 

package co.ledger.wallet.core.utils.logs

import java.io.{BufferedInputStream, FileOutputStream, File, InputStream}
import java.util.zip.GZIPOutputStream

import android.content.{Intent, Context}
import android.os.Build
import android.support.v4.content.FileProvider
import co.ledger.wallet.app.Config
import co.ledger.wallet.core.content.FileContentProvider
import co.ledger.wallet.core.utils.io.IOUtils

import scala.concurrent.{Promise, Future}
import scala.util.Try
import scala.concurrent.ExecutionContext.Implicits.global

object LogCatReader {

  def getInputStream: InputStream = Runtime.getRuntime.exec(Array("logcat", "-d")).getInputStream

  def createZipFile(file: File): Future[File] = Future {
    val input = new BufferedInputStream(getInputStream)
    val output = new GZIPOutputStream(new FileOutputStream(file))
    Logger.i(s"Create zip file ${file.getName}")
    IOUtils.copy(input, output)
    Logger.i(s"End zip file ${file.getName} creation")
    output.flush()
    input.close()
    output.close()
    file
  }

  def exportLogsToDefaultTempLogFile()(implicit context: Context): Future[File] = {
    val file = getDefaultTempLogFile
    file.map(createZipFile)
    .getOrElse {
      Promise[File]().failure(file.failed.get).future
    }
  }

  def getDefaultTempLogFile(implicit context: Context) = Try {
    val sharedDirectory = new File(context.getCacheDir, "shared/")
    sharedDirectory.mkdirs()
    File.createTempFile("logs_", ".gzip", sharedDirectory)
  }

  def createEmailIntent(context: Context): Future[Intent] = {
    exportLogsToDefaultTempLogFile()(context).map((f) => {
      val intent = new Intent(Intent.ACTION_SEND)
      intent.setType("application/x-gzip")
      f.setReadable(true, false)
      intent.putExtra(Intent.EXTRA_EMAIL, Array[String](Config.SupportEmailAddress))
      val pkg = context.getPackageManager.getPackageInfo(context.getPackageName, 0)
      val emailBody =
        s"""
          |--- Device informations ---
          | Model: ${Build.MANUFACTURER} ${Build.MODEL} - ${Build.DEVICE}
          | OS: ${Build.DISPLAY}
          | Android version: ${Build.VERSION.RELEASE}
          | Application: ${pkg.packageName}
          | Application version: ${pkg.versionName} - ${pkg.versionCode}
        """.stripMargin
      val uri =  FileProvider.getUriForFile(context, FileContentProvider.Authority, f)
      intent.setFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)
      intent.putExtra(Intent.EXTRA_TEXT, emailBody)
      intent.putExtra(Intent.EXTRA_STREAM, uri)
      intent
    })
  }

} 

package sbtassembly

import java.io.{FileOutputStream, File}

import scala.collection.JavaConverters.asJavaEnumerationConverter

import org.apache.logging.log4j.core.config.plugins.processor.PluginCache

object Log4j2MergeStrategy {
  val plugincache: MergeStrategy = new MergeStrategy {
    val name = "log4j2::plugincache"
    def apply(tempDir: File, path: String, files: Seq[File]): Either[String, Seq[(File, String)]] = {
      val file = MergeStrategy.createMergeTarget(tempDir, path)
      val out = new FileOutputStream(file)

      val aggregator = new PluginCache()
      val filesEnum = files.toIterator.map(_.toURI.toURL).asJavaEnumeration

      try {
          aggregator.loadCacheFiles(filesEnum)
          aggregator.writeCache(out)
          Right(Seq(file -> path))
      }
      finally {
          out.close()
      }
    }
  }
} 

package com.sksamuel.avro4s.github

import java.io.{FileOutputStream, ObjectOutputStream}

import com.sksamuel.avro4s.Encoder
import com.sksamuel.avro4s.github.Github415.PlaybackSession
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.must.Matchers

class Github415 extends AnyFunSuite with Matchers {

  test("github 415") {
    val fileOut = new FileOutputStream("remove_me")
    val out = new ObjectOutputStream(fileOut)
    out.writeObject(Encoder[PlaybackSession])
  }
}

object Github415 {
  object Rebuffers {
    case class Metrics(count: Int)
    case class EarlyLate(early: Metrics)
    case class Stats(session: Option[EarlyLate])
  }

  case class Rebuffers(network: Option[Rebuffers.Stats])

  case class PlaybackSession(rebuffers: Option[Rebuffers])
} 

import java.io.{ File, FileInputStream, FileOutputStream }
import java.nio.file.Files

object Assemblies {

  
  def mkBatAssembly(assembly: File): File = {
    val file = Files.createTempFile("akka-grpc-", ".tmp").toFile

    file.deleteOnExit()
    copySkippingUntil('@'.toByte, assembly, file)
    file
  }

  private def copySkippingUntil(b: Byte, src: File, dst: File): Unit = {
    val in = new FileInputStream(src)
    try {
      val out = new FileOutputStream(dst, false)
      val foundSkipByte = Iterator.continually(in.read()).takeWhile(_ >= 0).dropWhile(_ != b.toInt).nonEmpty

      try {
        if (foundSkipByte)
          out.write(b.toInt)

        val buffer = new Array[Byte](1024)
        var continue = true && foundSkipByte
        while (continue) {
          val r = in.read(buffer)
          if (r < 0) continue = false
          else out.write(buffer, 0, r)
        }
      } finally {
        out.close()
      }
    } finally {
      in.close()
    }
  }
} 

package org.apache.spark.graphx

import java.io.File
import java.io.FileOutputStream
import java.io.OutputStreamWriter
import java.nio.charset.StandardCharsets

import org.apache.spark.SparkFunSuite
import org.apache.spark.util.Utils

class GraphLoaderSuite extends SparkFunSuite with LocalSparkContext {

  test("GraphLoader.edgeListFile") {
    withSpark { sc =>
      val tmpDir = Utils.createTempDir()
      val graphFile = new File(tmpDir.getAbsolutePath, "graph.txt")
      val writer = new OutputStreamWriter(new FileOutputStream(graphFile), StandardCharsets.UTF_8)
      for (i <- (1 until 101)) writer.write(s"$i 0\n")
      writer.close()
      try {
        val graph = GraphLoader.edgeListFile(sc, tmpDir.getAbsolutePath)
        val neighborAttrSums = graph.aggregateMessages[Int](
          ctx => ctx.sendToDst(ctx.srcAttr),
          _ + _)
        assert(neighborAttrSums.collect.toSet === Set((0: VertexId, 100)))
      } finally {
        Utils.deleteRecursively(tmpDir)
      }
    }
  }
} 

package org.apache.spark.api.python

import java.io.{DataOutputStream, File, FileOutputStream}
import java.net.InetAddress
import java.nio.charset.StandardCharsets.UTF_8
import java.nio.file.Files

import py4j.GatewayServer

import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils


private[spark] object PythonGatewayServer extends Logging {
  initializeLogIfNecessary(true)

  def main(args: Array[String]): Unit = {
    val secret = Utils.createSecret(new SparkConf())

    // Start a GatewayServer on an ephemeral port. Make sure the callback client is configured
    // with the same secret, in case the app needs callbacks from the JVM to the underlying
    // python processes.
    val localhost = InetAddress.getLoopbackAddress()
    val gatewayServer: GatewayServer = new GatewayServer.GatewayServerBuilder()
      .authToken(secret)
      .javaPort(0)
      .javaAddress(localhost)
      .callbackClient(GatewayServer.DEFAULT_PYTHON_PORT, localhost, secret)
      .build()

    gatewayServer.start()
    val boundPort: Int = gatewayServer.getListeningPort
    if (boundPort == -1) {
      logError("GatewayServer failed to bind; exiting")
      System.exit(1)
    } else {
      logDebug(s"Started PythonGatewayServer on port $boundPort")
    }

    // Communicate the connection information back to the python process by writing the
    // information in the requested file. This needs to match the read side in java_gateway.py.
    val connectionInfoPath = new File(sys.env("_PYSPARK_DRIVER_CONN_INFO_PATH"))
    val tmpPath = Files.createTempFile(connectionInfoPath.getParentFile().toPath(),
      "connection", ".info").toFile()

    val dos = new DataOutputStream(new FileOutputStream(tmpPath))
    dos.writeInt(boundPort)

    val secretBytes = secret.getBytes(UTF_8)
    dos.writeInt(secretBytes.length)
    dos.write(secretBytes, 0, secretBytes.length)
    dos.close()

    if (!tmpPath.renameTo(connectionInfoPath)) {
      logError(s"Unable to write connection information to $connectionInfoPath.")
      System.exit(1)
    }

    // Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
    while (System.in.read() != -1) {
      // Do nothing
    }
    logDebug("Exiting due to broken pipe from Python driver")
    System.exit(0)
  }
} 

package org.apache.spark.deploy.worker

import java.io.{File, FileOutputStream, InputStream, IOException}

import scala.collection.JavaConverters._
import scala.collection.Map

import org.apache.spark.SecurityManager
import org.apache.spark.deploy.Command
import org.apache.spark.internal.Logging
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils


  def redirectStream(in: InputStream, file: File) {
    val out = new FileOutputStream(file, true)
    // TODO: It would be nice to add a shutdown hook here that explains why the output is
    //       terminating. Otherwise if the worker dies the executor logs will silently stop.
    new Thread("redirect output to " + file) {
      override def run() {
        try {
          Utils.copyStream(in, out, true)
        } catch {
          case e: IOException =>
            logInfo("Redirection to " + file + " closed: " + e.getMessage)
        }
      }
    }.start()
  }
} 

package org.apache.spark.storage

import java.io.{File, FileOutputStream}

import org.scalatest.{BeforeAndAfter, Matchers}

import org.apache.spark._
import org.apache.spark.util.Utils

class TopologyMapperSuite  extends SparkFunSuite
  with Matchers
  with BeforeAndAfter
  with LocalSparkContext {

  test("File based Topology Mapper") {
    val numHosts = 100
    val numRacks = 4
    val props = (1 to numHosts).map{i => s"host-$i" -> s"rack-${i % numRacks}"}.toMap
    val propsFile = createPropertiesFile(props)

    val sparkConf = (new SparkConf(false))
    sparkConf.set("spark.storage.replication.topologyFile", propsFile.getAbsolutePath)
    val topologyMapper = new FileBasedTopologyMapper(sparkConf)

    props.foreach {case (host, topology) =>
      val obtainedTopology = topologyMapper.getTopologyForHost(host)
      assert(obtainedTopology.isDefined)
      assert(obtainedTopology.get === topology)
    }

    // we get None for hosts not in the file
    assert(topologyMapper.getTopologyForHost("host").isEmpty)

    cleanup(propsFile)
  }

  def createPropertiesFile(props: Map[String, String]): File = {
    val testFile = new File(Utils.createTempDir(), "TopologyMapperSuite-test").getAbsoluteFile
    val fileOS = new FileOutputStream(testFile)
    props.foreach{case (k, v) => fileOS.write(s"$k=$v\n".getBytes)}
    fileOS.close
    testFile
  }

  def cleanup(testFile: File): Unit = {
    testFile.getParentFile.listFiles.filter { file =>
      file.getName.startsWith(testFile.getName)
    }.foreach { _.delete() }
  }

} 

package io.picnicml.doddlemodel.typeclasses

import java.io.{FileOutputStream, ObjectOutputStream}

// evidence needs to be serializable because it is persisted along with the actual
// estimators within the io.picnicml.doddlemodel.pipeline.Pipeline
trait Estimator[A] extends Serializable {

  def isFitted(model: A): Boolean

  def save(model: A, filePath: String): Unit = {
    val outputStream = new ObjectOutputStream(new FileOutputStream(filePath))
    outputStream.writeObject(model)
    outputStream.close()
  }
} 

package io.picnicml.doddlemodel.data

import java.io.{File, FileOutputStream}

import io.picnicml.doddlemodel.data.CsvLoader.loadCsvDataset

import scala.io.{BufferedSource, Source}

object ResourceDatasetLoaders {

  def loadBostonDataset: DatasetWithIndex = {
    val (data, featureIndex) = loadDatasetFromResources("boston_housing_prices")
    (data(::, 0 to -2), data(::, -1), featureIndex.drop(data.cols - 1))
  }

  def loadBreastCancerDataset: DatasetWithIndex = {
    val (data, featureIndex) = loadDatasetFromResources("breast_cancer")
    (data(::, 0 to -2), data(::, -1), featureIndex.drop(data.cols - 1))
  }

  def loadIrisDataset: DatasetWithIndex = {
    val (data, featureIndex) = loadDatasetFromResources("iris")
    (data(::, 0 to -2), data(::, -1), featureIndex.drop(data.cols - 1))
  }

  
  def loadHighSchoolTestDataset: DatasetWithIndex = {
    val (data, featureIndex) = loadDatasetFromResources("high_school_test")
    (data(::, 0 to -2), data(::, -1), featureIndex.drop(data.cols - 1))
  }

  private[data] def loadDummyCsvReadingDataset: DatasetWithIndex = {
    val (data, featureIndex) = loadDatasetFromResources("dummy_csv_reading")
    (data(::, 0 to -2), data(::, -1), featureIndex.drop(data.cols - 1))
  }

  private def loadDatasetFromResources(datasetName: String): FeaturesWithIndex =
    loadCsvDataset(getBufferedSourceFromResource(s"/datasets/$datasetName.csv"), na = "NA")

  private def getBufferedSourceFromResource(path: String): BufferedSource = {
    val resourceUrl = getClass.getResource(path)
    val file = if (resourceUrl.toString.startsWith("jar:"))
      // reads file from JAR
      readResourceFileWithinJar(path)
    else
      // reads file when using IDE
      new File(resourceUrl.getFile)
    if (file != null && !file.exists)
      throw new RuntimeException(s"Error: File $file not found!")
    Source.fromFile(file)
  }

  private def readResourceFileWithinJar(path: String): File = {
    val inputStream = getClass.getResourceAsStream(path)
    val tempFile = File.createTempFile("tempfile", ".tmp")
    val outputStream = new FileOutputStream(tempFile)

    val buffer = new Array[Byte](130 * 1024)
    Iterator.continually(inputStream.read(buffer)).takeWhile(_ != -1).foreach { bytesRead =>
      outputStream.write(buffer, 0, bytesRead)
      outputStream.flush()
    }

    inputStream.close()
    outputStream.close()

    tempFile.deleteOnExit()
    tempFile
  }
} 

package mimir.data.staging

import java.net.URL
import java.io.{ File, InputStream, OutputStream, FileOutputStream }
import java.sql.SQLException
import scala.util.Random
import com.typesafe.scalalogging.LazyLogging
import org.apache.spark.sql.DataFrame
import mimir.algebra.ID


  private def transferBytes(input: InputStream, output: OutputStream): Unit =
  {
    val buffer = Array.ofDim[Byte](1024*1024) // 1MB buffer
    var bytesRead = input.read(buffer)
    while(bytesRead >= 0) { 
      output.write(buffer, 0, bytesRead)
      bytesRead = input.read(buffer)
    }
  }

  def stage(input: InputStream, fileExtension: String, nameHint: Option[String]): String = 
  {
    val file = makeName(fileExtension, nameHint)
    transferBytes(input, new FileOutputStream(file))
    return file.toString
  }
  def stage(url: URL, nameHint: Option[String]): String =
  {
    val pathComponents = url.getPath.split("/")
    val nameComponents = pathComponents.reverse.head.split(".")
    val extension = 
      if(nameComponents.size > 1) { nameComponents.reverse.head }
      else { "data" } // default to generic 'data' if there's no extension
    stage(url.openStream(), extension, nameHint)
  }
  def stage(input: DataFrame, format: ID, nameHint:Option[String]): String =
  {
    val targetFile = makeName(format.id, nameHint).toString
    input.write
         .format(format.id)
         .save(targetFile)
    return targetFile
  }
  def drop(local: String): Unit = 
  {
    new File(local).delete()
  }
} 

package mimir.data.staging

import java.net.URL
import java.io.{ File, InputStream, OutputStream, FileOutputStream }
import java.sql.SQLException
import scala.util.Random
import com.typesafe.scalalogging.LazyLogging
import org.apache.spark.sql.DataFrame
import mimir.algebra.ID
import mimir.util.HadoopUtils
import mimir.exec.spark.MimirSpark


  private def makeName(extension: String, nameHint: Option[String]): File =
  {
    val rand = new Random().alphanumeric
    // Try 1000 times to create a randomly named file
    for(i <- 0 until 1000){
      val candidate = new File(basePath,
        nameHint match { 
          case Some(hint) => s"${hint.replaceAll("[^a-zA-Z0-9]", "")}-${rand.take(10).mkString}.${extension}"
          case None => s"${rand.take(20).mkString}.${extension}"
        }
      )
      // If the randomly named file doesn't exist, we're done.
      if(!candidate.exists()){ return candidate }
    }
    // Fail after 1000 attempts.
    throw new SQLException(s"Can't allocate name for $nameHint")
  }


  def stage(input: InputStream, fileExtension: String, nameHint: Option[String]): String = 
  {
    val file = makeName(fileExtension, nameHint)
    logger.debug("Stage File To HDFS: " +hdfsHome+File.separator+file.toString)
    //if(!HadoopUtils.fileExistsHDFS(sparkSql.sparkSession.sparkContext, fileName))
    HadoopUtils.writeToHDFS(MimirSpark.get.sparkSession.sparkContext, file.getName, input, true)
    logger.debug("... done\n")
    return s"$hdfsHome/${file.getName}"
  }
  def stage(url: URL, nameHint: Option[String]): String =
  {
    val pathComponents = url.getPath.split("/")
    val nameComponents = pathComponents.reverse.head.split(".")
    val extension = 
      if(nameComponents.size > 1) { nameComponents.reverse.head }
      else { "data" } // default to generic 'data' if there's no extension
    stage(url.openStream(), extension, nameHint)
  }
  def stage(input: DataFrame, format: ID, nameHint:Option[String]): String =
  {
    val targetFile = makeName(format.id, nameHint).toString
    input.write
         .format(format.id)
         .save(targetFile)
    return targetFile
  }
  def drop(local: String): Unit = 
  {
    new File(local).delete()
  }
} 

package mimir.util

import java.lang.reflect.Method
import java.net.URL
import java.io.{File, FileOutputStream, BufferedOutputStream, InputStream}
import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveInputStream}
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
import java.io.BufferedInputStream
import java.nio.file.Files

object FileUtils {
  def getListOfFiles(dir: String):List[File] = {
    val d = new File(dir)
    if (d.exists && d.isDirectory) {
        d.listFiles.filter(_.isFile).toList
    } else {
        List[File]()
    }
  }
  def addJarToClasspath(jar: File): Unit = {
// Get the ClassLoader class
    val cl: ClassLoader = ClassLoader.getSystemClassLoader
    val clazz: Class[_] = cl.getClass
// Get the protected addURL method from the parent URLClassLoader class
    val method: Method =
      clazz.getSuperclass.getDeclaredMethod("addURL", Seq(classOf[URL]):_*)
// Run projected addURL method to add JAR to classpath
    method.setAccessible(true)
    method.invoke(cl, Seq(jar.toURI().toURL()):_*)
  }
  
  def untar(in:InputStream, destinationDir: String): File = {

    val dest = new File(destinationDir)
    dest.mkdir()

    var tarIn: TarArchiveInputStream = null

    try {
      tarIn = new TarArchiveInputStream(
        new GzipCompressorInputStream(
          new BufferedInputStream(in)))
      var tarEntry = tarIn.getNextTarEntry
      while (tarEntry != null) {

        // create a file with the same name as the tarEntry
        val destPath = new File(dest, tarEntry.getName)
        if (tarEntry.isDirectory) {
          destPath.mkdirs()
        } else {
          // Create any necessary parent dirs
          val parent = destPath.getParentFile
          if (!Files.exists(parent.toPath)) {
            parent.mkdirs()
          }

          destPath.createNewFile()

          val btoRead = new Array[Byte](1024)
          var bout: BufferedOutputStream = null

          try {
            bout = new BufferedOutputStream(new FileOutputStream(destPath))

            var len = 0
            while (len != -1) {
              len = tarIn.read(btoRead)
              if (len != -1) {
                bout.write(btoRead, 0, len)
              }
            }
          } finally {
            if (bout != null) {
              bout.close()
            }
          }
        }
        tarEntry = tarIn.getNextTarEntry
      }
    } finally {
      if (tarIn != null) {
        tarIn.close()
      }
    }
    dest
  }
  
} 

package com.sumologic.shellbase.commands

import java.io.{FileOutputStream, PrintStream}

import com.sumologic.shellbase.ShellCommand
import com.sumologic.shellbase.cmdline.RichCommandLine._
import com.sumologic.shellbase.cmdline.{CommandLineArgument, CommandLineFlag, CommandLineOption}
import org.apache.commons.cli.{CommandLine, Options}
import org.apache.commons.io.output.TeeOutputStream

import scala.util.Try

class TeeCommand(runCommand: String => Boolean) extends ShellCommand("tee", "Forks the stdout of a command so it also prints to a file") {

  private val CommandArgument = new CommandLineArgument("command", 0, true)
  private val OutputFileOption = new CommandLineOption("o", "outputFile", false, "Filename of output file (defaults to ~/tee.out)")
  private val AppendFileFlag = new CommandLineFlag("a", "append", "Append the output to the file rather than overwriting it")

  override def maxNumberOfArguments = 1

  override def addOptions(opts: Options) {
    opts += CommandArgument
    opts += OutputFileOption
    opts += AppendFileFlag
  }

  import com.sumologic.shellbase.ShellBase.SubCommandExtractor

  def execute(cmdLine: CommandLine) = {
    val outputFile = cmdLine.get(OutputFileOption).getOrElse(System.getProperty("user.home") + s"/tee.out")
    val appendFile = cmdLine.checkFlag(AppendFileFlag)

    cmdLine.get(CommandArgument) match {
      case Some(SubCommandExtractor(cmd)) =>
        val fileOut = new FileOutputStream(outputFile, appendFile)
        val newOut = new PrintStream(new TeeOutputStream(Console.out, fileOut))
        val status = Console.withOut(newOut) {
          println(s"Running `$cmd` and outputting to '$outputFile' [append=$appendFile].")
          runCommand(cmd)
        }
        Try(fileOut.close())
        status
      case badCmd =>
        println(s"Usage: tee `<command>`, but found $badCmd.")
        false
    }
  }
} 

package com.klibisz.elastiknn.testing

import java.io.FileOutputStream
import java.util.zip.{GZIPInputStream, GZIPOutputStream}

import com.klibisz.elastiknn.api.{Similarity, Vec}
import io.circe._
import com.klibisz.elastiknn.api.ElasticsearchCodec._
import com.klibisz.elastiknn.models.ExactSimilarityFunction
import io.circe.syntax._
import io.circe.generic.semiauto._

import scala.util.{Random, Try}

case class Result(similarity: Similarity, values: Vector[Double])
object Result {
  implicit val codec: Codec[Result] = deriveCodec[Result]
}

case class Query(vector: Vec, results: Seq[Result])
object Query {
  implicit val codec: Codec[Query] = deriveCodec[Query]
}

case class TestData(corpus: Vector[Vec], queries: Vector[Query])
object TestData {

  implicit val codec: Codec[TestData] = deriveCodec[TestData]

  def read(fname: String): TestData = {
    val resource = getClass.getResource(fname)
    val gin = new GZIPInputStream(resource.openStream())
    val contents = new String(gin.readAllBytes())
    gin.close()
    io.circe.parser.decode[TestData](contents).toTry.get
  }

  def write(testData: TestData, fname: String): Unit = {
    val gout = new GZIPOutputStream(new FileOutputStream(fname))
    gout.write(testData.asJson.noSpaces.getBytes())
    gout.close()
  }

  def genSparseBool(dims: Int, numCorpus: Int, numQueries: Int, numNeighbors: Int)(implicit rng: Random): TestData = {
    // TODO: have a min and max bias to introduce more variety to the corpus.
    val corpus = Vec.SparseBool.randoms(dims, numCorpus, 0.2)
    val queries = Vec.SparseBool.randoms(dims, numQueries, 0.2).map { qv =>
      Query(
        qv,
        Seq(
          Result(Similarity.Jaccard, corpus.map(cv => ExactSimilarityFunction.Jaccard(cv, qv)).sorted.reverse.take(numNeighbors)),
          Result(Similarity.Hamming, corpus.map(cv => ExactSimilarityFunction.Hamming(cv, qv)).sorted.reverse.take(numNeighbors))
        )
      )
    }
    TestData(corpus, queries)
  }

  def genDenseFloat(dims: Int, numCorpus: Int, numQueries: Int, numNeighbors: Int, unit: Boolean = false)(
      implicit rng: Random): TestData = {
    val corpus = Vec.DenseFloat.randoms(dims, numCorpus)
    val queries = Vec.DenseFloat.randoms(dims, numQueries).map { qv =>
      Query(
        qv,
        Seq(
          Result(Similarity.L1, corpus.map(cv => ExactSimilarityFunction.L1(cv, qv)).sorted.reverse.take(numNeighbors)),
          Result(Similarity.L2, corpus.map(cv => ExactSimilarityFunction.L2(cv, qv)).sorted.reverse.take(numNeighbors)),
          Result(Similarity.Angular, corpus.map(cv => ExactSimilarityFunction.Angular(cv, qv)).sorted.reverse.take(numNeighbors))
        )
      )
    }
    TestData(corpus, queries)
  }
}


object Generate {

  import TestData._

  def main(args: Array[String]): Unit = {
    implicit val rng = new Random(0)
    val dims = 1024
    write(genSparseBool(dims, 5000, 50, 100), "testdata-sparsebool.json.gz")
    write(genDenseFloat(dims, 5000, 50, 100), "testdata-densefloat.json.gz")
    write(genDenseFloat(dims, 5000, 50, 100, unit = true), "testdata-densefloat-unit.json.gz")
  }

} 

package org.apache.spark.deploy.worker

import java.io.{File, FileOutputStream, InputStream, IOException}

import scala.collection.JavaConverters._
import scala.collection.Map

import org.apache.spark.Logging
import org.apache.spark.SecurityManager
import org.apache.spark.deploy.Command
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils


  def redirectStream(in: InputStream, file: File) {
    val out = new FileOutputStream(file, true)
    // TODO: It would be nice to add a shutdown hook here that explains why the output is
    //       terminating. Otherwise if the worker dies the executor logs will silently stop.
    new Thread("redirect output to " + file) {
      override def run() {
        try {
          Utils.copyStream(in, out, true)
        } catch {
          case e: IOException =>
            logInfo("Redirection to " + file + " closed: " + e.getMessage)
        }
      }
    }.start()
  }
} 

package sttp.client.internal

import java.io.{File, FileOutputStream, InputStream}

import scala.util.control.NonFatal

private[client] object FileHelpers {
  def saveFile(file: File, is: InputStream): Unit = {
    if (!file.exists()) {
      if (file.getParentFile != null) {
        file.getParentFile.mkdirs()
      }
      file.createNewFile()
    }

    withResources(new FileOutputStream(file))(os => transfer(is, os))
  }

  // based on https://medium.com/@dkomanov/scala-try-with-resources-735baad0fd7d
  private def withResources[T <: AutoCloseable, V](r: => T)(f: T => V): V = {
    val resource: T = r
    require(resource != null, "resource is null")
    var exception: Throwable = null
    try {
      f(resource)
    } catch {
      case NonFatal(e) =>
        exception = e
        throw e
    } finally {
      closeAndAddSuppressed(exception, resource)
    }
  }

  private def closeAndAddSuppressed(e: Throwable, resource: AutoCloseable): Unit = {
    if (e != null) {
      try {
        resource.close()
      } catch {
        case NonFatal(suppressed) =>
          e.addSuppressed(suppressed)
      }
    } else {
      resource.close()
    }
  }
} 

package sttp.client

import java.io.{File, FileOutputStream, IOException, InputStream}
import java.nio.file.AccessDeniedException

import sttp.client.internal._

object FileHelpers {
  private[client] def saveFile(file: File, is: InputStream): File = {
    if (!file.exists()) {
      if (file.getParentFile != null) {
        file.getParentFile.mkdirs()
      }
      try {
        file.createNewFile()
      } catch {
        case e: AccessDeniedException => throw new IOException("Permission denied", e) // aligns SN bahavior with Java
      }
    }

    val os = new FileOutputStream(file)

    transfer(is, os)
    file
  }
} 

package neuroflow.core

import java.io.{File, FileOutputStream, PrintWriter}

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import scala.util.Try


  def maybeGraph(loss: Double): Unit =
    self.settings.lossFuncOutput.foreach {
      lfo =>
        Future {
          Try {
            val handleOpt = lfo.file
              .map(f => new PrintWriter(new FileOutputStream(new File(f), true)))
            handleOpt.foreach(_.println(loss))
            handleOpt.foreach(_.close())
            lfo.action.foreach(_ (loss))
          }
        }
    }

}

case class LossFuncOutput(file: Option[String] = None, action: Option[Double => Unit] = None) 

package ai.diffy

import java.io.{File, FileOutputStream}
import java.net.ServerSocket
import java.nio.file.Files
import java.util.zip.{ZipEntry, ZipOutputStream}

import ai.diffy.examples.thrift.ExampleServers
import ai.diffy.proxy.DifferenceProxy
import ai.diffy.thriftscala.Adder
import com.google.inject.Stage
import com.twitter.finagle.http.Status
import com.twitter.finagle.util.DefaultTimer
import com.twitter.finagle.{Http, ThriftMux}
import com.twitter.finatra.http.EmbeddedHttpServer
import com.twitter.inject.Test
import com.twitter.util.{Await, Duration, Future, FuturePool}

import scala.io.Source

class ThriftFeatureTest extends Test {

  def getPort(): Int = {
    val s  = new ServerSocket(0)
    val port = s.getLocalPort
    s.close()
    port
  }

  val env@Seq(p,s,c,d) = Seq.fill(4)(getPort())
  val environment = FuturePool.unboundedPool(ExampleServers.main(env.take(3).map(_.toString).toArray))

  val diffy = new MainService
  lazy val differenceProxy = diffy.injector.instance[DifferenceProxy]


  val thriftFile = new File("src/test/thrift/example.thrift")
  val data = Source.fromInputStream(Files.newInputStream(thriftFile.toPath), "UTF-8").mkString
  val thriftJar = Files.createTempFile("thrift", "jar")
  thriftJar.toFile.deleteOnExit()
  val out = new ZipOutputStream(new FileOutputStream(thriftJar.toFile))
  out.putNextEntry(new ZipEntry(thriftFile.getAbsolutePath))
  out.write(data.getBytes)
  out.closeEntry()
  out.close()

  val server = new EmbeddedHttpServer(
    twitterServer = diffy,
    flags = Map(
      "proxy.port" -> s":$d",
      "candidate" -> s"localhost:$c",
      "master.primary" -> s"localhost:$p",
      "master.secondary" -> s"localhost:$s",
      "serviceName" -> "myThriftService",
      "service.protocol" -> "thrift",
      "thrift.jar" -> thriftJar.toAbsolutePath.toString,
      "thrift.serviceClass" -> "Adder",
      "summary.email" -> "test"
    ),
    stage = Stage.PRODUCTION
  )

  val client = ThriftMux.client.build[Adder.MethodPerEndpoint](s"localhost:$d")

  test("verify startup") {
    server.assertHealthy()
  }

  test("verify DifferenceCollector") {
    assert(differenceProxy.collector.fields.isEmpty)
    Await.result(client.add(1, 1).liftToTry)
    var tries = 0
    while(differenceProxy.outstandingRequests.get() > 0 && tries < 10) {
      Await.result(Future.sleep(Duration.fromSeconds(1))(DefaultTimer))
      tries = tries + 1
    }
    assert(!differenceProxy.collector.fields.isEmpty)
  }

  test("verify present differences via API") {
    val response =
      Await.result(Http.fetchUrl(s"http://${server.externalHttpHostAndPort}/api/1/endpoints/add/stats"))
    assertResult(Status.Ok)(response.status)
    assert(response.getContentString().contains(""""differences":1"""))
  }

  test("verify absent endpoint in API") {
    val response =
      Await.result(Http.fetchUrl(s"http://${server.externalHttpHostAndPort}/api/1/endpoints/subtract/stats"))
    assertResult(Status.Ok)(response.status)
    assertResult("""{"error":"key not found: subtract"}""")(response.getContentString())
  }

} 

package controllers.document

import controllers.HasConfig
import java.io.{File, FileInputStream, FileOutputStream, BufferedInputStream, ByteArrayInputStream, InputStream, PrintWriter}
import java.nio.file.Paths
import java.math.BigInteger
import java.security.{MessageDigest, DigestInputStream}
import java.util.UUID
import java.util.zip.{ZipEntry, ZipOutputStream}
import services.HasDate
import services.annotation.{Annotation, AnnotationService}
import services.document.{ExtendedDocumentMetadata, DocumentToJSON}
import services.generated.tables.records.{DocumentRecord, DocumentFilepartRecord}
import play.api.libs.json.Json
import play.api.libs.Files.TemporaryFileCreator
import scala.concurrent.{ExecutionContext, Future}
import storage.TempDir
import storage.uploads.Uploads

trait BackupWriter extends HasBackupValidation { self: HasConfig =>
  
  // Frontend annotation format
  import services.annotation.FrontendAnnotation._
  
  private val BUFFER_SIZE = 2048
  
  private def writeToZip(inputStream: InputStream, filename: String, zip: ZipOutputStream) = {
    zip.putNextEntry(new ZipEntry(filename))
     
    val md = MessageDigest.getInstance(ALGORITHM)    
    val in = new DigestInputStream(new BufferedInputStream(inputStream), md)

    var data= new Array[Byte](BUFFER_SIZE)
    var count: Int = 0

    while ({ count = in.read(data, 0, BUFFER_SIZE); count } > -1) {
      zip.write(data, 0, count)
    }

    in.close()
    zip.closeEntry()
    
    new BigInteger(1, md.digest()).toString(16)
  }
  
  def createBackup(doc: ExtendedDocumentMetadata)(implicit ctx: ExecutionContext, uploads: Uploads, 
      annotations: AnnotationService, tmpFile: TemporaryFileCreator): Future[File] = {
    
    def getFileAsStream(owner: String, documentId: String, filename: String) = {
      val dir = uploads.getDocumentDir(owner, documentId).get // Fail hard if the dir doesn't exist
      new FileInputStream(new File(dir, filename))
    }
    
    def getManifestAsStream() = {
      val manifest = "Recogito-Version: 2.0.1-alpha"
      new ByteArrayInputStream(manifest.getBytes)
    }
    
    def getMetadataAsStream(doc: ExtendedDocumentMetadata) = {
      
      // DocumentRecord JSON serialization
      import services.document.DocumentToJSON._
      
      val json = Json.prettyPrint(Json.toJson((doc.document, doc.fileparts)))
      new ByteArrayInputStream(json.getBytes)
    }
    
    def getAnnotationsAsStream(docId: String, annotations: Seq[Annotation], parts: Seq[DocumentFilepartRecord]): InputStream = {
      val path = Paths.get(TempDir.get()(self.config), s"${docId}_annotations.json")
      val tmp = tmpFile.create(path)
      val writer = new PrintWriter(path.toFile)
      annotations.foreach(a => writer.println(Json.stringify(Json.toJson(a))))
      writer.close()
      new FileInputStream(path.toFile)
    }
    
    Future {
      tmpFile.create(Paths.get(TempDir.get()(self.config), s"${doc.id}.zip"))
    } flatMap { zipFile =>
      val zipStream = new ZipOutputStream(new FileOutputStream(zipFile.path.toFile))

      writeToZip(getManifestAsStream(), "manifest", zipStream)
      val metadataHash = writeToZip(getMetadataAsStream(doc), "metadata.json", zipStream)

      val fileHashes = doc.fileparts.map { part =>
        writeToZip(getFileAsStream(doc.ownerName, doc.id, part.getFile), "parts" + File.separator + part.getFile, zipStream)
      }

      annotations.findByDocId(doc.id).map { annotations =>
        val annotationsHash = writeToZip(getAnnotationsAsStream(doc.id, annotations.map(_._1), doc.fileparts), "annotations.jsonl", zipStream)
        
        val signature = computeSignature(metadataHash, fileHashes, annotationsHash)
        writeToZip(new ByteArrayInputStream(signature.getBytes), "signature", zipStream)
        
        zipStream.close()
        zipFile.path.toFile
      }
    }
  }
  
} 

package korolev.effect.io

import java.io.{BufferedReader, FileInputStream, FileOutputStream, FileReader}
import java.nio.file.Path

import korolev.effect.syntax._
import korolev.effect.{Effect, Stream}

object FileIO {

  def readBytes[F[_]: Effect](path: Path): F[LazyBytes[F]] = {
    val inputStream = new FileInputStream(path.toFile)
    LazyBytes.fromInputStream(inputStream)
  }

  def readLines[F[_]: Effect](path: Path): F[Stream[F, String]] = {
    Stream.unfoldResource[F, BufferedReader, Unit, String](
      default = (),
      create = Effect[F].delay(new BufferedReader(new FileReader(path.toFile))),
      loop = (reader, _) => Effect[F].delay {
        ((), Option(reader.readLine()))
      }
    )
  }

  
  def write[F[_]: Effect](path: Path, append: Boolean = false): Stream[F, Array[Byte]] => F[Unit] = { stream =>
    val outputStream = new FileOutputStream(path.toFile, append)
    def aux(): F[Unit] = {
      stream.pull().flatMap {
        case Some(chunk) => Effect[F]
          .delay(outputStream.write(chunk))
          .after(aux())
          .recover {
            case error =>
              outputStream.close()
              throw error
          }
        case None =>
          Effect[F].delay(outputStream.close())
      }
    }
    aux()
  }
} 

package de.zalando.swagger

import java.io.{ File, FileOutputStream }

import scala.io.Source


trait ExpectedResults {

  val resourcesPath = "swagger-parser/src/test/resources/"

  def expectationsFolder: String = "/expected_results/"

  def dump(result: String, file: File, suffix: String): Unit = {
    if (result.nonEmpty) {
      val newFile = target(file, suffix)
      newFile.getParentFile.mkdirs()
      newFile.delete()
      newFile.createNewFile()
      val out = new FileOutputStream(newFile)
      out.write(result.getBytes)
      out.close()
    }
  }

  def asInFile(file: File, suffix: String): String = {
    val expectedFile = target(file, suffix)
    if (expectedFile.canRead) {
      val src = Source.fromFile(expectedFile)
      val result = src.getLines().mkString("\n")
      src.close()
      result
    } else
      ""
  }

  def target(file: File, suffix: String): File =
    new File(file.getParentFile.getParent + expectationsFolder + file.getName + "." + suffix)

  def clean(str: String): String = str.split("\n").map(_.trim).mkString("\n")
} 

package de.zalando

import java.io.{ File, FileOutputStream }

import de.zalando.apifirst.util.ScalaPrinter
import de.zalando.model._

import scala.io.Source


trait ExpectedResults {

  val model = Seq[WithModel](
    additional_properties_yaml,
    basic_polymorphism_yaml,
    nested_arrays_yaml,
    nested_options_yaml,
    basic_extension_yaml,
    expanded_polymorphism_yaml,
    nested_objects_yaml,
    options_yaml,
    wrong_field_name_yaml,
    all_of_imports_yaml,
    i038_invalid_enum_members_yaml
  )
  val examples = Seq[WithModel](
    basic_auth_api_yaml,
    cross_spec_references_yaml,
    echo_api_yaml,
    error_in_array_yaml,
    form_data_yaml,
    full_petstore_api_yaml,
    hackweek_yaml,
    heroku_petstore_api_yaml,
    instagram_api_yaml,
    minimal_api_yaml,
    nakadi_yaml,
    security_api_yaml,
    simple_petstore_api_yaml,
    split_petstore_api_yaml,
    string_formats_yaml,
    type_deduplication_yaml,
    uber_api_yaml,
    i041_no_json_deserialiser_yaml
  )
  val validations = Seq[WithModel](
    nested_arrays_validation_yaml,
    nested_objects_validation_yaml,
    nested_options_validation_yaml,
    numbers_validation_yaml,
    string_formats_validation_yaml
  )

  val resourcesPath = "play-scala-generator/src/test/resources/"

  def expectationsFolder: String = "/expected_results/"

  def dump(result: String, name: String, suffix: String): Unit = {
    if (result.nonEmpty) {
      val newFile = target(name, suffix)
      newFile.getParentFile.mkdirs()
      newFile.delete()
      newFile.createNewFile()
      val out = new FileOutputStream(newFile)
      out.write(result.getBytes)
      out.close()
    }
  }

  def asInFile(name: String, suffix: String): String = {
    val expectedFile = target(name, suffix)
    if (expectedFile.canRead) {
      val src = Source.fromFile(expectedFile)
      val result = src.getLines().mkString("\n")
      src.close()
      result
    } else
      ""
  }

  def target(name: String, suffix: String): File =
    new File(resourcesPath + expectationsFolder + name + "." + suffix)

  def clean(str: String): String = str.split("\n").map(_.trim).filter(_.nonEmpty).mkString("\n")

  def nameFromModel(ast: WithModel): String = ScalaPrinter.nameFromModel(ast)

} 

package com.ecfront.ez.framework.service.auth.helper

import java.io.{File, FileOutputStream}

import com.ecfront.ez.framework.core.logger.Logging
import com.github.cage.GCage


object CaptchaHelper extends Logging {

  def generate(text: String): File = {
    val temp = File.createTempFile("ez_captcha_", ".jpg")
    val os = new FileOutputStream(temp)
    try {
      temp.deleteOnExit()
      new GCage().draw(text, os)
      temp
    } catch {
      case e: Throwable =>
        logger.error("Generate captche error.", e)
        null
    } finally {
      os.close()
    }
  }
} 

package com.nulabinc.backlog.r2b.exporter.service

import java.io.{File, FileOutputStream}
import java.net.{HttpURLConnection, URL}
import java.nio.channels.Channels

import com.nulabinc.backlog.migration.common.utils.ControlUtil.using
import com.nulabinc.backlog.migration.common.utils.Logging

object AttachmentService extends Logging {
  private val MAX_REDIRECT_COUNT = 10

  def download(url: URL, file: File): Unit = {
    val redirected = followRedirect(url)

    doDownload(redirected, file)
  }

  private def doDownload(url: URL, file: File): Unit =
    try {
      val rbc = Channels.newChannel(url.openStream())
      val fos = new FileOutputStream(file)
      fos.getChannel.transferFrom(rbc, 0, java.lang.Long.MAX_VALUE)

      rbc.close()
      fos.close()
    } catch {
      case e: Throwable => logger.warn("Download attachment failed: " + e.getMessage)
    }

  private def followRedirect(url: URL, count: Int = 0): URL =
    url.openConnection match {
      case http: HttpURLConnection =>
        http.setRequestMethod("GET")
        http.connect()
        using(http) { connection =>
          connection.getResponseCode match {
            case 301 | 302 | 303 =>
              val newUrl = new URL(connection.getHeaderField("Location"))
              if (count < MAX_REDIRECT_COUNT) followRedirect(newUrl, count + 1) else newUrl
            case _ =>
              url
          }
        }
      case _ =>
        url
    }
} 

package com.tencent.angel.sona.tree.gbdt

import java.io.{FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream}

import com.tencent.angel.sona.tree.gbdt.tree.{GBDTParam, GBTNode}
import com.tencent.angel.sona.tree.regression.RegTree
import org.apache.spark.ml.linalg.Vector

import scala.collection.mutable.ArrayBuffer

object GBDTModel {
  type GBTTree = RegTree[GBTNode]

  def save(model: GBDTModel, path: String): Unit = {
    val oos = new ObjectOutputStream(new FileOutputStream(path))
    oos.writeObject(model)
    oos.close()
  }

  def load(path: String): GBDTModel = {
    val ois = new ObjectInputStream(new FileInputStream(path))
    ois.readObject().asInstanceOf[GBDTModel]
  }
}

import GBDTModel._
class GBDTModel(val param: GBDTParam) extends Serializable {
  private var forest: ArrayBuffer[GBTTree] = ArrayBuffer[GBTTree]()
  private var weights: ArrayBuffer[Float] = ArrayBuffer[Float]()

  def predict(instance: Vector): Array[Float] = {
    if (param.isRegression || param.numClass == 2) {
      var pred = 0.0f
      for (i <- forest.indices)
        pred += weights(i) * forest(i).predictBinary(instance)
      Array(pred)
    } else if (param.multiTree) {
      val preds = Array.ofDim[Float](param.numClass)
      for (i <- forest.indices)
        preds(i % param.numClass) += weights(i) *
          forest(i).predictBinary(instance)
      preds
    } else {
      val preds = Array.ofDim[Float](param.numClass)
      for (i <- forest.indices) {
        val p = forest(i).predictMulti(instance)
        val w = weights(i)
        for (k <- 0 until param.numClass)
          preds(k) += w * p(k)
      }
      preds
    }
  }

  def predict(instances: Array[Vector]): Array[Array[Float]] = {
    instances.map(predict)
  }

  def get(treeId: Int): GBTTree = forest(treeId)

  def add(tree: GBTTree, weight: Float): Unit = {
    forest += tree
    weights += weight
  }

  def keepFirstTrees(num: Int): Unit = {
    forest = forest.slice(0, num)
    weights = weights.slice(0, num)
  }

  def numTree: Int = forest.size
} 

package com.spotify.scio.examples.extra

import java.io.FileOutputStream
import java.nio.file.Files

import com.spotify.scio.io._
import com.spotify.scio.testing._

class SingleGZipFileExampleTest extends PipelineSpec {
  private val inData = Seq("a b c d e", "a b a b", "")
  private val expected = Seq("a: 3", "b: 3", "c: 1", "d: 1", "e: 1")

  "SingleGZipFileExample" should "work" in {
    JobTest[SingleGZipFileExample.type]
      .args("--input=in.txt", "--output=out.txt")
      .input(TextIO("in.txt"), inData)
      .output(TextIO("out.txt"))(coll => coll should containInAnyOrder(expected))
      .run()
  }

  it should "output compressed data" in {
    val tempDir = Files.createTempDirectory("single-gzip-file-")
    val in = tempDir.resolve("input").toFile
    val inFOS = new FileOutputStream(in.getAbsolutePath)
    inFOS.write(inData.mkString.getBytes)
    inFOS.close()
    val out = tempDir.resolve("output")
    SingleGZipFileExample.main(
      Array(s"--input=${in.getAbsolutePath}", s"--output=${out.toFile.getAbsolutePath}")
    )

    val outPartFile = out.resolve("part-00000-of-00001.txt.deflate").toFile
    outPartFile.exists() shouldBe true
    Files.deleteIfExists(in.toPath)
    Files.deleteIfExists(outPartFile.toPath)
    Files.deleteIfExists(out)
  }
} 

package com.spotify.scio.repl

import java.io.{File, FileOutputStream}
import java.net.{URL, URLClassLoader}
import java.nio.file.Files
import java.util.jar.{JarEntry, JarOutputStream}
import java.lang.invoke.{MethodHandles, MethodType}

import com.spotify.scio.repl.compat.ILoopClassLoader
import org.slf4j.LoggerFactory

import scala.tools.nsc.io._

object ScioReplClassLoader {
  private val Logger = LoggerFactory.getLogger(this.getClass)

  private[this] val JDK9OrHigher: Boolean = util.Properties.isJavaAtLeast("9")

  private[this] val BootClassLoader: ClassLoader = {
    if (!JDK9OrHigher) null
    else {
      try {
        MethodHandles
          .lookup()
          .findStatic(
            classOf[ClassLoader],
            "getPlatformClassLoader",
            MethodType.methodType(classOf[ClassLoader])
          )
          .invoke()
      } catch { case _: Throwable => null }
    }
  }

  def classLoaderURLs(cl: ClassLoader): Array[java.net.URL] = cl match {
    case null                       => Array.empty
    case u: java.net.URLClassLoader => u.getURLs ++ classLoaderURLs(cl.getParent)
    case _                          => classLoaderURLs(cl.getParent)
  }

  @inline final def apply(urls: Array[URL]) =
    new ScioReplClassLoader(urls, BootClassLoader)
}


  private def addVirtualDirectoryToJar(
    dir: Iterable[AbstractFile],
    entryPath: String,
    jarStream: JarOutputStream
  ): Unit = dir.foreach { file =>
    if (file.isDirectory) {
      // Recursively descend into subdirectories, adjusting the package name as we do.
      val dirPath = entryPath + file.name + "/"
      jarStream.putNextEntry(new JarEntry(dirPath))
      jarStream.closeEntry()
      addVirtualDirectoryToJar(file, dirPath, jarStream)
    } else if (file.hasExtension("class")) {
      // Add class files as an entry in the jar file and write the class to the jar.
      jarStream.putNextEntry(new JarEntry(entryPath + file.name))
      jarStream.write(file.toByteArray)
      jarStream.closeEntry()
    }
  }
} 

package sttp.tapir.server.finatra

import java.io.{File, FileOutputStream}

import com.twitter.util.logging.Logging
import com.twitter.util.{Future, FuturePool}
import sttp.tapir.Defaults
import sttp.tapir.server.{DecodeFailureHandler, LogRequestHandling, ServerDefaults}

case class FinatraServerOptions(
    createFile: Array[Byte] => Future[File],
    decodeFailureHandler: DecodeFailureHandler,
    logRequestHandling: LogRequestHandling[Unit]
)

object FinatraServerOptions extends Logging {
  implicit lazy val default: FinatraServerOptions = FinatraServerOptions(
    defaultCreateFile(futurePool),
    ServerDefaults.decodeFailureHandler,
    defaultLogRequestHandling
  )

  def defaultCreateFile(futurePool: FuturePool)(bytes: Array[Byte]): Future[File] = {
    // TODO: Make this streaming
    futurePool {
      val file = Defaults.createTempFile()
      val outputStream = new FileOutputStream(file)
      outputStream.write(bytes)
      outputStream.close()
      file
    }
  }

  private lazy val futurePool = FuturePool.unboundedPool

  lazy val defaultLogRequestHandling: LogRequestHandling[Unit] = LogRequestHandling(
    doLogWhenHandled = debugLog,
    doLogAllDecodeFailures = debugLog,
    doLogLogicExceptions = (msg: String, ex: Throwable) => error(msg, ex),
    noLog = ()
  )

  private def debugLog(msg: String, exOpt: Option[Throwable]): Unit =
    exOpt match {
      case None     => debug(msg)
      case Some(ex) => debug(msg, ex)
    }
} 

package org.mule.weave.dwnative.utils

import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.io.InputStream
import java.util.zip.ZipEntry
import java.util.zip.ZipInputStream

object UnzipHelper {

  
  def unZipIt(zipFile: InputStream, outputFolder: File): Unit = {
    val buffer = new Array[Byte](1024)
    try {
      //output directory
      val folder: File = outputFolder;
      if (!folder.exists()) {
        folder.mkdirs()
      }
      //zip file content
      val zis: ZipInputStream = new ZipInputStream(zipFile)
      //get the zipped file list entry
      var ze: ZipEntry = zis.getNextEntry();

      while (ze != null) {
        val fileName = ze.getName();
        val newFile = new File(outputFolder + File.separator + fileName);
        //create folders
        new File(newFile.getParent()).mkdirs();
        val fos = new FileOutputStream(newFile);
        var len: Int = zis.read(buffer);
        while (len > 0) {
          fos.write(buffer, 0, len)
          len = zis.read(buffer)
        }
        fos.close()
        ze = zis.getNextEntry()
      }
      zis.closeEntry()
      zis.close()
    } catch {
      case e: IOException => println("exception caught: " + e.getMessage)
    }
  }
} 

package com.protectwise.testing.ccm

import java.io.{FileOutputStream, FileInputStream, BufferedInputStream, File}
import java.util.jar.{JarEntry, JarOutputStream}

object JarHelper {

  
  def createJarForPath(path: File, targetJar: File): File = {
    def add(source: File, target: JarOutputStream): Unit = {
      var in: BufferedInputStream = null
      try {
        var name = source.getPath.replace("\\", "/").drop(path.getPath.length()+1)
        if (source.isDirectory && !name.isEmpty && !name.endsWith("/")) name += "/"
        println(s"      $name")
        if (source.isDirectory) {
          if (!name.isEmpty) {
            val entry = new JarEntry(name)
            entry.setTime(source.lastModified())
            target.putNextEntry(entry)
            target.closeEntry()
          }
          source.listFiles.foreach(add(_, target))
          return
        }

        val entry = new JarEntry(name)
        entry.setTime(source.lastModified())
        target.putNextEntry(entry)
        in = new BufferedInputStream(new FileInputStream(source))

        val buffer = Array.ofDim[Byte](1024)
        var count = 0
        while (count != -1) {
          count = in.read(buffer)
          if (count >= 0) target.write(buffer, 0, count)
        }
        target.closeEntry()
      } finally {
        if (in != null) in.close()
      }
    }

    //    val manifest = new java.util.jar.Manifest()
    val target = new JarOutputStream(new FileOutputStream(targetJar))
    add(path, target)
    target.close()

    targetJar
  }
} 

package io.deepsense.deeplang.doperations.readwritedataframe.filestorage

import java.io.{BufferedWriter, FileOutputStream, IOException, OutputStreamWriter}
import java.nio.file.{Files, Paths}
import java.util.UUID

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}

import io.deepsense.deeplang.ExecutionContext
import io.deepsense.deeplang.doperations.exceptions.DeepSenseIOException
import io.deepsense.deeplang.doperations.readwritedataframe.FilePath

private[filestorage] object FileDownloader {

  def downloadFile(url: String)(implicit context: ExecutionContext): FilePath = {
    if (context.tempPath.startsWith("hdfs://")) {
      downloadFileToHdfs(url)
    } else {
      downloadFileToDriver(url)
    }
  }

  private def downloadFileToHdfs(url: String)(implicit context: ExecutionContext) = {
    val content = scala.io.Source.fromURL(url).getLines()
    val hdfsPath = s"${context.tempPath}/${UUID.randomUUID()}"

    val configuration = new Configuration()
    val hdfs = FileSystem.get(configuration)
    val file = new Path(hdfsPath)
    val hdfsStream = hdfs.create(file)
    val writer = new BufferedWriter(new OutputStreamWriter(hdfsStream))
    try {
      content.foreach {s =>
        writer.write(s)
        writer.newLine()
      }
    } finally {
      safeClose(writer)
      hdfs.close()
    }

    FilePath(hdfsPath)
  }

  private def downloadFileToDriver(url: String)
                                  (implicit context: ExecutionContext) = {
    val outputDirPath = Paths.get(context.tempPath)
    // We're checking if the output is a directory following symlinks.
    // The default behaviour of createDirectories is NOT to follow symlinks
    if (!Files.isDirectory(outputDirPath)) {
      Files.createDirectories(outputDirPath)
    }

    val outFilePath = Files.createTempFile(outputDirPath, "download", ".csv")
    // content is a stream. Do not invoke stuff like .toList() on it.
    val content = scala.io.Source.fromURL(url).getLines()
    val writer: BufferedWriter =
      new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFilePath.toFile)))
    try {
      content.foreach {s =>
        writer.write(s)
        writer.newLine()
      }
    } finally {
      safeClose(writer)
    }
    FilePath(s"file:///$outFilePath")
  }

  private def safeClose(bufferedWriter: BufferedWriter): Unit = {
    try {
      bufferedWriter.flush()
      bufferedWriter.close()
    } catch {
      case e: IOException => throw new DeepSenseIOException(e)
    }
  }

} 

package io.deepsense.deeplang.doperations.readwritedataframe.googlestorage

import java.io.{ByteArrayInputStream, FileOutputStream}
import java.util

import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
import com.google.api.client.http.FileContent
import com.google.api.client.json.gson.GsonFactory
import com.google.api.services.drive.model.File
import com.google.api.services.drive.{Drive, DriveScopes}

import io.deepsense.commons.resources.ManagedResource
import io.deepsense.commons.utils.LoggerForCallerClass
import io.deepsense.deeplang.doperations.inout.CsvParameters.ColumnSeparatorChoice

private[googlestorage] object GoogleDriveClient {

  val logger = LoggerForCallerClass()

  val googleSheetCsvSeparator = ColumnSeparatorChoice.Comma()

  private val ApplicationName = "Seahorse"

  private val Scopes = util.Arrays.asList(DriveScopes.DRIVE)

  def uploadCsvFileAsGoogleSheet(
      credentials: GoogleCretendialsJson,
      sheetId: GoogleSheetId,
      filePath: String
    ): Unit = {
    val fileMetadata = new File().setMimeType("application/vnd.google-apps.spreadsheet")
    val mediaContent = new FileContent("text/csv", new java.io.File(filePath))

    driveService(credentials).files.update(sheetId, fileMetadata, mediaContent).execute
  }

  def downloadGoogleSheetAsCsvFile(
      credentials: GoogleCretendialsJson,
      sheetId: GoogleSheetId,
      filePath: String
    ): Unit = {
    val file = new java.io.File(filePath)
    file.getParentFile.mkdirs()

    ManagedResource(new FileOutputStream(file)) { fos =>
      driveService(credentials).files().export(sheetId, "text/csv").executeMediaAndDownloadTo(fos)
      logger.info(s"Downloaded google sheet id=$sheetId to the file $filePath")
    }
  }

  private def driveService(serviceAccountCredentials: String): Drive = {
    val credential = {
      val in = new ByteArrayInputStream(serviceAccountCredentials.getBytes)
      GoogleCredential.fromStream(in).createScoped(Scopes)
    }
    new Drive.Builder(
      GoogleNetHttpTransport.newTrustedTransport(),
      jsonFactory,
      credential
    ).setApplicationName(ApplicationName).build
  }

  // Default choice is JacksonFactory. However spark depends on Jackson as well
  // and google/spark jackson versions are binary incompatible with each other.
  private val jsonFactory = GsonFactory.getDefaultInstance

}