org.apache.commons.codec.binary.Base64 Scala Examples

package org.apache.spark.sql.catalyst.plans.logical

import org.apache.commons.codec.binary.Base64

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.sql.types._


case class ColumnStat(statRow: InternalRow) {

  def forNumeric[T <: AtomicType](dataType: T): NumericColumnStat[T] = {
    NumericColumnStat(statRow, dataType)
  }
  def forString: StringColumnStat = StringColumnStat(statRow)
  def forBinary: BinaryColumnStat = BinaryColumnStat(statRow)
  def forBoolean: BooleanColumnStat = BooleanColumnStat(statRow)

  override def toString: String = {
    // use Base64 for encoding
    Base64.encodeBase64String(statRow.asInstanceOf[UnsafeRow].getBytes)
  }
}

object ColumnStat {
  def apply(numFields: Int, str: String): ColumnStat = {
    // use Base64 for decoding
    val bytes = Base64.decodeBase64(str)
    val unsafeRow = new UnsafeRow(numFields)
    unsafeRow.pointTo(bytes, bytes.length)
    ColumnStat(unsafeRow)
  }
}

case class NumericColumnStat[T <: AtomicType](statRow: InternalRow, dataType: T) {
  // The indices here must be consistent with `ColumnStatStruct.numericColumnStat`.
  val numNulls: Long = statRow.getLong(0)
  val max: T#InternalType = statRow.get(1, dataType).asInstanceOf[T#InternalType]
  val min: T#InternalType = statRow.get(2, dataType).asInstanceOf[T#InternalType]
  val ndv: Long = statRow.getLong(3)
}

case class StringColumnStat(statRow: InternalRow) {
  // The indices here must be consistent with `ColumnStatStruct.stringColumnStat`.
  val numNulls: Long = statRow.getLong(0)
  val avgColLen: Double = statRow.getDouble(1)
  val maxColLen: Long = statRow.getInt(2)
  val ndv: Long = statRow.getLong(3)
}

case class BinaryColumnStat(statRow: InternalRow) {
  // The indices here must be consistent with `ColumnStatStruct.binaryColumnStat`.
  val numNulls: Long = statRow.getLong(0)
  val avgColLen: Double = statRow.getDouble(1)
  val maxColLen: Long = statRow.getInt(2)
}

case class BooleanColumnStat(statRow: InternalRow) {
  // The indices here must be consistent with `ColumnStatStruct.booleanColumnStat`.
  val numNulls: Long = statRow.getLong(0)
  val numTrues: Long = statRow.getLong(1)
  val numFalses: Long = statRow.getLong(2)
} 

package org.github.xubo245.carbonDataLearning.test

import org.apache.commons.codec.binary.Base64

object base64 {
    def main(args: Array[String]): Unit = {
        var bytes = "abc".getBytes()
        var value = Base64.isArrayByteBase64(bytes)
        println(value)

        bytes = "\u0001abc".getBytes()
        value = Base64.isArrayByteBase64(bytes)
        println(value)

        bytes = "^abc".getBytes()
        value = Base64.isArrayByteBase64(bytes)
        println(value)

        var result = Base64.decodeBase64(bytes);
        println(new String(result))

        result = Base64.encodeBase64(bytes);
        println(new String(result))

    }
} 

package com.webank.wedatasphere.linkis.storage.resultset.io

import com.webank.wedatasphere.linkis.common.io.resultset.ResultSerializer
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
import org.apache.commons.codec.binary.Base64


class IOResultSerializer extends ResultSerializer{

  override def metaDataToBytes(metaData: MetaData): Array[Byte] = {
    val ioMetaData = metaData.asInstanceOf[IOMetaData]
    lineToBytes(s"${ioMetaData.off}${Dolphin.COL_SPLIT}${ioMetaData.len}")
  }

  override def recordToBytes(record: Record): Array[Byte] = {
    val ioRecord = record.asInstanceOf[IORecord]
    lineToBytes(Base64.encodeBase64String(ioRecord.value))
  }

  def lineToBytes(value: String): Array[Byte] = {
    val bytes = if(value == null) Dolphin.NULL_BYTES else Dolphin.getBytes(value)
    Dolphin.getIntBytes(bytes.length) ++ bytes
  }
} 

package org.apache.iota.fey

import java.io.FileOutputStream
import java.net.URL
import java.io.File

import com.eclipsesource.schema._
import org.slf4j.LoggerFactory
import play.api.libs.json._
import JSON_PATH._
import java.nio.file.{Files, Paths}

import org.apache.commons.io.IOUtils
import org.apache.commons.codec.binary.Base64
import scala.util.Properties._


  def exceptionOnRun(e: Exception): Unit
}

object HttpBasicAuth {
  val BASIC = "Basic"
  val AUTHORIZATION = "Authorization"

  def encodeCredentials(username: String, password: String): String = {
    new String(Base64.encodeBase64((username + ":" + password).getBytes))
  }

  def getHeader(username: String, password: String): String =
    BASIC + " " + encodeCredentials(username, password)
} 

package tsec.jws

import io.circe.{DecodingFailure, Error}
import org.apache.commons.codec.binary.Base64
import tsec.common.NonFatal

trait JWSSerializer[A] {
  def serializeToUtf8(body: A): Array[Byte]
  def fromUtf8Bytes(array: Array[Byte]): Either[Error, A]
  def toB64URL(elem: A): String = Base64.encodeBase64URLSafeString(serializeToUtf8(elem))
  def fromB64URL(encoded: String): Either[Error, A] =
    try {
      fromUtf8Bytes(Base64.decodeBase64(encoded))
    } catch {
      case t: Throwable if NonFatal(t) =>
        Left(DecodingFailure("Invalid encoding", Nil))
    }
} 

package io.flow.dependency.api.lib

import io.flow.dependency.v0.models.{Credentials, CredentialsUndefinedType, UsernamePassword}
import org.htmlcleaner.HtmlCleaner
import org.apache.commons.codec.binary.Base64
import org.apache.commons.lang3.StringUtils
import org.apache.commons.text.StringEscapeUtils
import java.net.URL


import scala.util.{Failure, Success, Try}



object RemoteDirectory {

  case class Result(
    directories: Seq[String] = Nil,
    files: Seq[String] = Nil
  )

  def fetch(
    url: String,
    credentials: Option[Credentials] = None
  ) (
    filter: String => Boolean = { !_.startsWith(".") }
  ): Result = {
    val base = Result()
    val cleaner = new HtmlCleaner()

    val uc = (new URL(url)).openConnection()
    credentials.map { cred =>
      cred match {
        case UsernamePassword(username, password) =>{
          val userpass = username + ":" + password.getOrElse("")
          val basicAuth = "Basic " + new String(new Base64().encode(userpass.getBytes()))
          uc.setRequestProperty ("Authorization", basicAuth)
        }
        case CredentialsUndefinedType(_) => {
          // No-op
        }
      }
    }

    Try(cleaner.clean(uc.getInputStream())) match {
      case Failure(_) => {
        base
      }
      case Success(rootNode) => {
        rootNode.getElementsByName("a", true).foldLeft(base) { case (result, elem) =>
          Option(elem.getAttributeByName("href")) match {
            case None => {
              result
            }
            case Some(_) => {
              val text = StringEscapeUtils.unescapeHtml4(elem.getText.toString)
              filter(StringUtils.stripEnd(text, "/")) match {
                case false => {
                  result
                }
                case true => {
                  text.endsWith("/") match {
                    case true => result.copy(directories = result.directories ++ Seq(text))
                    case false => result.copy(files = result.files ++ Seq(text))
                  }
                }
              }
            }
          }
        }
      }
    }
  }
} 

package stellar.sdk.util

import java.security.MessageDigest

import okio.ByteString
import org.apache.commons.codec.binary.Base64

import scala.annotation.tailrec
import scala.language.implicitConversions

object ByteArrays {

  object Implicits {
    implicit def byteArrayToByteString(arr: Array[Byte]): ByteString =
      new ByteString(arr)

    implicit def byteStringToByteArray(byteString: ByteString): Array[Byte] =
      byteString.toByteArray
  }

  def paddedByteArray(bs: Array[Byte], length: Int): Array[Byte] = {
    val padded = Array.ofDim[Byte](math.max(length, bs.length))
    System.arraycopy(bs, 0, padded, 0, bs.length)
    padded
  }

  def paddedByteArray(s: String, length: Int): Array[Byte] = paddedByteArray(s.getBytes("US-ASCII"), length)

  def paddedByteArrayToString(bs: Array[Byte]): String = new String(bs, "US-ASCII").split("\u0000")(0)

  def trimmedByteArray(bs: Array[Byte]): Seq[Byte] = trimmedByteArray(bs.toIndexedSeq)
  def trimmedByteArray(bs: Seq[Byte]): Seq[Byte] = bs.reverse.dropWhile(_ == 0).reverse

  def sha256(bs: Array[Byte]): Array[Byte] = sha256(bs.toIndexedSeq)
  def sha256(bs: Seq[Byte]): Array[Byte] = {
    val md = MessageDigest.getInstance("SHA-256")
    md.update(bs.toArray)
    md.digest
  }

  def base64(bs: Seq[Byte]): String = base64(bs.toArray)
  def base64(bs: Array[Byte]): String = Base64.encodeBase64String(bs)

  def base64(s: String): Array[Byte] = Base64.decodeBase64(s)

  def bytesToHex(bs: Array[Byte]): String = bytesToHex(bs.toIndexedSeq)
  def bytesToHex(bs: Seq[Byte]): String = bs.map("%02X".format(_)).mkString

  def hexToBytes(hex: String): Seq[Byte] = hex.toSeq.sliding(2, 2).map(_.unwrap).map(Integer.parseInt(_, 16).toByte).toIndexedSeq

  def checksum(bytes: Array[Byte]): Array[Byte] = {
    // This code calculates CRC16-XModem checksum
    // Ported from https://github.com/alexgorbatchev/node-crc, via https://github.com/stellar/java-stellar-sdk

    @tailrec
    def loop(bs: Seq[Byte], crc: Int): Int = {
      bs match {
        case h +: t =>
          var code = crc >>> 8 & 0xFF
          code ^= h & 0xFF
          code ^= code >>> 4
          var crc_ = crc << 8 & 0xFFFF
          crc_ ^= code
          code = code << 5 & 0xFFFF
          crc_ ^= code
          code = code << 7 & 0xFFFF
          crc_ ^= code
          loop(t, crc_)
        case Nil => crc
      }
    }

    val crc = loop(bytes.toIndexedSeq, 0x0000)
    Array(crc.toByte, (crc >>> 8).toByte)
  }

} 

package stellar.sdk.model.op

import org.apache.commons.codec.binary.Base64
import org.json4s.NoTypeHints
import org.json4s.native.JsonMethods.parse
import org.json4s.native.Serialization
import org.scalacheck.{Arbitrary, Gen}
import org.specs2.mutable.Specification
import stellar.sdk.util.ByteArrays.base64
import stellar.sdk.{ArbitraryInput, DomainMatchers, PublicKey}

class ManageDataOperationSpec extends Specification with ArbitraryInput with DomainMatchers with JsonSnippets {

  implicit val arbDelete: Arbitrary[Transacted[DeleteDataOperation]] = Arbitrary(genTransacted(genDeleteDataOperation))
  implicit val arbWrite: Arbitrary[Transacted[WriteDataOperation]] = Arbitrary(genTransacted(genWriteDataOperation))
  implicit val formats = Serialization.formats(NoTypeHints) + TransactedOperationDeserializer

  def doc[O <: ManageDataOperation](op: Transacted[O]) = {
    val dataValue = op.operation match {
      case WriteDataOperation(_, value, _) => Base64.encodeBase64String(value.toArray)
      case _ => ""
    }

    s"""
      |{
      |  "_links": {
      |    "self": {"href": "https://horizon-testnet.stellar.org/operations/10157597659144"},
      |    "transaction": {"href": "https://horizon-testnet.stellar.org/transactions/17a670bc424ff5ce3b386dbfaae9990b66a2a37b4fbe51547e8794962a3f9e6a"},
      |    "effects": {"href": "https://horizon-testnet.stellar.org/operations/10157597659144/effects"},
      |    "succeeds": {"href": "https://horizon-testnet.stellar.org/effects?order=desc\u0026cursor=10157597659144"},
      |    "precedes": {"href": "https://horizon-testnet.stellar.org/effects?order=asc\u0026cursor=10157597659144"}
      |  },
      |  "id": "${op.id}",
      |  "paging_token": "10157597659137",
      |  "source_account": "${op.operation.sourceAccount.get.accountId}",
      |  "type": "manage_data",
      |  "type_i": 1,
      |  "created_at": "${formatter.format(op.createdAt)}",
      |  "transaction_hash": "${op.txnHash}",
      |  "name": "${op.operation.name}",
      |  "value": "$dataValue"
      |}""".stripMargin
  }

  "a write data operation" should {
    "serde via xdr string" >> prop { actual: WriteDataOperation =>
      Operation.decodeXDR(base64(actual.encode)) must beEquivalentTo(actual)
    }

    "serde via xdr bytes" >> prop { actual: WriteDataOperation =>
      val (remaining, decoded) = Operation.decode.run(actual.encode).value
      decoded must beEquivalentTo(actual)
      remaining must beEmpty
    }

    "parse from json" >> prop { op: Transacted[WriteDataOperation] =>
      parse(doc(op)).extract[Transacted[ManageDataOperation]] must beEquivalentTo(op)
    }.setGen(genTransacted(genWriteDataOperation.suchThat(_.sourceAccount.nonEmpty)))

    "encode a string payload as UTF-8 in base64" >> prop { (s: String, source: PublicKey) =>
      val value = new String(s.take(64).getBytes("UTF-8").take(60), "UTF-8")
      WriteDataOperation("name", value).value.toSeq mustEqual value.getBytes("UTF-8").toSeq
      WriteDataOperation("name", value, None).value.toSeq mustEqual value.getBytes("UTF-8").toSeq
      WriteDataOperation("name", value, Some(source)).value.toSeq mustEqual value.getBytes("UTF-8").toSeq
    }.setGen1(Arbitrary.arbString.arbitrary.suchThat(_.nonEmpty))

    "fail if the key is greater than 64 bytes" >> prop { s: String =>
      WriteDataOperation(s, "value") must throwAn[IllegalArgumentException]
    }.setGen(Gen.identifier.suchThat(_.getBytes("UTF-8").length > 64))

    "fail if the value is greater than 64 bytes" >> prop { s: String =>
      WriteDataOperation("name", s) must throwAn[IllegalArgumentException]
    }.setGen(Gen.identifier.suchThat(_.getBytes("UTF-8").length > 64))
  }

  "a delete data operation" should {
    "serde via xdr string" >> prop { actual: DeleteDataOperation =>
      Operation.decodeXDR(base64(actual.encode)) must beEquivalentTo(actual)
    }

    "serde via xdr bytes" >> prop { actual: DeleteDataOperation =>
      val (remaining, decoded) = Operation.decode.run(actual.encode).value
      decoded mustEqual actual
      remaining must beEmpty
    }

    "parse from json" >> prop { op: Transacted[DeleteDataOperation] =>
      parse(doc(op)).extract[Transacted[ManageDataOperation]] mustEqual op
    }.setGen(genTransacted(genDeleteDataOperation.suchThat(_.sourceAccount.nonEmpty)))
  }

} 

package org.apache.livy

import java.io.{Closeable, File, InputStreamReader}
import java.net.URL
import java.nio.charset.StandardCharsets.UTF_8
import java.security.SecureRandom
import java.util.Properties

import scala.annotation.tailrec
import scala.collection.JavaConverters._
import scala.concurrent.TimeoutException
import scala.concurrent.duration.Duration

import org.apache.commons.codec.binary.Base64

object Utils {
  def getPropertiesFromFile(file: File): Map[String, String] = {
    loadProperties(file.toURI().toURL())
  }

  def loadProperties(url: URL): Map[String, String] = {
    val inReader = new InputStreamReader(url.openStream(), UTF_8)
    try {
      val properties = new Properties()
      properties.load(inReader)
      properties.stringPropertyNames().asScala.map { k =>
        (k, properties.getProperty(k).trim())
      }.toMap
    } finally {
      inReader.close()
    }
  }

  
  def isProcessAlive(process: Process): Boolean = {
    try {
      process.exitValue()
      false
    } catch {
      case _: IllegalThreadStateException =>
        true
    }
  }

  def startDaemonThread(name: String)(f: => Unit): Thread = {
    val thread = new Thread(name) {
      override def run(): Unit = f
    }
    thread.setDaemon(true)
    thread.start()
    thread
  }

  def usingResource[A <: Closeable, B](resource: A)(f: A => B): B = {
    try {
      f(resource)
    } finally {
      resource.close()
    }
  }

  def createSecret(secretBitLength: Int): String = {
    val rnd = new SecureRandom()
    val secretBytes = new Array[Byte](secretBitLength / java.lang.Byte.SIZE)
    rnd.nextBytes(secretBytes)

    Base64.encodeBase64String(secretBytes)
  }
} 

package controller.upload

import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec

import _root_.controller.UploadController
import lib._
import model.Upload
import model.typebinder.UserId
import org.apache.commons.codec.binary.Base64
import org.joda.time.format.DateTimeFormat
import org.joda.time._
import skinny._

object S3UploadController extends UploadController {

  override def destination: UploadDestination = UploadDestination("s3")

  // --------------
  // sign
  val AWS_ACCESS_KEY = "AWS_ACCESS_KEY"
  val AWS_SECRET_KEY = "AWS_SECRET_KEY"
  def sign: String = {
    val userId = policiesParams.getAs[UserId]("user_id").get
    val filename = params("filename")
    val ext = filename.split('.').last
    val seed = userId.value + "_" + DateTime.now().toString + "_" + filename
    val baseDir = SkinnyConfig.stringConfigValue("upload.s3.baseDir").getOrElse("")
    val path = baseDir + new Sha1Digest(seed).digestString + "." + ext

    val bucket = SkinnyConfig.stringConfigValue("upload.s3.bucket").getOrElse(throw new IllegalArgumentException)
    val policyDocument = toJSONString(Map(
      "expiration" -> new DateTime(DateTimeZone.UTC).plusMinutes(1).toString(DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z")),
      "conditions" -> Array(
        Map("bucket" -> bucket),
        Map("key" -> path),
        Map("Content-Type" -> policiesParams.getAs("content_type")),
        Array("content-length-range", policiesParams.getAs("size"), policiesParams.getAs("size"))
      )
    ), underscoreKeys = false)
    val policy = Base64.encodeBase64String(policyDocument.getBytes("UTF-8"))
    val hmac = Mac.getInstance("HmacSHA1")
    hmac.init(new SecretKeySpec(sys.env(AWS_SECRET_KEY).getBytes("UTF-8"), "HmacSHA1"))
    val signature = Base64.encodeBase64String(hmac.doFinal(policy.getBytes("UTF-8")))

    // add to uploads table
    Upload.createWithAttributes(
      'user_id -> userId.value,
      'original_filename -> filename,
      'filename -> path
    )

    // response
    toJSONString(Map(
      "url" -> s"https://$bucket.s3.amazonaws.com/",
      "form" -> Map(
        "AWSAccessKeyId" -> sys.env(AWS_ACCESS_KEY),
        "signature" -> signature,
        "policy" -> policy,
        "key" -> path,
        "Content-Type" -> policiesParams.getAs("content_type")
      )
    ), underscoreKeys = false)
  }

  // --------------
  override def upload: Any = throw new UnsupportedOperationException

  // --------------
  override def uploadedFileBaseUrl: UploadedBaseURL = UploadedBaseURL("s3")

} 

package com.aol.one.dwh.infra.consul

import com.aol.one.dwh.infra.util.LogTrait
import com.ecwid.consul.ConsulException
import com.ecwid.consul.v1.kv.model.GetValue
import com.ecwid.consul.v1.{ConsulClient, Response}
import org.apache.commons.codec.binary.Base64



class ConsulManager(consulClient: ConsulClient) extends LogTrait {

  def getFlag(path: String): Option[String] =
    try {
      fromOptional(consulClient.getKVValue(path))
    } catch {
      case e: ConsulException =>
        logger.warn(s"Could not access Consul instance to get flag", e)
        None
    }

  private def fromOptional[T >: String](x: Response[GetValue]): Option[T] = {
    Option(x.getValue).flatMap(x => decodeBase64(x.getValue))
  }

  private def decodeBase64(value: String): Option[String] = {
    Option(Base64.decodeBase64(value)).map(new String(_))
  }
} 

package akka.http.extensions.security

import java.security.MessageDigest
import javax.crypto.Cipher
import javax.crypto.spec.{IvParameterSpec, SecretKeySpec}

import org.apache.commons.codec.binary.Base64

object CryptoConfig {

  implicit def fromString(secret:String): CryptoConfig = CryptoConfig(secret)
}

case class CryptoConfig( secret: String, transformation: String = "AES/CTR/NoPadding")

class CryptoException(val message: String = null, val throwable: Throwable = null)
  extends RuntimeException(message, throwable)

trait Encryption {

  def algorithm:String

  def encrypt(value: String, config:CryptoConfig): String

  def decrypt(value: String, config:CryptoConfig): String
}

object AES extends Encryption {

  val algorithm = "AES"

  protected def secretKeyWithSha256(privateKey: String) = {
    val messageDigest = MessageDigest.getInstance("SHA-256")
    messageDigest.update(privateKey.getBytes("utf-8"))
    // max allowed length in bits / (8 bits to a byte)
    val maxAllowedKeyLength = Cipher.getMaxAllowedKeyLength(algorithm) / 8
    val raw = messageDigest.digest().slice(0, maxAllowedKeyLength)
    new SecretKeySpec(raw, algorithm)
  }


  def encrypt(value: String, config:CryptoConfig): String = {
    val skeySpec = secretKeyWithSha256(config.secret)
    val cipher = Cipher.getInstance(config.transformation)
    cipher.init(Cipher.ENCRYPT_MODE, skeySpec)
    val encryptedValue = cipher.doFinal(value.getBytes("utf-8"))
    Option(cipher.getIV) match {
      case Some(iv) => s"2-${Base64.encodeBase64String(iv ++ encryptedValue)}"
      case None => s"1-${Base64.encodeBase64String(encryptedValue)}"
    }
  }



  def decrypt(value: String, config:CryptoConfig): String =  value.indexOf("-") match
  {
    case sepIndex if sepIndex<0=> throw new CryptoException("Outdated AES version")
    case sepIndex=>
      val data = value.substring(sepIndex + 1, value.length())
      val version = value.substring(0, sepIndex)
      version match {
        case "1" => decryptAES1(data, config)
        case "2" => decryptAES2(data, config)
        case _ =>  throw new CryptoException("Unknown version")
        }
  }


  
  private def decryptAES2(value: String, config:CryptoConfig): String = {
    val cipher = Cipher.getInstance(config.transformation)
    val blockSize = cipher.getBlockSize
    val data = Base64.decodeBase64(value)
    val iv = data.slice(0, blockSize)
    val payload = data.slice(blockSize, data.size)
    val skeySpec = secretKeyWithSha256(config.secret)
    cipher.init(Cipher.DECRYPT_MODE, skeySpec, new IvParameterSpec(iv))
    new String(cipher.doFinal(payload), "utf-8")
  }

} 

package services.user

import javax.crypto.Cipher
import javax.crypto.spec.SecretKeySpec
import org.apache.commons.codec.binary.Base64
import java.security.MessageDigest
import java.util.Arrays
import controllers.HasConfig

trait HasEncryption { self: HasConfig =>

  private val CIPHER = "AES/ECB/PKCS5Padding"

  private lazy val keySpec = self.config.getOptional[String]("recogito.email.key").flatMap { key =>
    if (key.isEmpty) {
      None
    } else {
      val md = MessageDigest.getInstance("SHA-1")
      val keyDigest = md.digest(key.getBytes)
      Some(new SecretKeySpec(Arrays.copyOf(keyDigest, 16), "AES"))
    }
  }

  def encrypt(plaintext: String) = keySpec match {
    case Some(spec) => {
      val cipher = Cipher.getInstance(CIPHER)
      cipher.init(Cipher.ENCRYPT_MODE, spec)
      Base64.encodeBase64String(cipher.doFinal(plaintext.getBytes("UTF-8")))
    }

    case None => plaintext
  }

  def decrypt(encrypted: String) = keySpec match {
    case Some(spec) => {
      val cipher = Cipher.getInstance(CIPHER)
      cipher.init(Cipher.DECRYPT_MODE, spec)
      new String(cipher.doFinal(Base64.decodeBase64(encrypted)))
    }

    case None => encrypted
  }

} 

package com.eharmony.aloha.models.vw.jni.multilabel

import java.io.{ByteArrayOutputStream, File, FileInputStream}

import com.eharmony.aloha.ModelSerializationTestHelper
import com.eharmony.aloha.io.sources.{Base64StringSource, ExternalSource, ModelSource}
import org.apache.commons.codec.binary.Base64
import org.apache.commons.io.IOUtils
import org.junit.Assert._
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.BlockJUnit4ClassRunner
import vowpalWabbit.learner.{VWActionScoresLearner, VWLearners}


@RunWith(classOf[BlockJUnit4ClassRunner])
class VwSparseMultilabelPredictorTest extends ModelSerializationTestHelper {
  import VwSparseMultilabelPredictorTest._

  @Test def testSerializability(): Unit = {
    val predictor = getPredictor(getModelSource(), 3)
    val ds = serializeDeserializeRoundTrip(predictor)
    assertEquals(predictor, ds)
    assertEquals(predictor.vwParams(), ds.vwParams())
    assertNotNull(ds.vwModel)
  }

  @Test def testVwParameters(): Unit = {
    val numLabelsInTrainingSet = 3
    val predictor = getPredictor(getModelSource(), numLabelsInTrainingSet)

    predictor.vwParams() match {
      case Data(vwBinFilePath, ringSize) =>
        checkVwBinFile(vwBinFilePath)
        checkVwRingSize(numLabelsInTrainingSet, ringSize.toInt)
      case ps => fail(s"Unexpected VW parameters format.  Found string: $ps")
    }
  }
}

object VwSparseMultilabelPredictorTest {
  private val Data = """\s*-i\s+(\S+)\s+--ring_size\s+(\d+)\s+--testonly\s+--quiet""".r

  private def getModelSource(): ModelSource = {
    val f = File.createTempFile("i_dont", "care")
    f.deleteOnExit()
    val learner = VWLearners.create[VWActionScoresLearner](s"--quiet --csoaa_ldf mc --csoaa_rank -f ${f.getCanonicalPath}")
    learner.close()
    val baos = new ByteArrayOutputStream()
    IOUtils.copy(new FileInputStream(f), baos)
    val src = Base64StringSource(Base64.encodeBase64URLSafeString(baos.toByteArray))
    ExternalSource(src.localVfs)
  }

  private def getPredictor(modelSrc: ModelSource, numLabelsInTrainingSet: Int) =
    VwSparseMultilabelPredictor[Any](modelSrc, Nil, Nil, numLabelsInTrainingSet)

  private def checkVwBinFile(vwBinFilePath: String): Unit = {
    val vwBinFile = new File(vwBinFilePath)
    assertTrue("VW binary file should have been written to disk", vwBinFile.exists())
    vwBinFile.deleteOnExit()
  }

  private def checkVwRingSize(numLabelsInTrainingSet: Int, ringSize: Int): Unit = {
    assertEquals(
      "vw --ring_size parameter is incorrect:",
      numLabelsInTrainingSet + VwSparseMultilabelPredictor.AddlVwRingSize,
      ringSize.toInt
    )
  }
} 

package com.eharmony.aloha.cli.dataset

import java.io.{ByteArrayOutputStream, IOException}
import java.util.Arrays

import com.eharmony.aloha.test.proto.Testing.{PhotoProto, UserProto}
import com.eharmony.aloha.test.proto.Testing.GenderProto.{FEMALE, MALE}
import com.google.protobuf.GeneratedMessage
import org.apache.commons.codec.binary.Base64
import org.junit.runner.RunWith
import org.junit.runners.BlockJUnit4ClassRunner
import org.junit.{Ignore, Test}


@RunWith(classOf[BlockJUnit4ClassRunner])
@Ignore
class PrintProtosTest {
    @Test def testPrintProtos(): Unit = {
        System.out.println(alan)
        System.out.println(kate)
    }

    @throws(classOf[IOException])
    def alan: String = {
        val t = UserProto.newBuilder.
            setId(1).
            setName("Alan").
            setGender(MALE).
            setBmi(23).
            addAllPhotos(Arrays.asList(
                PhotoProto.newBuilder.
                    setId(1).
                    setAspectRatio(1).
                    setHeight(1).
                    build,
                PhotoProto.newBuilder.
                    setId(2).
                    setAspectRatio(2).
                    setHeight(2).build
            )).build
        b64(t)
    }

    def kate: String = {
        val t = UserProto.newBuilder.
            setId(1).
            setName("Kate").
            setGender(FEMALE).
            addAllPhotos(Arrays.asList(
                PhotoProto.newBuilder.
                    setId(3).
                    setAspectRatio(3).
                    setHeight(3).
                    build
            )).build
        b64(t)
    }

    def b64[M <: GeneratedMessage](p: M): String = {
        val baos: ByteArrayOutputStream = new ByteArrayOutputStream
        p.writeTo(baos)
        new String(Base64.encodeBase64(baos.toByteArray))
    }
} 

package com.spotify.scio.jdbc.sharded

import java.nio.charset.StandardCharsets
import org.apache.commons.codec.binary.Base64

sealed trait ShardString extends Serializable {
  val value: String
}

object ShardString {

  final case class HexUpperString(value: String) extends ShardString {
    override def toString: String = value.toUpperCase
  }

  final case class HexLowerString(value: String) extends ShardString {
    override def toString: String = value.toLowerCase
  }

  final case class Base64String(value: String) extends ShardString

}

trait RangeShardStringCodec[T <: ShardString] extends Serializable {
  def encode(bigInt: BigInt): T
  def decode(str: T): BigInt
  def lift(str: String): T
}

object RangeShardStringCodec {
  import ShardString._

  implicit val hexUpperShardRangeStringCodec: RangeShardStringCodec[HexUpperString] =
    new RangeShardStringCodec[HexUpperString] {
      def encode(bigInt: BigInt): HexUpperString = lift(bigInt.toString(16).toUpperCase)
      def decode(str: HexUpperString): BigInt = BigInt(str.value, 16)
      def lift(str: String): HexUpperString = HexUpperString(str)
    }

  implicit val hexLowerShardRangeStringCodec: RangeShardStringCodec[HexLowerString] =
    new RangeShardStringCodec[HexLowerString] {
      def encode(bigInt: BigInt): HexLowerString = lift(bigInt.toString(16).toLowerCase)
      def decode(str: HexLowerString): BigInt = BigInt(str.value, 16)
      def lift(str: String): HexLowerString = HexLowerString(str)
    }

  implicit val base64ShardRangeStringCodec: RangeShardStringCodec[Base64String] =
    new RangeShardStringCodec[Base64String] {
      def encode(bigInt: BigInt): Base64String =
        lift(new String(Base64.encodeInteger(bigInt.bigInteger), StandardCharsets.UTF_8))
      def decode(str: Base64String): BigInt =
        Base64.decodeInteger(str.value.getBytes(StandardCharsets.UTF_8))
      def lift(str: String): Base64String = Base64String(str)
    }

}