scala.collection.mutable.Map Scala Examples

The following examples show how to use scala.collection.mutable.Map. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
Example 1
Source File: Structs.scala    From Graphviz4S   with MIT License 5 votes vote down vote up
package com.liangdp.graphviz4s.examples

import com.liangdp.graphviz4s.Digraph
import scala.collection.mutable.Map


object Structs {
  def main(args: Array[String]): Unit = {
    val s = new Digraph("structs")

    s.node("struct1", """<
      <TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
        <TR>
          <TD>left</TD>
          <TD PORT="f1">middle</TD>
          <TD PORT="f2">right</TD>
        </TR>
      </TABLE>>""")
    s.node("struct2", """<
      <TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
        <TR>
          <TD PORT="f0">one</TD>
          <TD>two</TD>
        </TR>
      </TABLE>>""")
    s.node("struct3", """<
    <TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0" CELLPADDING="4">
      <TR>
        <TD ROWSPAN="3">hello<BR/>world</TD>
        <TD COLSPAN="3">b</TD>
        <TD ROWSPAN="3">g</TD>
        <TD ROWSPAN="3">h</TD>
      </TR>
      <TR>
        <TD>c</TD>
        <TD PORT="here">d</TD>
        <TD>e</TD>
      </TR>
      <TR>
        <TD COLSPAN="3">f</TD>
      </TR>
    </TABLE>>""")

    s.edges(Array(("struct1:f1", "struct2:f0"), ("struct1:f2", "struct3:here")))

    s.view(fileName = "structs.gv", directory = ".")
  }
} 
Example 2
Source File: driver.scala    From SparkSMOTE   with MIT License 5 votes vote down vote up
import java.io._
import utils._
import SMOTE._
import org.apache.log4j.Logger
import org.apache.log4j.Level
import breeze.linalg._
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import scala.collection.mutable.{ArrayBuffer,Map}


object driver {
	
	def main(args: Array[String]) {
			
		val conf = new SparkConf()

		val options = args.map { arg =>
			arg.dropWhile(_ == '-').split('=') match {
				case Array(opt, v) => (opt -> v)
				case Array(opt) => (opt -> "")
				case _ => throw new IllegalArgumentException("Invalid argument: "+arg)
			}
		}.toMap

        val rootLogger = Logger.getRootLogger()
        rootLogger.setLevel(Level.ERROR)

		val sc = new SparkContext(conf)	

		// read in general inputs
		val inputDirectory = options.getOrElse("inputDirectory","")
		val outputDirectory = options.getOrElse("outputDirectory","")
		val numFeatures = options.getOrElse("numFeatures","0").toInt
		val oversamplingPctg = options.getOrElse("oversamplingPctg","1.0").toDouble
        val kNN = options.getOrElse("K","5").toInt
		val delimiter = options.getOrElse("delimiter",",")
		val numPartitions = options.getOrElse("numPartitions","20").toInt

		SMOTE.runSMOTE(sc, inputDirectory, outputDirectory, numFeatures, oversamplingPctg, kNN, delimiter, numPartitions)	

		println("The algorithm has finished running")
		sc.stop()
	}
} 
Example 3
Source File: KyuubiDistributedCacheManager.scala    From kyuubi   with Apache License 2.0 5 votes vote down vote up
package org.apache.spark.deploy.yarn

import java.net.URI

import scala.collection.mutable.{HashMap, Map}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.hadoop.yarn.api.records.{LocalResource, LocalResourceType}


  def addResource(
      fs: FileSystem,
      conf: Configuration,
      destPath: Path,
      localResources: HashMap[String, LocalResource],
      resourceType: LocalResourceType,
      link: String,
      statCache: Map[URI, FileStatus]): Unit = {
    cacheManager.addResource(fs, conf, destPath,
      localResources, resourceType, link, statCache, appMasterOnly = true)
  }
} 
Example 4
Source File: KyuubiDistributedCacheManagerSuite.scala    From kyuubi   with Apache License 2.0 5 votes vote down vote up
package org.apache.spark.deploy.yarn

import java.net.URI

import scala.collection.mutable.{HashMap, Map}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.hadoop.yarn.api.records.{LocalResource, LocalResourceType, LocalResourceVisibility}
import org.apache.hadoop.yarn.util.ConverterUtils
import org.apache.spark.{KyuubiSparkUtil, SparkFunSuite}
import org.mockito.Mockito.when
import org.scalatest.mock.MockitoSugar

import yaooqinn.kyuubi.utils.ReflectUtils

class KyuubiDistributedCacheManagerSuite extends SparkFunSuite with MockitoSugar {

  class MockClientDistributedCacheManager extends ClientDistributedCacheManager {
    override def getVisibility(conf: Configuration, uri: URI, statCache: Map[URI, FileStatus]):
    LocalResourceVisibility = {
      LocalResourceVisibility.PRIVATE
    }
  }

  test("add resource") {
    val fs = mock[FileSystem]
    val conf = new Configuration()
    val destPath = new Path("file:///foo.bar.com:8080/tmp/testing")
    val localResources = HashMap[String, LocalResource]()
    val statCache = HashMap[URI, FileStatus]()
    val status = new FileStatus()
    when(fs.getFileStatus(destPath)).thenReturn(status)
    val fileLink = "link"
    ReflectUtils.setFieldValue(
      KyuubiDistributedCacheManager, "cacheManager", new MockClientDistributedCacheManager)
    KyuubiDistributedCacheManager.addResource(
      fs, conf, destPath, localResources, LocalResourceType.FILE, fileLink, statCache)
    val res = localResources(fileLink)
    assert(res.getVisibility === LocalResourceVisibility.PRIVATE)
    assert(ConverterUtils.getPathFromYarnURL(res.getResource) === destPath)
    assert(res.getSize === 0)
    assert(res.getTimestamp === 0)
    assert(res.getType === LocalResourceType.FILE)
    val status2 = new FileStatus(
      10, false, 1, 1024, 10,
      10, null, KyuubiSparkUtil.getCurrentUserName, null, new Path("/tmp/testing2"))
    val destPath2 = new Path("file:///foo.bar.com:8080/tmp/testing2")
    when(fs.getFileStatus(destPath2)).thenReturn(status2)
    val fileLink2 = "link2"
    KyuubiDistributedCacheManager.addResource(
      fs, conf, destPath2, localResources, LocalResourceType.FILE, fileLink2, statCache)
    val res2 = localResources(fileLink2)
    assert(res2.getVisibility === LocalResourceVisibility.PRIVATE)
    assert(ConverterUtils.getPathFromYarnURL(res2.getResource) === destPath2)
    assert(res2.getSize === 10)
    assert(res2.getTimestamp === 10)
    assert(res2.getType === LocalResourceType.FILE)
  }

  test("add resource when link null") {
    val distMgr = new MockClientDistributedCacheManager()
    val fs = mock[FileSystem]
    val conf = new Configuration()
    val destPath = new Path("file:///foo.bar.com:8080/tmp/testing")
    ReflectUtils.setFieldValue(KyuubiDistributedCacheManager, "cacheManager", distMgr)
    val localResources = HashMap[String, LocalResource]()
    val statCache = HashMap[URI, FileStatus]()
    when(fs.getFileStatus(destPath)).thenReturn(new FileStatus())
    intercept[Exception] {
      KyuubiDistributedCacheManager.addResource(
        fs, conf, destPath, localResources, LocalResourceType.FILE, null, statCache)
    }
    assert(localResources.get("link") === None)
    assert(localResources.size === 0)
  }

  test("test addResource archive") {
    val distMgr = new MockClientDistributedCacheManager()
    ReflectUtils.setFieldValue(KyuubiDistributedCacheManager, "cacheManager", distMgr)
    val fs = mock[FileSystem]
    val conf = new Configuration()
    val destPath = new Path("file:///foo.bar.com:8080/tmp/testing")
    val localResources = HashMap[String, LocalResource]()
    val statCache = HashMap[URI, FileStatus]()
    val realFileStatus = new FileStatus(10, false, 1, 1024, 10, 10, null, "testOwner",
      null, new Path("/tmp/testing"))
    when(fs.getFileStatus(destPath)).thenReturn(realFileStatus)

    KyuubiDistributedCacheManager.addResource(
      fs, conf, destPath, localResources, LocalResourceType.ARCHIVE, "link", statCache)
    val resource = localResources("link")
    assert(resource.getVisibility === LocalResourceVisibility.PRIVATE)
    assert(ConverterUtils.getPathFromYarnURL(resource.getResource) === destPath)
    assert(resource.getTimestamp === 10)
    assert(resource.getSize === 10)
    assert(resource.getType === LocalResourceType.ARCHIVE)

  }

} 
Example 5
Source File: SparkSQLOperationManager.scala    From spark1.52   with Apache License 2.0 5 votes vote down vote up
package org.apache.spark.sql.hive.thriftserver.server

import java.util.{Map => JMap}
import scala.collection.mutable.Map

import org.apache.hive.service.cli._
import org.apache.hive.service.cli.operation.{ExecuteStatementOperation, Operation, OperationManager}
import org.apache.hive.service.cli.session.HiveSession
import org.apache.spark.Logging
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.hive.thriftserver.{SparkExecuteStatementOperation, ReflectionUtils}


private[thriftserver] class SparkSQLOperationManager(hiveContext: HiveContext)
  extends OperationManager with Logging {

  val handleToOperation = ReflectionUtils
    .getSuperField[JMap[OperationHandle, Operation]](this, "handleToOperation")

  val sessionToActivePool = Map[SessionHandle, String]()

  override def newExecuteStatementOperation(
      parentSession: HiveSession,
      statement: String,
      confOverlay: JMap[String, String],
      async: Boolean): ExecuteStatementOperation = synchronized {

    val runInBackground = async && hiveContext.hiveThriftServerAsync
    val operation = new SparkExecuteStatementOperation(parentSession, statement, confOverlay,
      runInBackground)(hiveContext, sessionToActivePool)
    handleToOperation.put(operation.getHandle, operation)
    logDebug(s"Created Operation for $statement with session=$parentSession, " +
      s"runInBackground=$runInBackground")
    operation
  }
} 
Example 6
Source File: TensorFlowWeightCollection.scala    From SparkNet   with MIT License 5 votes vote down vote up
package libs

import scala.collection.mutable.Map
import scala.collection.mutable.MutableList

object TensorFlowWeightCollection {
	def scalarDivide(weights: Map[String, NDArray], v: Float) = {
		for (name <- weights.keys) {
      weights(name).scalarDivide(v)
    }
	}

  def add(wc1: Map[String, NDArray], wc2: Map[String, NDArray]): Map[String, NDArray] = {
    assert(wc1.keys == wc2.keys)
    // add the WeightCollection objects together
    var newWeights = Map[String, NDArray]()
    for (name <- wc1.keys) {
      newWeights += (name -> NDArray.plus(wc1(name), wc2(name)))
    }
    newWeights
  }

  def checkEqual(wc1: Map[String, NDArray], wc2: Map[String, NDArray], tol: Float): Boolean = {
    if (wc1.keys != wc2.keys) {
      return false
    }
    for (name <- wc1.keys) {
      if (!NDArray.checkEqual(wc1(name), wc2(name), tol)) {
        return false
      }
    }
    return true
  }
} 
Example 7
Source File: CaffeWeightCollection.scala    From SparkNet   with MIT License 5 votes vote down vote up
package libs

import scala.collection.mutable.Map
import scala.collection.mutable.MutableList

object CaffeWeightCollection {
  def scalarDivide(weights: Map[String, MutableList[NDArray]], v: Float) = {
    for (name <- weights.keys) {
      for (j <- 0 to weights(name).length - 1) {
        weights(name)(j).scalarDivide(v)
      }
    }
  }

  def add(weights1: Map[String, MutableList[NDArray]], weights2: Map[String, MutableList[NDArray]]): Map[String, MutableList[NDArray]] = {
    if (weights1.keys != weights2.keys) {
      throw new Exception("weights1.keys != weights2.keys, weights1.keys = " + weights1.keys.toString + ", and weights2.keys = " + weights2.keys.toString + "\n")
    }
    val newWeights = Map[String, MutableList[NDArray]]()
    for (name <- weights1.keys) {
      newWeights += (name -> MutableList())
      if (weights1(name).length != weights2(name).length) {
        throw new Exception("weights1(name).length != weights2(name).length, name = " + name + ", weights1(name).length = " + weights1(name).length.toString + ", weights2(name).length = " + weights2(name).length.toString)
      }
      for (j <- 0 to weights1(name).length - 1) {
        if (weights1(name)(j).shape.deep != weights2(name)(j).shape.deep) {
          throw new Exception("weights1(name)(j).shape != weights2(name)(j).shape, name = " + name + ", j = " + j.toString + ", weights1(name)(j).shape = " + weights1(name)(j).shape.deep.toString + ", weights2(name)(j).shape = " + weights2(name)(j).shape.deep.toString)
        }
        newWeights(name) += NDArray.plus(weights1(name)(j), weights2(name)(j))
      }
    }
    newWeights
  }

  def checkEqual(weights1: Map[String, MutableList[NDArray]], weights2: Map[String, MutableList[NDArray]], tol: Float): Boolean = {
    if (weights1.keys != weights2.keys) {
      return false
    }
    for (name <- weights1.keys) {
      if (weights1(name).length != weights2(name).length) {
        return false
      }
      for (j <- 0 to weights1(name).length - 1) {
        if (!NDArray.checkEqual(weights1(name)(j), weights2(name)(j), tol)) {
          return false
        }
      }
    }
    return true
  }

} 
Example 8
Source File: Process.scala    From Graphviz4S   with MIT License 5 votes vote down vote up
package com.liangdp.graphviz4s.examples

import com.liangdp.graphviz4s.Graph
import scala.collection.mutable.Map


object Process {
  def main(args: Array[String]): Unit = {
    val g = new Graph("G")

    g.edge("run", "intr")
    g.edge("intr", "runbl")
    g.edge("runbl", "run")
    g.edge("run", "kernel")
    g.edge("kernel", "zombie")
    g.edge("kernel", "sleep")
    g.edge("kernel", "runmem")
    g.edge("sleep", "swap")
    g.edge("swap", "runswap")
    g.edge("runswap", "new")
    g.edge("runswap", "runmem")
    g.edge("new", "runmem")
    g.edge("sleep", "runmem")

    g.view(engine = "sfdp", fileName = "process.gv", directory = ".")
  }
} 
Example 9
Source File: Fsm.scala    From Graphviz4S   with MIT License 5 votes vote down vote up
package com.liangdp.graphviz4s.examples

import com.liangdp.graphviz4s.Digraph
import scala.collection.mutable.Map


object Fsm {
  def main(args: Array[String]): Unit = {
    val f = new Digraph("finite_state_machine")
    f.body += "rankdir=LR"
    f.body += """size="8,5""""

    f.attr("node", Map("shape" -> "doublecircle"))
    f.node("LR_0")
    f.node("LR_3")
    f.node("LR_4")
    f.node("LR_8")

    f.attr("node", Map("shape" -> "circle"))
    f.edge("LR_0", "LR_2", label = "SS(B)")
    f.edge("LR_0", "LR_1", label = "SS(S)")
    f.edge("LR_1", "LR_3", label = "S($end)")
    f.edge("LR_2", "LR_6", label = "SS(b)")
    f.edge("LR_2", "LR_5", label = "SS(a)")
    f.edge("LR_2", "LR_4", label = "S(A)")
    f.edge("LR_5", "LR_7", label = "S(b)")
    f.edge("LR_5", "LR_5", label = "S(a)")
    f.edge("LR_6", "LR_6", label = "S(b)")
    f.edge("LR_6", "LR_5", label = "S(a)")
    f.edge("LR_7", "LR_8", label = "S(b)")
    f.edge("LR_7", "LR_5", label = "S(a)")
    f.edge("LR_8", "LR_6", label = "S(b)")
    f.edge("LR_8", "LR_5", label = "S(a)")

    f.view(fileName = "fsm.gv", directory = ".")
  }
} 
Example 10
Source File: Cluster.scala    From Graphviz4S   with MIT License 5 votes vote down vote up
package com.liangdp.graphviz4s.examples

import com.liangdp.graphviz4s.Digraph
import scala.collection.mutable.Map


object Cluster {
  def main(args: Array[String]): Unit = {
    val g = new Digraph("G")

    val c0 = new Digraph("cluster_0")
    c0.body += "style=filled"
    c0.body += "color=lightgrey"
    c0.nodeAttr("style") = "filled"
    c0.nodeAttr("color") = "white"
    c0.edges(Array(("a0", "a1"), ("a1", "a2"), ("a2", "a3")))
    c0.body += """label = "process #1""""

    val c1 = new Digraph("cluster_1")
    c1.nodeAttr("style") = "filled"
    c1.edges(Array(("b0", "b1"), ("b1", "b2"), ("b2", "b3")))
    c1.body += """label = "process #2""""
    c1.body += "color=blue"

    g.subGraph(c0)
    g.subGraph(c1)

    g.edge("start", "a0")
    g.edge("start", "b0")
    g.edge("a1", "b3")
    g.edge("b2", "a3")
    g.edge("a3", "a0")
    g.edge("a3", "end")
    g.edge("b3", "end")

    g.node("start", attrs = Map("shape" -> "Mdiamond"))
    g.node("end", attrs = Map("shape" -> "Msquare"))

    g.view(fileName = "cluster.gv", directory = ".")
  }
} 
Example 11
Source File: Er.scala    From Graphviz4S   with MIT License 5 votes vote down vote up
package com.liangdp.graphviz4s.examples

import com.liangdp.graphviz4s.Graph
import scala.collection.mutable.Map


object Er {
  def main(args: Array[String]): Unit = {
    val e = new Graph("ER")

    e.attr("node", Map("shape" -> "box"))
    e.node("course")
    e.node("institute")
    e.node("student")

    e.attr("node", Map("shape" -> "ellipse"))
    e.node("name0", attrs = Map("label" -> "name"))
    e.node("name1", attrs = Map("label" -> "name"))
    e.node("name2", attrs = Map("label" -> "name"))
    e.node("code")
    e.node("grade")
    e.node("number")

    e.attr("node", attrs = Map("shape" -> "diamond",
        "style" -> "filled", "color" -> "lightgrey"))
    e.node("C-I")
    e.node("S-C")
    e.node("S-I")

    e.edge("name0", "course")
    e.edge("code", "course")
    e.edge("course", """"C-I"""", label = "n", attrs = Map("len" -> "1.00"))
    e.edge(""""C-I"""", "institute", label = "1", attrs = Map("len" -> "1.00"))
    e.edge("institute", "name1")
    e.edge("institute", """"S-I"""", label = "1", attrs = Map("len" -> "1.00"))
    e.edge(""""S-I"""", "student", label = "n", attrs = Map("len" -> "1.00"))
    e.edge("student", "grade")
    e.edge("student", "name2")
    e.edge("student", "number")
    e.edge("student", """"S-C"""", label = "m", attrs = Map("len" -> "1.00"))
    e.edge(""""S-C"""", "course", label = "n", attrs = Map("len" -> "1.00"))

    e.body += """label = "\n\nEntity Relation Diagram\ndrawn by NEATO""""
    e.body += "fontsize=20"

    e.view(engine = "neato", fileName = "er.gv", directory = ".")
  }
} 
Example 12
Source File: Unix.scala    From Graphviz4S   with MIT License 5 votes vote down vote up
package com.liangdp.graphviz4s.examples

import com.liangdp.graphviz4s.Digraph
import scala.collection.mutable.Map


object Unix {
  def main(args: Array[String]): Unit = {
    val u = new Digraph("unix")
    u.body += """size="6,6""""
    u.nodeAttr("color") = "lightblue2"
    u.nodeAttr("style") = "filled"

    u.edge(""""5th Edition"""", """"6th Edition"""")
    u.edge(""""5th Edition"""", """"PWB 1.0"""")
    u.edge(""""6th Edition"""", """"LSX"""")
    u.edge(""""6th Edition"""", """"1 BSD"""")
    u.edge(""""6th Edition"""", """"Mini Unix"""")
    u.edge(""""6th Edition"""", """"Wollongong"""")
    u.edge(""""6th Edition"""", """"Interdata"""")
    u.edge(""""Interdata"""", """"Unix/TS 3.0"""")
    u.edge(""""Interdata"""", """"PWB 2.0"""")
    u.edge(""""Interdata"""", """"7th Edition"""")
    u.edge(""""7th Edition"""", """"8th Edition"""")
    u.edge(""""7th Edition"""", """"32V"""")
    u.edge(""""7th Edition"""", """"V7M"""")
    u.edge(""""7th Edition"""", """"Ultrix-11"""")
    u.edge(""""7th Edition"""", """"Xenix"""")
    u.edge(""""7th Edition"""", """"UniPlus+"""")
    u.edge(""""V7M"""", """"Ultrix-11"""")
    u.edge(""""8th Edition"""", """"9th Edition"""")
    u.edge(""""1 BSD"""", """"2 BSD"""")
    u.edge(""""2 BSD"""", """"2.8 BSD"""")
    u.edge(""""2.8 BSD"""", """"Ultrix-11"""")
    u.edge(""""2.8 BSD"""", """"2.9 BSD"""")
    u.edge(""""32V"""", """"3 BSD"""")
    u.edge(""""3 BSD"""", """"4 BSD"""")
    u.edge(""""4 BSD"""", """"4.1 BSD"""")
    u.edge(""""4.1 BSD"""", """"4.2 BSD"""")
    u.edge(""""4.1 BSD"""", """"2.8 BSD"""")
    u.edge(""""4.1 BSD"""", """"8th Edition"""")
    u.edge(""""4.2 BSD"""", """"4.3 BSD"""")
    u.edge(""""4.2 BSD"""", """"Ultrix-32"""")
    u.edge(""""PWB 1.0"""", """"PWB 1.2"""")
    u.edge(""""PWB 1.0"""", """"USG 1.0"""")
    u.edge(""""PWB 1.2"""", """"PWB 2.0"""")
    u.edge(""""USG 1.0"""", """"CB Unix 1"""")
    u.edge(""""USG 1.0"""", """"USG 2.0"""")
    u.edge(""""CB Unix 1"""", """"CB Unix 2"""")
    u.edge(""""CB Unix 2"""", """"CB Unix 3"""")
    u.edge(""""CB Unix 3"""", """"Unix/TS++"""")
    u.edge(""""CB Unix 3"""", """"PDP-11 Sys V"""")
    u.edge(""""USG 2.0"""", """"USG 3.0"""")
    u.edge(""""USG 3.0"""", """"Unix/TS 3.0"""")
    u.edge(""""PWB 2.0"""", """"Unix/TS 3.0"""")
    u.edge(""""Unix/TS 1.0"""", """"Unix/TS 3.0"""")
    u.edge(""""Unix/TS 3.0"""", """"TS 4.0"""")
    u.edge(""""Unix/TS++"""", """"TS 4.0"""")
    u.edge(""""CB Unix 3"""", """"TS 4.0"""")
    u.edge(""""TS 4.0"""", """"System V.0"""")
    u.edge(""""System V.0"""", """"System V.2"""")
    u.edge(""""System V.2"""", """"System V.3"""")

    u.view(fileName = "unix.gv", directory = ".")
  }
} 
Example 13
Source File: TaskResult.scala    From iolap   with Apache License 2.0 5 votes vote down vote up
package org.apache.spark.scheduler

import java.io._
import java.nio.ByteBuffer

import scala.collection.mutable.Map

import org.apache.spark.SparkEnv
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.storage.BlockId
import org.apache.spark.util.Utils

// Task result. Also contains updates to accumulator variables.
private[spark] sealed trait TaskResult[T]


  def value(): T = {
    if (valueObjectDeserialized) {
      valueObject
    } else {
      // This should not run when holding a lock because it may cost dozens of seconds for a large
      // value.
      val resultSer = SparkEnv.get.serializer.newInstance()
      valueObject = resultSer.deserialize(valueBytes)
      valueObjectDeserialized = true
      valueObject
    }
  }
} 
Example 14
Source File: SparkSQLOperationManager.scala    From BigDatalog   with Apache License 2.0 5 votes vote down vote up
package org.apache.spark.sql.hive.thriftserver.server

import java.util.{Map => JMap}
import scala.collection.mutable.Map

import org.apache.hive.service.cli._
import org.apache.hive.service.cli.operation.{ExecuteStatementOperation, Operation, OperationManager}
import org.apache.hive.service.cli.session.HiveSession
import org.apache.spark.Logging
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.hive.thriftserver.{SparkExecuteStatementOperation, ReflectionUtils}


private[thriftserver] class SparkSQLOperationManager()
  extends OperationManager with Logging {

  val handleToOperation = ReflectionUtils
    .getSuperField[JMap[OperationHandle, Operation]](this, "handleToOperation")

  val sessionToActivePool = Map[SessionHandle, String]()
  val sessionToContexts = Map[SessionHandle, HiveContext]()

  override def newExecuteStatementOperation(
      parentSession: HiveSession,
      statement: String,
      confOverlay: JMap[String, String],
      async: Boolean): ExecuteStatementOperation = synchronized {
    val hiveContext = sessionToContexts(parentSession.getSessionHandle)
    val runInBackground = async && hiveContext.hiveThriftServerAsync
    val operation = new SparkExecuteStatementOperation(parentSession, statement, confOverlay,
      runInBackground)(hiveContext, sessionToActivePool)
    handleToOperation.put(operation.getHandle, operation)
    logDebug(s"Created Operation for $statement with session=$parentSession, " +
      s"runInBackground=$runInBackground")
    operation
  }
} 
Example 15
Source File: Metadata.scala    From marvin-engine-executor   with Apache License 2.0 5 votes vote down vote up
package org.marvin.model

import scala.collection.mutable.Map

case class EngineMetadata(name:String,
                          version:String,
                          engineType:String,
                          actions:List[EngineActionMetadata],
                          artifactsRemotePath:String,
                          artifactManagerType:String,
                          s3BucketName:String,
                          pipelineActions: List[String],
                          onlineActionTimeout:Double,
                          metricsTimeout:Double=10000,
                          healthCheckTimeout:Double,
                          reloadTimeout:Double,
                          reloadStateTimeout: Option[Double],
                          batchActionTimeout:Double,
                          hdfsHost:String){
  
  override def toString: String = name

  val artifactsLocalPath: String = sys.env.getOrElse("MARVIN_DATA_PATH", "/tmp").mkString.concat( "/.artifacts")

  val pipelineTimeout: Double = (reloadTimeout + batchActionTimeout) * pipelineActions.length * 1.20

  val actionsMap: Map[String, EngineActionMetadata] = {
    val map = Map[String, EngineActionMetadata]()
    if (actions != null) {
      for (action <- actions) {
        map += ((action.name) -> action)
      }
    }
    map
  }
}

sealed abstract class ActionTypes(val name:String) {
  override def toString: String = name
}

case object BatchType extends ActionTypes(name="batch")
case object OnlineType extends ActionTypes(name="online")

case class EngineActionMetadata(name:String, actionType:String, port:Int, host:String, artifactsToPersist:List[String], artifactsToLoad:List[String]){
  override def toString: String = name
} 
Example 16
Source File: CEPEngine.scala    From piglet   with Apache License 2.0 5 votes vote down vote up
package dbis.piglet.cep.engines

import dbis.piglet.backends.{SchemaClass => Event}
import scala.reflect.ClassTag
import scala.collection.mutable.ListBuffer
import dbis.piglet.cep.nfa.NFAStructure
import dbis.piglet.cep.nfa.NFAController
import dbis.piglet.cep.nfa.NormalState
import scala.collection.mutable.Map
import dbis.piglet.cep.ops.MatchCollector

abstract class CEPEngine[T <: Event: ClassTag](nfaController: NFAController[T], collector: MatchCollector[T]) extends Serializable {
   val structureID = { var sid: Long = 0; () => { sid += 1; sid } }
   var runningStructursPool: Map[Long, NFAStructure[T]] = Map()
   var wantToDeletedStructurs: ListBuffer[Long] = new ListBuffer()
  def createNewStructue(event: T): Unit = {
    val start = nfaController.getStartState
    start.edges.foreach { e =>
      if (e.evaluate(event)) {
        val newStr = new NFAStructure[T](nfaController)
        newStr.addEvent(event, e)
        runningStructursPool += (structureID() -> newStr)
      }
    }
  }
  def runGCStructures(): Unit = {
    if(runningStructursPool.size > 0) {
      runningStructursPool --=  wantToDeletedStructurs
    //wantToDeletedStructurs.foreach { x =>  runningStructursPool -= x  }
      wantToDeletedStructurs.clear()
    }
  }

  def checkPredicate(event: T, currenStr: NFAStructure[T]): Int = {
    var result: Int = -1
    if (currenStr.getCurrentState.isInstanceOf[NormalState[T]]) {
      val currentState = currenStr.getCurrentState.asInstanceOf[NormalState[T]]
      currentState.edges.zipWithIndex.foreach {
        case (e, i) =>
          if (e.evaluate(event, currenStr)) {
            result = i
          }
      }
    }
    result
  }
  def runEngine(event: T): Unit
  //def printNumMatches(): Unit
} 
Example 17
Source File: SpatialFilter.scala    From piglet   with Apache License 2.0 5 votes vote down vote up
package dbis.piglet.op

import scala.collection.mutable.Map
import dbis.piglet.expr.Predicate
import dbis.piglet.expr.Ref
import dbis.piglet.expr.Expr
import dbis.piglet.expr.SpatialFilterPredicate
import dbis.piglet.op.IndexMethod.IndexMethod

import scala.collection.mutable


  override def lineageString: String = {
    s"""SPATIALFILTER%$pred%$idx""" + super.lineageString
  }

  override def resolveReferences(mapping: mutable.Map[String, Ref]): Unit = pred.resolveReferences(mapping)

  override def checkSchemaConformance: Boolean = {
    schema match {
      case Some(s) =>
        // if we know the schema we check all named fields
        pred.traverseAnd(s, Expr.checkExpressionConformance)
      case None =>
        // if we don't have a schema all expressions should contain only positional fields
        pred.traverseAnd(null, Expr.containsNoNamedFields)
    }
  }


  override def toString =
    s"""SPATIALFILTER
       |  out = $outPipeName
       |  in = $inPipeName
       |  schema = $schema
       |  expr = $pred
       |  idx = $idx""".stripMargin


} 
Example 18
Source File: DoubleMill.scala    From Scalaprof   with GNU General Public License v2.0 5 votes vote down vote up
package models

import scala.collection.mutable.{Stack,Map}
import scala.util._
//

object DoubleMill {
  val conv: String=>Try[Double] = DoubleMill.valueOf _
  val lookup: String=>Option[Double] = DoubleMill.constants.get _
  implicit val store = Map[String,Double]()
  implicit val parser = new ExpressionParser[Double](conv,lookup)
  def apply(): Mill[Double] = new MillNumeric(Stack[Double]()) {
    def apply(s: String): Try[Double] = DoubleMill.valueOf(s)    
  }
  def valueOf(s: String): Try[Double] = Try(s.toDouble)
  val constants = Map("e"->math.E, "pi"->math.Pi)
} 
Example 19
Source File: SpireMill.scala    From Scalaprof   with GNU General Public License v2.0 5 votes vote down vote up
package models

import scala.collection.mutable.{Stack,Map}
import scala.util._
import spire.math._
import spire.implicits._


object SpireMill {

  val conv: String=>Try[Real] = SpireMill.valueOf _
  val lookup: String=>Option[Real] = SpireMill.constants.get _
  implicit val store = Map[String,Real]()
  implicit val parser = new ExpressionParser[Real](conv,lookup)
  def apply(): Mill[Real] = new MillSpire(Stack[Real]()) {
    def apply(s: String): Try[Real] = SpireMill.valueOf(s)    
  }
  def valueOf(s: String): Try[Real] = Try(Real(s))
 val constants = Map("e"->Real.e, "pi"->Real.pi)
} 
Example 20
Source File: MillNumeric.scala    From Scalaprof   with GNU General Public License v2.0 5 votes vote down vote up
package models

import scala.collection.mutable.{Stack,Map}
import scala.util._


abstract class MillNumeric[A : Numeric](stack: Stack[A])(implicit store: Map[String,A]) extends Mill[A](stack)(store) { self =>

  def operate(s: String): Unit = s match {
    case "+" => operate("plus")
    case "plus" => dyadic(implicitly[Numeric[A]].plus)
    case "-" => operate("chs"); operate("plus")
    case "chs" => monoadic(implicitly[Numeric[A]].negate)
    case "*" => operate("times")
    case "times" => dyadic(implicitly[Numeric[A]].times)
    case "div" => operate("/")
    case "/" => operate("inv"); operate("times")
    case "inv" => val i = implicitly[Numeric[A]]; if (i.isInstanceOf[Fractional[A]]) monoadic2(i.asInstanceOf[Fractional[A]].div _)(i.one)
    case "swap" => has(2); val (top,next) = (pop,pop); push(top); push(next)
    case "del" => has(1); pop
    case "clr" => stack.clear
    case x => throw new IllegalArgumentException(s"operator $x is not supported")
  }
  
} 
Example 21
Source File: RationalMill.scala    From Scalaprof   with GNU General Public License v2.0 5 votes vote down vote up
package models

import scala.collection.mutable.{Stack,Map}
import scala.util._
import edu.neu.coe.scala.numerics.Rational


object RationalMill {

  val conv: String=>Try[Rational] = RationalMill.valueOf _
  val lookup: String=>Option[Rational] = RationalMill.constants.get _
  implicit val store = Map[String,Rational]()
  implicit val parser = new ExpressionParser[Rational](conv,lookup)
  def apply(): Mill[Rational] = new MillNumeric(Stack[Rational]()) {
    def apply(s: String): Try[Rational] = RationalMill.valueOf(s)    
  }
  def valueOf(s: String): Try[Rational] = Try(Rational(s))
 val constants = Map("e"->Rational(BigDecimal(math.E)), "pi"->Rational(BigDecimal(math.Pi)))
} 
Example 22
Source File: Mill.scala    From Scalaprof   with GNU General Public License v2.0 5 votes vote down vote up
package models

import scala.collection.mutable.{Stack,Map}
import scala.util._


abstract class Mill[A](stack: Stack[A])(implicit store: Map[String,A]) extends Function1[Valuable[A],Try[A]] { self =>
  
  var debugMill = false;
  def value = if (stack.size>0) Success(stack.top) else Failure(new IllegalArgumentException("stack is empty"))
  def toSeq = stack.toSeq
  def show = println(stack)
  def push(x: A) = { if (debugMill) println(s"push $x");stack.push(x)}
  def pop = {val x = stack.pop; if (debugMill) println(s"popped $x"); x}
  def setDebug(b: Boolean) { debugMill = b }
  def has(n: Int) = assert(stack.size>=n,s"operation requires $n element(s) on stack")
 
  def apply(v: Valuable[A]) = v match {
    case n @ Number(x) => n.apply match {case Success(x) => push(x); case Failure(e) => throw e}; value
    case k @ Constant(x) => k.apply match {case Success(x) => push(x); case Failure(e) => throw e}; value
    case Operator(s) => operate(s); value
    case MemInst(s,n) => memInst(s,n); value
  }
  
  def dyadic(f: (A,A)=>A) = { has(2); push(f(pop,pop)) }
  def monoadic(f: (A)=>A) = { has(1); push(f(pop)) }
  def monoadic2(f: (A,A)=>A)(a: A) = { has(1); push(f(a,pop)) }
  
  def operate(s: String): Unit
  
  def memInst(s: String, k: String) = s.toLowerCase match {
    case "sto" => value match {case Success(x) => store.put(k,x); case Failure(e) => throw e}
    case "rcl" => store.get(k) match {case Some(x) => push(x); case None => throw new IllegalArgumentException(s"no value at memory location $k")}
  }
  
  def parse(s: String)(implicit parser: ExpressionParser[A]): Try[A] = 
    parser.parseAll(parser.expr,s) match {
      case parser.Success(ws,_) => try {
          (for (w <- ws) yield apply(w)).reverse.head
        } catch {
        case t: Throwable => Failure(t)
      }
      case parser.Failure(e,_) => Failure(new IllegalArgumentException(s"parseResult error: $e"))
      case r @ _ => Failure(new IllegalArgumentException(s"logic error: parseResult is $r"))
    }
} 
Example 23
Source File: MillSpire.scala    From Scalaprof   with GNU General Public License v2.0 5 votes vote down vote up
package models

import scala.collection.mutable.{Stack,Map}
import scala.util._
import spire.math._
import spire.implicits._


abstract class MillSpire[A : Numeric](stack: Stack[A])(implicit store: Map[String,A]) extends Mill[A](stack)(store) { self =>
  
  def operate(s: String): Unit = s match {
    case "+" => operate("plus")
    case "plus" => dyadic(implicitly[Numeric[A]].plus _)
    case "-" => operate("chs"); operate("plus")
    case "chs" => monoadic(implicitly[Numeric[A]].negate)
    case "*" => operate("times")
    case "times" => dyadic(implicitly[Numeric[A]].times)
    case "div" => operate("/")
    case "/" => operate("inv"); operate("times")
    case "inv" => val i = implicitly[Numeric[A]]; if (i.isInstanceOf[Fractional[A]]) monoadic2(i.asInstanceOf[Fractional[A]].div _)(i.one)
    case "swap" => has(2); val (top,next) = (pop,pop); push(top); push(next)
    case "del" => has(1); pop
    case "clr" => stack.clear
    case x => throw new IllegalArgumentException(s"operator $x is not supported")
  }
  
} 
Example 24
Source File: plot.scala    From Scientific-Computing-with-Scala   with MIT License 5 votes vote down vote up
import scala.collection.mutable.{MutableList, Map}
import scala.math._
import org.jfree.chart._
import org.jfree.data.xy._
import org.jfree.data.statistics._
import java.io.{FileReader, BufferedReader}
import java.awt.GridLayout
import javax.swing.JFrame
import javax.swing.JPanel

object ScatterPlotMatrix {
  def readCSVFile(filename: String): Map[String, MutableList[String]] = {
    val file = new FileReader(filename)
    val reader = new BufferedReader(file)
    val csvdata: Map[String, MutableList[String]] = Map()
    try {
      val alldata = new MutableList[Array[String]]
      var line:String = null
      while ({line = reader.readLine(); line} != null) {
        if (line.length != 0) {
          val delimiter: String = ","
          var splitline: Array[String] = line.split(delimiter).map(_.trim)
          alldata += splitline
        }
      }
      val labels = MutableList("sepal length", "sepal width",
        "petal length", "petal width", "class")
      val labelled = labels.zipWithIndex.map {
        case (label, index) => label -> alldata.map(x => x(index))
      }
      for (pair <- labelled) {
        csvdata += pair
      }
    } finally {
      reader.close()
    }
    csvdata
  }

  def main(args: Array[String]) {
    val data = readCSVFile("iris.csv")
    val frame = new JFrame("Scatter Plot Matrix")
    frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE)
    frame.setLayout(new GridLayout(4, 4))
    val attributes = List("sepal length", "sepal width", 
      "petal length", "petal width")
    val classes = List("Iris-setosa", "Iris-versicolor", "Iris-virginica")
    for ((a1, i) <- attributes.zipWithIndex) {
      for ((a2, j) <- attributes.zipWithIndex) {
        if (a1 == a2) {
          val dataset = new HistogramDataset();
          dataset.setType(HistogramType.RELATIVE_FREQUENCY);
          val xs = (for (x <- data(a1)) yield { x.toDouble }).toArray
          dataset.addSeries(a1, xs, 11);
          val chart = ChartFactory.createHistogram(null, a1, "frequency",
            dataset, org.jfree.chart.plot.PlotOrientation.VERTICAL,
            false, false, false)
          frame.add(new ChartPanel(chart, 200, 200, 200, 200, 200, 200,
            false, true, true, true, true, true))
        } else {
          val dataset = new DefaultXYDataset
          for (cls <- classes) {
            val xs = (for ((x, index) <- data(a1).zipWithIndex
              if data("class")(index) == cls)
            yield { x.toDouble }).toArray
            val ys = (for ((y, index) <- data(a2).zipWithIndex
              if data("class")(index) == cls)
            yield { y.toDouble }).toArray
            dataset.addSeries(cls, Array(xs, ys))
          }
          val chart = ChartFactory.createScatterPlot(null, 
            a1, a2, dataset, org.jfree.chart.plot.PlotOrientation.VERTICAL, 
            false, false, false)
          frame.add(new ChartPanel(chart, 200, 200, 200, 200, 200, 200,
            false, true, true, true, true, true))
        }
      }
    }
    frame.pack()
    frame.setVisible(true)
  }
} 
Example 25
Source File: Main.scala    From ros_hadoop   with Apache License 2.0 5 votes vote down vote up
package de.valtech.foss

import scala.io.Source
import scala.collection.mutable.Map
import scala.collection.mutable.ListBuffer
import scala.collection.JavaConverters._
import Console.{GREEN, RED, RESET}
import scala.language.reflectiveCalls

import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.nio.channels.FileChannel.MapMode._
import java.nio.ByteOrder._
import java.nio.ByteBuffer

import de.valtech.foss.proto.RosbagIdxOuterClass.RosbagIdx

object Main extends App {
  def help() = {
    Console.err.printf(s"""
${RESET}${GREEN}Usage:
	--file <ros.bag> file to process
	--version print Rosbag version and exit
	--offset <offset> --number <records> Seek at offset < 1073741824 and read the specified number of records
${RESET}By default will just create the protobuf idx file needed for configuration.\n\n""")
    sys.exit(0)
  }

  val pargs = Map[String,AnyRef]()
  def process_cli(args: List[String]) :Boolean = args match {
    case Nil => true // parse success
    case "-v" :: rest => pargs += ("version" -> Some(true)); process_cli(rest)
    case "--version" :: rest => pargs += ("version" -> Some(true)); process_cli(rest)
    case "-f" :: x :: rest => pargs += ("file" -> x); process_cli(rest)
    case "--file" :: x :: rest => pargs += ("file" -> x); process_cli(rest)
    case "-n" :: x :: rest => pargs += ("number" -> Some(x.toInt)); process_cli(rest)
    case "--number" :: x :: rest => pargs += ("number" -> Some(x.toInt)); process_cli(rest)
    case "-o" :: x :: rest => pargs += ("offset" -> Some(x.toInt)); process_cli(rest)
    case "--offset" :: x :: rest => pargs += ("offset" -> Some(x.toInt)); process_cli(rest)
    case "-h" :: rest => help(); false
    case "--help" :: rest => help(); false
    case _ => Console.err.printf(s"${RESET}${RED}Unknown argument " + args.head); false
  }
  process_cli(args.toList)

  def use[T <: { def close() }]
    (resource: T)
    (code: T ⇒ Unit) =
    try
      code(resource)
    finally
      resource.close()

  pargs("file") match {
    case f:String => process()
    case _ => help()
  }

  def process(): Unit = {
    val fin = new File(pargs("file").asInstanceOf[String])
    use(new FileInputStream(fin)) { stream => {
      //printf("min: %s\n", Math.min(1073741824, fin.length) )
      val buffer = stream.getChannel.map(READ_ONLY, 0, Math.min(1073741824, fin.length)).order(LITTLE_ENDIAN)
      val p:RosbagParser = new RosbagParser(buffer)
      val version = p.read_version()
      val h = p.read_record().get
      if(pargs contains "version") {
        printf("%s\n%s\n\n", version, h)
        return
      }
      if(pargs contains "number"){
        buffer position pargs.getOrElse("offset",None).asInstanceOf[Option[Int]].getOrElse(0)
        for(i <- List.range(0,pargs("number").asInstanceOf[Option[Int]].getOrElse(0)))
          println(p.read_record)
        return
      }
      val idxpos = h.header.fields("index_pos").asInstanceOf[Long]
      //printf("idxpos: %s %s\n", idxpos, Math.min(1073741824, fin.length) )
      val b = stream.getChannel.map(READ_ONLY, idxpos, Math.min(1073741824, fin.length - idxpos)).order(LITTLE_ENDIAN)
      val pp:RosbagParser = new RosbagParser(b)
      val c = pp.read_connections(h.header, Nil)
      val chunk_idx = pp.read_chunk_infos(c)
      Console.err.printf(s"""${RESET}${GREEN}Found: """
          + chunk_idx.size
          +s""" chunks\n${RESET}It should be the same number reported by rosbag tool.\nIf you encounter any issues try reindexing your file and submit an issue.
          ${RESET}\n""")
      val fout = new FileOutputStream(pargs("file").asInstanceOf[String] + ".idx.bin")
      val builder = RosbagIdx.newBuilder
      for(i <- chunk_idx) builder.addArray(i)
      builder.build().writeTo(fout)
      fout.close()
      //printf("[%s]\n",chunk_idx.toArray.mkString(","))
    }}
  }
} 
Example 26
Source File: PythonBigDLValidator.scala    From BigDL   with Apache License 2.0 5 votes vote down vote up
package com.intel.analytics.bigdl.python.api

import java.lang.{Boolean => JBoolean}
import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap}

import com.intel.analytics.bigdl.tensor.Tensor
import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
import com.intel.analytics.bigdl.utils.Table

import scala.collection.JavaConverters._
import scala.collection.mutable.Map
import scala.language.existentials
import scala.reflect.ClassTag

object PythonBigDLValidator {

  def ofFloat(): PythonBigDLValidator[Float] = new PythonBigDLValidator[Float]()

  def ofDouble(): PythonBigDLValidator[Double] = new PythonBigDLValidator[Double]()
}

class PythonBigDLValidator[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonBigDL[T]{

  def testDict(): JMap[String, String] = {
    return Map("jack" -> "40", "lucy" -> "50").asJava
  }

  def testDictJTensor(): JMap[String, JTensor] = {
    return Map("jack" -> JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float")).asJava
  }

  def testDictJMapJTensor(): JMap[String, JMap[String, JTensor]] = {
    val table = new Table()
    val tensor = JTensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1), "float")
    val result = Map("jack" -> tensor).asJava
    table.insert(tensor)
    return Map("nested" -> result).asJava
  }

  def testActivityWithTensor(): JActivity = {
    val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1))
    return JActivity(tensor)
  }

  def testActivityWithTableOfTensor(): JActivity = {
    val tensor1 = Tensor(Array(1.0f, 1.0f), Array(2))
    val tensor2 = Tensor(Array(2.0f, 2.0f), Array(2))
    val tensor3 = Tensor(Array(3.0f, 3.0f), Array(2))
    val table = new Table()
    table.insert(tensor1)
    table.insert(tensor2)
    table.insert(tensor3)
    return JActivity(table)
  }

  def testActivityWithTableOfTable(): JActivity = {
    val tensor = Tensor(Array(1.0f, 2.0f, 3.0f, 4.0f), Array(4, 1))
    val table = new Table()
    table.insert(tensor)
    val nestedTable = new Table()
    nestedTable.insert(table)
    nestedTable.insert(table)
    return JActivity(nestedTable)
  }
} 
Example 27
Source File: Metrics.scala    From BigDL   with Apache License 2.0 5 votes vote down vote up
package com.intel.analytics.bigdl.optim

import com.google.common.util.concurrent.AtomicDouble
import org.apache.spark.{Accumulable, Accumulator, SparkContext}

import scala.collection.mutable.{ArrayBuffer, Map}


class Metrics extends Serializable {
  private val localMetricsMap: Map[String, LocalMetricsEntry] = Map()
  private val aggregateDistributeMetricsMap: Map[String, AggregateDistributeMetricsEntry] = Map()
  private val distributeMetricsMap: Map[String, DistributeMetricsEntry] = Map()

  def add(name: String, value: Double): this.type = {
    require(localMetricsMap.contains(name) || aggregateDistributeMetricsMap.contains(name) ||
      distributeMetricsMap.contains(name))
    if (localMetricsMap.contains(name)) {
      localMetricsMap(name).value.addAndGet(value)
    }

    if (aggregateDistributeMetricsMap.contains(name)) {
      aggregateDistributeMetricsMap(name).value += value
    }

    if (distributeMetricsMap.contains(name)) {
      distributeMetricsMap(name).value += value
    }
    this
  }

  def set(name: String, value: Double, parallel: Int = 1): this.type = {
    require(!aggregateDistributeMetricsMap.contains(name), "duplicated distribute metric")
    require(!distributeMetricsMap.contains(name), "duplicated distribute metric2")
    if (localMetricsMap.contains(name)) {
      localMetricsMap(name).value.set(value)
      localMetricsMap(name).parallel = parallel
    } else {
      localMetricsMap(name) = LocalMetricsEntry(new AtomicDouble(value), parallel)
    }
    this
  }

  def set(name: String, value: Double, sc: SparkContext, parallel: Int): this.type = {
    require(!localMetricsMap.contains(name), "duplicated local metric")
    if (aggregateDistributeMetricsMap.contains(name)) {
      aggregateDistributeMetricsMap(name).value.setValue(value)
      aggregateDistributeMetricsMap(name).parallel = parallel
    } else {
      aggregateDistributeMetricsMap(name) =
        AggregateDistributeMetricsEntry(sc.accumulator(value, name), parallel)
    }
    this
  }

  def set(name: String, value: ArrayBuffer[Double], sc: SparkContext): this.type = {
    require(!localMetricsMap.contains(name), "duplicated local metric")
    require(!aggregateDistributeMetricsMap.contains(name), "duplicated distribute metric")
    if (distributeMetricsMap.contains(name)) {
      distributeMetricsMap(name).value.setValue(value)
    } else {
      distributeMetricsMap(name) = DistributeMetricsEntry(sc.accumulableCollection(value))
    }
    this
  }

  def get(name: String): (Double, Int) = {
    require(localMetricsMap.contains(name) || aggregateDistributeMetricsMap.contains(name))
    if (localMetricsMap.contains(name)) {
      (localMetricsMap(name).value.get(), localMetricsMap(name).parallel)
    } else {
      (aggregateDistributeMetricsMap(name).value.value,
        aggregateDistributeMetricsMap(name).parallel)
    }
  }

  def get(name: String, number: Int): Array[Double] = {
    require(distributeMetricsMap.contains(name))
    distributeMetricsMap(name).value.value.toArray.dropRight(number)
  }

  def summary(unit: String = "s", scale: Double = 1e9): String = {
    "========== Metrics Summary ==========\n" +
      localMetricsMap.map(
        entry => s"${entry._1} : ${entry._2.value.get() / entry._2.parallel / scale} $unit\n")
        .mkString("") +
      aggregateDistributeMetricsMap.map(
        entry => s"${entry._1} : ${entry._2.value.value / entry._2.parallel / scale} $unit\n")
        .mkString("") +
      distributeMetricsMap.map { entry =>
        s"${entry._1} : ${entry._2.value.value.map(_ / scale).mkString(" ")} \n"
      }.mkString("") +
      "====================================="
  }
}


private case class LocalMetricsEntry(value: AtomicDouble, var parallel: Int)

private case class AggregateDistributeMetricsEntry(value: Accumulator[Double], var parallel: Int)

private case class DistributeMetricsEntry(value: Accumulable[ArrayBuffer[Double], Double]) 
Example 28
Source File: Series.scala    From salt-core   with Apache License 2.0 5 votes vote down vote up
package software.uncharted.salt.core.generation

import software.uncharted.salt.core.analytic.Aggregator
import software.uncharted.salt.core.projection.Projection
import software.uncharted.salt.core.spreading.SpreadingFunction
import software.uncharted.salt.core.generation.request.TileRequest
import software.uncharted.salt.core.generation.output.{SeriesData, Tile}
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import scala.reflect.ClassTag
import scala.collection.mutable.Map


class Series[
  RT,
  DC,
  TC,
  BC,
  @specialized(Int, Long, Double) T,
  @specialized(Int, Long, Double) U,
  @specialized(Int, Long, Double) V,
  W,
  X
](
  val maxBin: BC,
  val cExtractor: (RT) => Option[DC],
  val projection: Projection[DC,TC,BC],
  val vExtractor: (RT) => Option[T],
  val binAggregator: Aggregator[T, U, V],
  val tileAggregator: Option[Aggregator[V, W, X]] = None,
  val spreadingFunction: Option[SpreadingFunction[TC, BC, T]] = None) extends Serializable {

  private[salt] val id: String = java.util.UUID.randomUUID.toString

  def apply(tile: Tile[TC]): Option[SeriesData[TC,BC,V,X]] = {
    tile.seriesData.get(id).asInstanceOf[Option[SeriesData[TC, BC, V, X]]]
  }
} 
Example 29
Source File: JsonMerkleRootsSerializer.scala    From Sidechains-SDK   with MIT License 5 votes vote down vote up
package com.horizen.serialization

import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind
import com.fasterxml.jackson.databind.SerializerProvider
import com.horizen.utils.{ByteArrayWrapper, BytesUtils}

import scala.collection.mutable.Map
import scala.collection.mutable.Iterable

class JsonMerkleRootsSerializer extends databind.JsonSerializer[Option[Map[ByteArrayWrapper, Array[Byte]]]] {

  override def serialize(t: Option[Map[ByteArrayWrapper, Array[Byte]]], jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider): Unit = {
    if(t.isDefined){
      var listOfPair : Iterable[Pair] = t.get.map(k => Pair(BytesUtils.toHexString(k._1.data), BytesUtils.toHexString(k._2)))
      jsonGenerator.writeObject(listOfPair)
    }else{
      jsonGenerator.writeStartArray()
      jsonGenerator.writeEndArray()
    }
  }
}

private case class Pair(key : String, value : String) 
Example 30
Source File: TestBroadcastVariables.scala    From spark-dev   with GNU General Public License v3.0 5 votes vote down vote up
package examples

import org.apache.spark.{ SparkContext, SparkConf }
import org.apache.spark.rdd.RDD
import org.apache.spark.broadcast.Broadcast

import scala.io.Source
import scala.util.{ Try, Success, Failure }
import scala.collection.mutable.Map


	def loadCSVFile(filename: String): Option[Map[String, String]] = {
		val countries = Map[String, String]()

		Try {
			val bufferedSource = Source.fromFile(filename)

			for (line <- bufferedSource.getLines) {
				val Array(country, capital) = line.split(",").map(_.trim)
				countries += country -> capital
			}

			bufferedSource.close()
			return Some(countries)

		}.toOption
	}
} 
Example 31
Source File: LeanMessagingProvider.scala    From openwhisk   with Apache License 2.0 5 votes vote down vote up
package org.apache.openwhisk.connector.lean

import java.util.concurrent.BlockingQueue
import java.util.concurrent.LinkedBlockingQueue

import scala.collection.mutable.Map
import scala.collection.concurrent.TrieMap
import scala.concurrent.duration.FiniteDuration
import scala.util.Success
import scala.util.Try

import akka.actor.ActorSystem
import org.apache.openwhisk.common.Logging
import org.apache.openwhisk.core.WhiskConfig
import org.apache.openwhisk.core.connector.MessageConsumer
import org.apache.openwhisk.core.connector.MessageProducer
import org.apache.openwhisk.core.connector.MessagingProvider
import org.apache.openwhisk.core.entity.ByteSize


  val queues: Map[String, BlockingQueue[Array[Byte]]] =
    new TrieMap[String, BlockingQueue[Array[Byte]]]

  def getConsumer(config: WhiskConfig, groupId: String, topic: String, maxPeek: Int, maxPollInterval: FiniteDuration)(
    implicit logging: Logging,
    actorSystem: ActorSystem): MessageConsumer = {

    val queue = queues.getOrElseUpdate(topic, new LinkedBlockingQueue[Array[Byte]]())

    new LeanConsumer(queue, maxPeek)
  }

  def getProducer(config: WhiskConfig, maxRequestSize: Option[ByteSize] = None)(
    implicit logging: Logging,
    actorSystem: ActorSystem): MessageProducer =
    new LeanProducer(queues)

  def ensureTopic(config: WhiskConfig, topic: String, topicConfigKey: String, maxMessageBytes: Option[ByteSize] = None)(
    implicit logging: Logging): Try[Unit] = {
    if (queues.contains(topic)) {
      Success(logging.info(this, s"topic $topic already existed"))
    } else {
      queues.put(topic, new LinkedBlockingQueue[Array[Byte]]())
      Success(logging.info(this, s"topic $topic created"))
    }
  }
} 
Example 32
Source File: ConcurrentHashMap.scala    From zio   with Apache License 2.0 5 votes vote down vote up
package zio.test

import scala.collection.mutable.Map

private[test] final case class ConcurrentHashMap[K, V] private (private val map: Map[K, V]) {
  def foldLeft[B](z: B)(f: (B, (K, V)) => B): B =
    map.foldLeft(z)(f)
  def getOrElseUpdate(key: K, op: => V): V =
    map.getOrElseUpdate(key, op)
}

private[test] object ConcurrentHashMap {
  def empty[K, V]: ConcurrentHashMap[K, V] =
    new ConcurrentHashMap[K, V](Map.empty[K, V])
} 
Example 33
Source File: KustoAzureFsSetupCache.scala    From azure-kusto-spark   with Apache License 2.0 5 votes vote down vote up
package com.microsoft.kusto.spark.utils
import org.joda.time.{DateTime, DateTimeZone, Period}
import scala.collection.mutable.Map

private[kusto] object KustoAzureFsSetupCache {
  private var storageAccountKeyMap: Map[String, String] = Map.empty[String,String]
  private var storageSasMap: Map[String, String] = Map.empty[String,String]
  private var nativeAzureFsSet = false
  private var lastRefresh: DateTime = new DateTime(DateTimeZone.UTC)

  // Return 'true' iff the entry exists in the cache. If it doesn't, or differs - update the cache
  // now is typically 'new DateTime(DateTimeZone.UTC)'
  def updateAndGetPrevStorageAccountAccess(account: String, secret: String, now: DateTime): Boolean = {
    var secretCached = storageAccountKeyMap.getOrElse(account, "")
    if (!secretCached.isEmpty && (secretCached != secret)) {
      // Entry exists but with a different secret - remove it and update
      storageAccountKeyMap.remove(account)
      secretCached = ""
    }

    if (secretCached.isEmpty || checkIfRefreshNeeded(now)) {
      storageAccountKeyMap.put(account, secret)
      lastRefresh = now
      false
    } else true
  }

  def updateAndGetPrevSas(container: String, account: String, secret: String, now: DateTime): Boolean = {
    val key = container + "." + account
    var secretCached = storageSasMap.getOrElse(key, "")
    if (!secretCached.isEmpty && (secretCached != secret)) {
      // Entry exists but with a different secret - remove it and update
      storageSasMap.remove(key)
      secretCached = ""
    }

    if (secretCached.isEmpty || checkIfRefreshNeeded(now)) {
      storageSasMap.put(key, secret)
      lastRefresh = now
      false
    } else true
  }

  def updateAndGetPrevNativeAzureFs(now: DateTime): Boolean = {
    if (nativeAzureFsSet || checkIfRefreshNeeded(now)) true else {
      nativeAzureFsSet = true
      false
    }
  }

  private[kusto] def checkIfRefreshNeeded(utcNow: DateTime) = {
    new Period(utcNow, lastRefresh).getMinutes > KustoConstants.sparkSettingsRefreshMinutes
  }
} 
Example 34
Source File: CSVReader.scala    From Scientific-Computing-with-Scala   with MIT License 5 votes vote down vote up
import scala.collection.mutable.{MutableList, Map}
import java.io.{FileReader, BufferedReader}

object CSVReader {
 def main(args: Array[String]) {
   val file = new FileReader("iris.csv")
   val reader = new BufferedReader(file)
   try {
     val alldata = new MutableList[Array[String]]
     var line:String = null
     while ({line = reader.readLine(); line} != null) {
       if (line.length != 0) {
         val delimiter: String = ","
         var splitline: Array[String] = line.split(delimiter).map(_.trim)
         alldata += splitline
       }
     }
     val labels = MutableList("sepal length", "sepal width",
       "petal length", "petal width", "class")
     val labelled = labels.zipWithIndex.map {
       case (label, index) => label -> alldata.map(x => x(index))
     }
     val csvdata: Map[String, MutableList[String]] = Map()
     for (pair <- labelled) {
       csvdata += pair                                           
     }
   }
   finally {
     reader.close()
   }
 }
} 
Example 35
Source File: plot.scala    From Scientific-Computing-with-Scala   with MIT License 5 votes vote down vote up
import org.jfree.chart._
import org.jfree.data.xy._
import scala.math._
import scala.collection.mutable.{MutableList, Map}
import java.io.{FileReader, BufferedReader}

object ParallelCoordinates {
  def readCSVFile(filename: String): Map[String, MutableList[String]] = {
    val file = new FileReader(filename)
    val reader = new BufferedReader(file)
    val csvdata: Map[String, MutableList[String]] = Map()
    try {
      val alldata = new MutableList[Array[String]]
      var line:String = null
      while ({line = reader.readLine(); line} != null) {
        if (line.length != 0) {
          val delimiter: String = ","
          var splitline: Array[String] = line.split(delimiter).map(_.trim)
          alldata += splitline
        }
      }
      val labels = MutableList("sepal length", "sepal width",
        "petal length", "petal width", "class")
      val labelled = labels.zipWithIndex.map {
        case (label, index) => label -> alldata.map(x => x(index))
      }
      for (pair <- labelled) {
        csvdata += pair
      }
    } finally {
      reader.close()
    }
    csvdata
  }

  def main(args: Array[String]) {
    val data = readCSVFile("iris.csv")
    val dataset = new DefaultXYDataset
    for (i <- 0 until data("sepal length").size) {
      val x = Array(0.0, 1.0, 2.0, 3.0)
      val y1 = data("sepal length")(i).toDouble
      val y2 = data("sepal width")(i).toDouble
      val y3 = data("petal length")(i).toDouble
      val y4 = data("petal width")(i).toDouble
      val y = Array(y1, y2, y3, y4)
      val cls = data("class")(i)
      dataset.addSeries(cls + i, Array(x, y))
    }
    val frame = new ChartFrame("Parallel Coordinates",
      ChartFactory.createXYLineChart("Parallel Coordinates", "x", "y",
      dataset, org.jfree.chart.plot.PlotOrientation.VERTICAL,
      false, false, false))
    frame.pack()
    frame.setVisible(true)
  }
} 
Example 36
Source File: ListViewExample.scala    From scalajs-react-native   with Apache License 2.0 5 votes vote down vote up
package chandu0101.scalajs.rn.examples.uiexplorer.components

import chandu0101.scalajs.rn._
import chandu0101.scalajs.rn.components._
import chandu0101.scalajs.rn.examples.uiexplorer.{UIExplorerPage, UIExample}
import japgolly.scalajs.react.BackendScope
import chandu0101.scalajs.rn.styles.NativeStyleSheet
import scala.collection.mutable.Map
import scala.scalajs.js


object ListViewExample extends UIExample {

  val THUMB_URLS = js.Array("https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-ash3/t39.1997/p128x128/851549_767334479959628_274486868_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851561_767334496626293_1958532586_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-ash3/t39.1997/p128x128/851579_767334503292959_179092627_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851589_767334513292958_1747022277_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851563_767334559959620_1193692107_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851593_767334566626286_1953955109_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851591_767334523292957_797560749_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851567_767334529959623_843148472_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851548_767334489959627_794462220_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851575_767334539959622_441598241_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-ash3/t39.1997/p128x128/851573_767334549959621_534583464_n.png", "https://fbcdn-dragon-a.akamaihd.net/hphotos-ak-prn1/t39.1997/p128x128/851583_767334573292952_1519550680_n.png")
  val LOREM_IPSUM = "Lorem ipsum dolor sit amet, ius ad pertinax oportere accommodare, an vix civibus corrumpit referrentur. Te nam case ludus inciderint, te mea facilisi adipiscing. Sea id integre luptatum. In tota sale consequuntur nec. Erat ocurreret mei ei. Eu paulo sapientem vulputate est, vel an accusam intellegam interesset. Nam eu stet pericula reprimique, ea vim illud modus, putant invidunt reprehendunt ne qui.";

  case class State(datasource : ListViewDataSource[String] = createListViewDataSource[String,js.Object](rowHasChanged = (r1,r2) => r1 != r2))

  class Backend(t: BackendScope[_, State]) {

    val pressedData = scala.collection.mutable.Map[String,Boolean]().withDefaultValue(false)

    def genRows(pressedData : Map[String,Boolean]) = {
      val dataBlob = js.Array[String]()
      (1 to 100).toList.zipWithIndex.foreach {
        case (i,index) =>  {
          val pressedText = if(pressedData.getOrElse(index.toString,false)) "pressed" else ""
          dataBlob += s"Row $i $pressedText"
        }
      }
      dataBlob
    }


    def pressRow(rowID : String) = {
      pressedData.updated(rowID,pressedData(rowID))
      t.modState(s => s.copy(s.datasource.cloneWithRows(genRows(pressedData))))
    }

    def hashCode2(str : String) = {
      var hash = 15
      str.reverse.foreach( c => {
        hash = ((hash << 5) - hash) + c.toInt
      })
      hash
    }

    def renderRow(rowData : String, sectionID : String,rowID : String) = {
      val rowHash = Math.abs(hashCode2(rowData))
      val imageSource = ImageSource(uri = THUMB_URLS(rowHash % THUMB_URLS.length))
      TouchableHighlight(onPress = () => pressRow(rowID))(
       View()(
        View(style = styles.row)(
         Image(style = styles.thumb , source = imageSource),
         Text(style = styles.text)(
          s"$rowData - ${LOREM_IPSUM.substring(0,rowHash % 301 + 10)}"
         ),
         View(style = styles.separator)()
        )
       )
      )
    }

    val propsDynamic = t.propsDynamic
  }
  val component = ReactNativeComponentB[Unit]("ListViewExample")
    .initialState(State())
    .backend(new Backend(_))
    .render((P,S,B) => {
      View()(
       ListView(dataSource = S.datasource,renderRow = B.renderRow _)
      )
    })
    .componentWillMount(scope => scope.modState(s => s.copy(s.datasource.cloneWithRows(scope.backend.genRows(Map())))))
    .buildNative

  object styles extends NativeStyleSheet {

    val row = style(
     flexDirection.row,
     justifyContent.center,
     padding := 10,
     backgroundColor := "#F6F6F6"
    )

    val separator = style(
     height := 1,
    backgroundColor := "#F6F6F6"
    )

    val thumb = style(width := 64, height := 64)

    val text = style(flex := 1)

  }

  override def title: String = "ListView - simple"

  override def description: String = "Performant, scrollable list of data."

} 
Example 37
Source File: plot.scala    From Scientific-Computing-with-Scala   with MIT License 5 votes vote down vote up
import org.jfree.chart._
import org.jfree.data.xy._
import scala.math._
import scala.collection.mutable.{MutableList, Map}
import java.io.{FileReader, BufferedReader}

object AndrewsCurve {
  def readCSVFile(filename: String): Map[String, MutableList[String]] = {
    val file = new FileReader(filename)
    val reader = new BufferedReader(file)
    val csvdata: Map[String, MutableList[String]] = Map()
    try {
      val alldata = new MutableList[Array[String]]
      var line:String = null
      while ({line = reader.readLine(); line} != null) {
        if (line.length != 0) {
          val delimiter: String = ","
          var splitline: Array[String] = line.split(delimiter).map(_.trim)
          alldata += splitline
        }
      }
      val labels = MutableList("sepal length", "sepal width",
        "petal length", "petal width", "class")
      val labelled = labels.zipWithIndex.map {
        case (label, index) => label -> alldata.map(x => x(index))
      }
      for (pair <- labelled) {
        csvdata += pair
      }
    } finally {
      reader.close()
    }
    csvdata
  }

  def andrewsCurve(row: Array[Double]) = (t: Double) => {
    var result: Double = 0.0
    for ((attr, i) <- row.zipWithIndex) {
      if (i == 0) {
        result = result + row(i) / sqrt(2.0)
      } else if (i % 2 != 0) {
        result = result + row(i) * sin(((i + 1) / 2) * t)
      } else {
        result = result + row(i) * cos(((i + 1) / 2) * t)
      }
    }
    result
  }

  def main(args: Array[String]) {
    val data = readCSVFile("iris.csv")
    val x: Array[Double] = Array.tabulate(100) {
      (i: Int) => -Pi + 2.0 * Pi * (i / 100.0)
    }
    val dataset = new DefaultXYDataset
    for (i <- 0 until data("sepal length").size) {
      val x1 = data("sepal length")(i).toDouble
      val x2 = data("sepal width")(i).toDouble
      val x3 = data("petal length")(i).toDouble
      val x4 = data("petal width")(i).toDouble
      val cls = data("class")(i)
      val curve = x.map(andrewsCurve(Array(x1, x2, x3, x4)))
      dataset.addSeries(cls + i, Array(x, curve))
    }
    val frame = new ChartFrame("Andrews Curve",
      ChartFactory.createXYLineChart("Andrews Curve", "x", "y",
      dataset, org.jfree.chart.plot.PlotOrientation.VERTICAL,
      false, false, false))
    frame.pack()
    frame.setVisible(true)
  }
} 
Example 38
Source File: TaskResult.scala    From SparkCore   with Apache License 2.0 5 votes vote down vote up
package org.apache.spark.scheduler

import java.io._
import java.nio.ByteBuffer

import scala.collection.mutable.Map

import org.apache.spark.SparkEnv
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.storage.BlockId
import org.apache.spark.util.Utils

// Task result. Also contains updates to accumulator variables.
private[spark] sealed trait TaskResult[T]


private[spark]
class DirectTaskResult[T](var valueBytes: ByteBuffer, var accumUpdates: Map[Long, Any],
    var metrics: TaskMetrics)
  extends TaskResult[T] with Externalizable {

  def this() = this(null.asInstanceOf[ByteBuffer], null, null)

  override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException {

    out.writeInt(valueBytes.remaining);
    Utils.writeByteBuffer(valueBytes, out)

    out.writeInt(accumUpdates.size)
    for ((key, value) <- accumUpdates) {
      out.writeLong(key)
      out.writeObject(value)
    }
    out.writeObject(metrics)
  }

  override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {

    val blen = in.readInt()
    val byteVal = new Array[Byte](blen)
    in.readFully(byteVal)
    valueBytes = ByteBuffer.wrap(byteVal)

    val numUpdates = in.readInt
    if (numUpdates == 0) {
      accumUpdates = null
    } else {
      accumUpdates = Map()
      for (i <- 0 until numUpdates) {
        accumUpdates(in.readLong()) = in.readObject()
      }
    }
    metrics = in.readObject().asInstanceOf[TaskMetrics]
  }

  def value(): T = {
    val resultSer = SparkEnv.get.serializer.newInstance()
    resultSer.deserialize(valueBytes)
  }
} 
Example 39
Source File: Student.scala    From Scala-Design-Patterns-Second-Edition   with MIT License 5 votes vote down vote up
package com.ivan.nikolov.behavioral.mediator

import scala.collection.mutable.Map
import scala.collection.mutable.Set

trait Notifiable {
  def notify(message: String)
}

case class Student(name: String, age: Int) extends Notifiable {
  override def notify(message: String): Unit = {
    System.out.println(s"Student $name was notified with message: '$message'.")
  }
}

case class Group(name: String)

trait Mediator {
  def addStudentToGroup(student: Student, group: Group)
  
  def isStudentInGroup(student: Student, group: Group): Boolean
  
  def removeStudentFromGroup(student: Student, group: Group)
  
  def getStudentsInGroup(group: Group): List[Student]
  
  def getGroupsForStudent(student: Student): List[Group]
  
  def notifyStudentsInGroup(group: Group, message: String)
}

class School extends Mediator {
  val studentsToGroups: Map[Student, Set[Group]] = Map()
  val groupsToStudents: Map[Group, Set[Student]] = Map()

  override def addStudentToGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElseUpdate(student, Set()) += group
    groupsToStudents.getOrElseUpdate(group, Set()) += student
  }

  override def isStudentInGroup(student: Student, group: Group): Boolean = 
    groupsToStudents.getOrElse(group, Set()).contains(student) && studentsToGroups.getOrElse(student, Set()).contains(group)

  override def getStudentsInGroup(group: Group): List[Student] = 
    groupsToStudents.getOrElse(group, Set()).toList

  override def getGroupsForStudent(student: Student): List[Group] = 
    studentsToGroups.getOrElse(student, Set()).toList

  override def notifyStudentsInGroup(group: Group, message: String): Unit = {
    groupsToStudents.getOrElse(group, Set()).foreach(_.notify(message))
  }

  override def removeStudentFromGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElse(student, Set()) -= group
    groupsToStudents.getOrElse(group, Set()) -= student
  }
}

object SchoolExample {
  def main(args: Array[String]): Unit = {
    val school = new School
    // create students
    val student1 = Student("Ivan", 26)
    val student2 = Student("Maria", 26)
    val student3 = Student("John", 25)
    // create groups
    val group1 = Group("Scala design patterns")
    val group2 = Group("Databases")
    val group3 = Group("Cloud computing")
    
    school.addStudentToGroup(student1, group1)
    school.addStudentToGroup(student1, group2)
    school.addStudentToGroup(student1, group3)
    
    school.addStudentToGroup(student2, group1)
    school.addStudentToGroup(student2, group3)

    school.addStudentToGroup(student3, group1)
    school.addStudentToGroup(student3, group2)
    
    // notify
    school.notifyStudentsInGroup(group1, "Design patterns in Scala are amazing!")
    
    // see groups
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    // remove from group
    school.removeStudentFromGroup(student3, group2)
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    
    // see students in group
    System.out.println(s"Students in $group1 are ${school.getStudentsInGroup(group1)}")
  }
} 
Example 40
Source File: Student.scala    From Scala-Design-Patterns-Second-Edition   with MIT License 5 votes vote down vote up
package com.ivan.nikolov.behavioral.mediator

import scala.collection.mutable.Map
import scala.collection.mutable.Set

trait Notifiable {
  def notify(message: String)
}

case class Student(name: String, age: Int) extends Notifiable {
  override def notify(message: String): Unit = {
    System.out.println(s"Student $name was notified with message: '$message'.")
  }
}

case class Group(name: String)

trait Mediator {
  def addStudentToGroup(student: Student, group: Group)
  
  def isStudentInGroup(student: Student, group: Group): Boolean
  
  def removeStudentFromGroup(student: Student, group: Group)
  
  def getStudentsInGroup(group: Group): List[Student]
  
  def getGroupsForStudent(student: Student): List[Group]
  
  def notifyStudentsInGroup(group: Group, message: String)
}

class School extends Mediator {
  val studentsToGroups: Map[Student, Set[Group]] = Map()
  val groupsToStudents: Map[Group, Set[Student]] = Map()

  override def addStudentToGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElseUpdate(student, Set()) += group
    groupsToStudents.getOrElseUpdate(group, Set()) += student
  }

  override def isStudentInGroup(student: Student, group: Group): Boolean = 
    groupsToStudents.getOrElse(group, Set()).contains(student) && studentsToGroups.getOrElse(student, Set()).contains(group)

  override def getStudentsInGroup(group: Group): List[Student] = 
    groupsToStudents.getOrElse(group, Set()).toList

  override def getGroupsForStudent(student: Student): List[Group] = 
    studentsToGroups.getOrElse(student, Set()).toList

  override def notifyStudentsInGroup(group: Group, message: String): Unit = {
    groupsToStudents.getOrElse(group, Set()).foreach(_.notify(message))
  }

  override def removeStudentFromGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElse(student, Set()) -= group
    groupsToStudents.getOrElse(group, Set()) -= student
  }
}

object SchoolExample {
  def main(args: Array[String]): Unit = {
    val school = new School
    // create students
    val student1 = Student("Ivan", 26)
    val student2 = Student("Maria", 26)
    val student3 = Student("John", 25)
    // create groups
    val group1 = Group("Scala design patterns")
    val group2 = Group("Databases")
    val group3 = Group("Cloud computing")
    
    school.addStudentToGroup(student1, group1)
    school.addStudentToGroup(student1, group2)
    school.addStudentToGroup(student1, group3)
    
    school.addStudentToGroup(student2, group1)
    school.addStudentToGroup(student2, group3)

    school.addStudentToGroup(student3, group1)
    school.addStudentToGroup(student3, group2)
    
    // notify
    school.notifyStudentsInGroup(group1, "Design patterns in Scala are amazing!")
    
    // see groups
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    // remove from group
    school.removeStudentFromGroup(student3, group2)
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    
    // see students in group
    System.out.println(s"Students in $group1 are ${school.getStudentsInGroup(group1)}")
  }
} 
Example 41
Source File: Student.scala    From Scala-Design-Patterns-Second-Edition   with MIT License 5 votes vote down vote up
package com.ivan.nikolov.behavioral.mediator

import scala.collection.mutable.Map
import scala.collection.mutable.Set

trait Notifiable {
  def notify(message: String)
}

case class Student(name: String, age: Int) extends Notifiable {
  override def notify(message: String): Unit = {
    System.out.println(s"Student $name was notified with message: '$message'.")
  }
}

case class Group(name: String)

trait Mediator {
  def addStudentToGroup(student: Student, group: Group)
  
  def isStudentInGroup(student: Student, group: Group): Boolean
  
  def removeStudentFromGroup(student: Student, group: Group)
  
  def getStudentsInGroup(group: Group): List[Student]
  
  def getGroupsForStudent(student: Student): List[Group]
  
  def notifyStudentsInGroup(group: Group, message: String)
}

class School extends Mediator {
  val studentsToGroups: Map[Student, Set[Group]] = Map()
  val groupsToStudents: Map[Group, Set[Student]] = Map()

  override def addStudentToGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElseUpdate(student, Set()) += group
    groupsToStudents.getOrElseUpdate(group, Set()) += student
  }

  override def isStudentInGroup(student: Student, group: Group): Boolean = 
    groupsToStudents.getOrElse(group, Set()).contains(student) && studentsToGroups.getOrElse(student, Set()).contains(group)

  override def getStudentsInGroup(group: Group): List[Student] = 
    groupsToStudents.getOrElse(group, Set()).toList

  override def getGroupsForStudent(student: Student): List[Group] = 
    studentsToGroups.getOrElse(student, Set()).toList

  override def notifyStudentsInGroup(group: Group, message: String): Unit = {
    groupsToStudents.getOrElse(group, Set()).foreach(_.notify(message))
  }

  override def removeStudentFromGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElse(student, Set()) -= group
    groupsToStudents.getOrElse(group, Set()) -= student
  }
}

object SchoolExample {
  def main(args: Array[String]): Unit = {
    val school = new School
    // create students
    val student1 = Student("Ivan", 26)
    val student2 = Student("Maria", 26)
    val student3 = Student("John", 25)
    // create groups
    val group1 = Group("Scala design patterns")
    val group2 = Group("Databases")
    val group3 = Group("Cloud computing")
    
    school.addStudentToGroup(student1, group1)
    school.addStudentToGroup(student1, group2)
    school.addStudentToGroup(student1, group3)
    
    school.addStudentToGroup(student2, group1)
    school.addStudentToGroup(student2, group3)

    school.addStudentToGroup(student3, group1)
    school.addStudentToGroup(student3, group2)
    
    // notify
    school.notifyStudentsInGroup(group1, "Design patterns in Scala are amazing!")
    
    // see groups
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    // remove from group
    school.removeStudentFromGroup(student3, group2)
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    
    // see students in group
    System.out.println(s"Students in $group1 are ${school.getStudentsInGroup(group1)}")
  }
} 
Example 42
Source File: Student.scala    From Scala-Design-Patterns-Second-Edition   with MIT License 5 votes vote down vote up
package com.ivan.nikolov.behavioral.mediator

import scala.collection.mutable.Map
import scala.collection.mutable.Set

trait Notifiable {
  def notify(message: String)
}

case class Student(name: String, age: Int) extends Notifiable {
  override def notify(message: String): Unit = {
    System.out.println(s"Student $name was notified with message: '$message'.")
  }
}

case class Group(name: String)

trait Mediator {
  def addStudentToGroup(student: Student, group: Group)
  
  def isStudentInGroup(student: Student, group: Group): Boolean
  
  def removeStudentFromGroup(student: Student, group: Group)
  
  def getStudentsInGroup(group: Group): List[Student]
  
  def getGroupsForStudent(student: Student): List[Group]
  
  def notifyStudentsInGroup(group: Group, message: String)
}

class School extends Mediator {
  val studentsToGroups: Map[Student, Set[Group]] = Map()
  val groupsToStudents: Map[Group, Set[Student]] = Map()

  override def addStudentToGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElseUpdate(student, Set()) += group
    groupsToStudents.getOrElseUpdate(group, Set()) += student
  }

  override def isStudentInGroup(student: Student, group: Group): Boolean = 
    groupsToStudents.getOrElse(group, Set()).contains(student) && studentsToGroups.getOrElse(student, Set()).contains(group)

  override def getStudentsInGroup(group: Group): List[Student] = 
    groupsToStudents.getOrElse(group, Set()).toList

  override def getGroupsForStudent(student: Student): List[Group] = 
    studentsToGroups.getOrElse(student, Set()).toList

  override def notifyStudentsInGroup(group: Group, message: String): Unit = {
    groupsToStudents.getOrElse(group, Set()).foreach(_.notify(message))
  }

  override def removeStudentFromGroup(student: Student, group: Group): Unit = {
    studentsToGroups.getOrElse(student, Set()) -= group
    groupsToStudents.getOrElse(group, Set()) -= student
  }
}

object SchoolExample {
  def main(args: Array[String]): Unit = {
    val school = new School
    // create students
    val student1 = Student("Ivan", 26)
    val student2 = Student("Maria", 26)
    val student3 = Student("John", 25)
    // create groups
    val group1 = Group("Scala design patterns")
    val group2 = Group("Databases")
    val group3 = Group("Cloud computing")
    
    school.addStudentToGroup(student1, group1)
    school.addStudentToGroup(student1, group2)
    school.addStudentToGroup(student1, group3)
    
    school.addStudentToGroup(student2, group1)
    school.addStudentToGroup(student2, group3)

    school.addStudentToGroup(student3, group1)
    school.addStudentToGroup(student3, group2)
    
    // notify
    school.notifyStudentsInGroup(group1, "Design patterns in Scala are amazing!")
    
    // see groups
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    // remove from group
    school.removeStudentFromGroup(student3, group2)
    System.out.println(s"$student3 is in groups: ${school.getGroupsForStudent(student3)}")
    
    // see students in group
    System.out.println(s"Students in $group1 are ${school.getStudentsInGroup(group1)}")
  }
} 
Example 43
Source File: Circle.scala    From Scala-Design-Patterns-Second-Edition   with MIT License 5 votes vote down vote up
package com.ivan.nikolov.structural.flyweight

import scala.collection.mutable.{ListBuffer, Map}

class Circle(color: Color) {
  System.out.println(s"Creating a circle with $color color.")
  
  override def toString(): String =
    s"Circle($color)"
}

object Circle {
  val cache = Map.empty[Color, Circle]
  
  def apply(color: Color): Circle =
    cache.getOrElseUpdate(color, new Circle(color))
  
  def circlesCreated(): Int = cache.size
}

class Graphic {
  val items = ListBuffer.empty[(Int, Int, Double, Circle)]
  
  def addCircle(x: Int, y: Int, radius: Double, circle: Circle): Unit = {
    items += ((x, y, radius, circle))
  }
  
  def draw(): Unit = {
    items.foreach {
      case (x, y, radius, circle) =>
        System.out.println(s"Drawing a circle at ($x, $y) with radius $radius: $circle")
    }
  }
}

object FlyweightExample {
  def main(args: Array[String]): Unit = {
    val graphic = new Graphic
    graphic.addCircle(1, 1, 1.0, Circle(Green))
    graphic.addCircle(1, 2, 1.0, Circle(Red))
    graphic.addCircle(2, 1, 1.0, Circle(Blue))
    graphic.addCircle(2, 2, 1.0, Circle(Green))
    graphic.addCircle(2, 3, 1.0, Circle(Yellow))
    graphic.addCircle(3, 2, 1.0, Circle(Magenta))
    graphic.addCircle(3, 3, 1.0, Circle(Blue))
    graphic.addCircle(4, 3, 1.0, Circle(Blue))
    graphic.addCircle(3, 4, 1.0, Circle(Yellow))
    graphic.addCircle(4, 4, 1.0, Circle(Red))
    
    graphic.draw()
    
    System.out.println(s"Total number of circle objects created: ${Circle.circlesCreated()}")
  }
} 
Example 44
Source File: Circle.scala    From Scala-Design-Patterns-Second-Edition   with MIT License 5 votes vote down vote up
package com.ivan.nikolov.structural.flyweight

import scala.collection.mutable.{ListBuffer, Map}

class Circle(color: Color) {
  System.out.println(s"Creating a circle with $color color.")
  
  override def toString(): String =
    s"Circle($color)"
}

object Circle {
  val cache = Map.empty[Color, Circle]
  
  def apply(color: Color): Circle =
    cache.getOrElseUpdate(color, new Circle(color))
  
  def circlesCreated(): Int = cache.size
}

class Graphic {
  val items = ListBuffer.empty[(Int, Int, Double, Circle)]
  
  def addCircle(x: Int, y: Int, radius: Double, circle: Circle): Unit = {
    items += ((x, y, radius, circle))
  }
  
  def draw(): Unit = {
    items.foreach {
      case (x, y, radius, circle) =>
        System.out.println(s"Drawing a circle at ($x, $y) with radius $radius: $circle")
    }
  }
}

object FlyweightExample {
  def main(args: Array[String]): Unit = {
    val graphic = new Graphic
    graphic.addCircle(1, 1, 1.0, Circle(Green))
    graphic.addCircle(1, 2, 1.0, Circle(Red))
    graphic.addCircle(2, 1, 1.0, Circle(Blue))
    graphic.addCircle(2, 2, 1.0, Circle(Green))
    graphic.addCircle(2, 3, 1.0, Circle(Yellow))
    graphic.addCircle(3, 2, 1.0, Circle(Magenta))
    graphic.addCircle(3, 3, 1.0, Circle(Blue))
    graphic.addCircle(4, 3, 1.0, Circle(Blue))
    graphic.addCircle(3, 4, 1.0, Circle(Yellow))
    graphic.addCircle(4, 4, 1.0, Circle(Red))
    
    graphic.draw()
    
    System.out.println(s"Total number of circle objects created: ${Circle.circlesCreated()}")
  }
} 
Example 45
Source File: LexicalTrie.scala    From jgo   with GNU General Public License v3.0 5 votes vote down vote up
package jgo.tools.compiler
package lexer

import scala.collection.mutable.Map
import scala.util.parsing.input.Reader

object LexicalTrie {
  //this class is mutable, but LexicalTrie's public interface prevents modification
  private class TrieNode {
    private[LexicalTrie] val branches:  Map[Char, TrieNode] = Map()
    private[LexicalTrie] var accepting: Option[String]      = None
  }
  
  def apply(elems: String*) = {
    val ret = new LexicalTrie
    elems foreach { ret += _ }
    ret
  }
}

final class LexicalTrie {
  import LexicalTrie.TrieNode
  
  private val root = new TrieNode
  
  private def += (str: String): LexicalTrie = {
    var cur = root
    var i = 0
    while (i < str.length) {
      cur = cur.branches.getOrElseUpdate(str(i), new TrieNode)
      i += 1
    }
    cur.accepting = Some(str)
    this
  }
  
  def contains(str: String): Boolean = {
    var cur = root
    var i = 0
    while (i < str.length) {
      if (cur.branches contains str(i)) {
        cur = cur.branches(str(i))
        i += 1
      }
      else
        return false
    }
    assert(if (cur.accepting.isDefined) cur.accepting.get == str else true)
    return cur.accepting.isDefined
  }
  
  def matchingPrefixOf(r: Reader[Char]): (Option[String], Reader[Char]) =
    prefixOfFromNode(root, r)
  
  private def prefixOfFromNode(cur: TrieNode, r: Reader[Char]): (Option[String], Reader[Char]) =
    if (cur.branches contains r.first) {
      val fromNextNode = prefixOfFromNode(cur.branches get r.first get, r.rest)
      if (fromNextNode._1.isDefined) //_1 refers to the first (there is no zeroth) term of the pair
        fromNextNode
      else
        (cur.accepting, r)
    }
    else
      (cur.accepting, r)
} 
Example 46
Source File: SparkSQLOperationManager.scala    From iolap   with Apache License 2.0 5 votes vote down vote up
package org.apache.spark.sql.hive.thriftserver.server

import java.util.{Map => JMap}
import scala.collection.mutable.Map

import org.apache.hive.service.cli._
import org.apache.hive.service.cli.operation.{ExecuteStatementOperation, Operation, OperationManager}
import org.apache.hive.service.cli.session.HiveSession
import org.apache.spark.Logging
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.hive.thriftserver.{SparkExecuteStatementOperation, ReflectionUtils}


private[thriftserver] class SparkSQLOperationManager(hiveContext: HiveContext)
  extends OperationManager with Logging {

  val handleToOperation = ReflectionUtils
    .getSuperField[JMap[OperationHandle, Operation]](this, "handleToOperation")

  val sessionToActivePool = Map[SessionHandle, String]()

  override def newExecuteStatementOperation(
      parentSession: HiveSession,
      statement: String,
      confOverlay: JMap[String, String],
      async: Boolean): ExecuteStatementOperation = synchronized {

    val operation = new SparkExecuteStatementOperation(parentSession, statement, confOverlay)(
      hiveContext, sessionToActivePool)
    handleToOperation.put(operation.getHandle, operation)
    operation
  }
}