org.scalatest.mock.MockitoSugar Scala Examples
The following examples show how to use org.scalatest.mock.MockitoSugar.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: StreamMethodsSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.api
import akka.actor.ActorSystem
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5.KernelMessage
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, BeforeAndAfter, Matchers}
import play.api.libs.json.Json
import test.utils.MaxAkkaTestTimeout
import org.mockito.Mockito._
class StreamMethodsSpec extends TestKit(
ActorSystem(
"StreamMethodsSpec",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
)
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar
with BeforeAndAfter
{
private var kernelMessageRelayProbe: TestProbe = _
private var mockParentHeader: v5.ParentHeader = _
private var mockActorLoader: v5.kernel.ActorLoader = _
private var mockKernelMessage: v5.KernelMessage = _
private var streamMethods: StreamMethods = _
before {
kernelMessageRelayProbe = TestProbe()
mockParentHeader = mock[v5.ParentHeader]
mockActorLoader = mock[v5.kernel.ActorLoader]
doReturn(system.actorSelection(kernelMessageRelayProbe.ref.path))
.when(mockActorLoader).load(v5.SystemActorType.KernelMessageRelay)
mockKernelMessage = mock[v5.KernelMessage]
doReturn(mockParentHeader).when(mockKernelMessage).header
streamMethods = new StreamMethods(mockActorLoader, mockKernelMessage)
}
describe("StreamMethods") {
describe("#()") {
it("should put the header of the given message as the parent header") {
val expected = mockKernelMessage.header
val actual = streamMethods.kmBuilder.build.parentHeader
actual should be (expected)
}
}
describe("#sendAll") {
it("should send a message containing all of the given text") {
val expected = "some text"
streamMethods.sendAll(expected)
val outgoingMessage = kernelMessageRelayProbe.receiveOne(MaxAkkaTestTimeout)
val kernelMessage = outgoingMessage.asInstanceOf[KernelMessage]
val actual = Json.parse(kernelMessage.contentString)
.as[v5.content.StreamContent].text
actual should be (expected)
}
}
}
}
Example 2
| Source File: LSMagicSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import java.io.OutputStream
import java.net.URL
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.dependencies.{IncludeOutputStream, IncludeInterpreter}
import org.apache.toree.magic.{CellMagic, LineMagic}
import org.apache.spark.SparkContext
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
class TestLSMagic(sc: SparkContext, intp: Interpreter, os: OutputStream)
extends LSMagic
with IncludeInterpreter
with IncludeOutputStream
{
override val interpreter: Interpreter = intp
override val outputStream: OutputStream = os
}
class LSMagicSpec extends FunSpec with Matchers with MockitoSugar {
describe("LSMagic") {
describe("#execute") {
it("should call println with a magics message") {
val lsm = spy(new TestLSMagic(
mock[SparkContext], mock[Interpreter], mock[OutputStream])
)
val classList = new BuiltinLoader().loadClasses()
lsm.execute("")
verify(lsm).magicNames("%", classOf[LineMagic], classList)
verify(lsm).magicNames("%%", classOf[CellMagic], classList)
}
}
describe("#magicNames") {
it("should filter classnames by interface") {
val prefix = "%"
val interface = classOf[LineMagic]
val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
val lsm = new TestLSMagic(
mock[SparkContext], mock[Interpreter], mock[OutputStream])
lsm.magicNames(prefix, interface, classes).length should be(1)
}
it("should prepend prefix to each name"){
val prefix = "%"
val className = classOf[LSMagic].getSimpleName
val interface = classOf[LineMagic]
val expected = s"${prefix}${className}"
val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
val lsm = new TestLSMagic(
mock[SparkContext], mock[Interpreter], mock[OutputStream])
lsm.magicNames(prefix, interface, classes) should be(List(expected))
}
}
}
}
Example 3
| Source File: BuiltinLoaderSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpec}
class BuiltinLoaderSpec extends FunSpec with Matchers with MockitoSugar {
describe("BuiltinLoader") {
describe("#getClasses") {
it("should return classes in a package") {
val pkg = this.getClass.getPackage.getName
val classes = new BuiltinLoader().getClasses(pkg)
classes.size shouldNot be(0)
}
}
describe("#loadClasses") {
it("should return class objects for classes in a package") {
val pkg = this.getClass.getPackage.getName
val classes = new BuiltinLoader().loadClasses(pkg).toList
classes.contains(this.getClass) should be (true)
}
}
}
}
Example 4
| Source File: HtmlSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import org.apache.toree.kernel.protocol.v5.MIMEType
import org.apache.toree.magic.CellMagicOutput
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers}
class HtmlSpec extends FunSpec with Matchers with MockitoSugar {
describe("Html"){
describe("#execute") {
it("should return the entire cell's contents with the MIME type of " +
"text/html") {
val htmlMagic = new Html
val code = "some code on a line\nanother line"
val expected = CellMagicOutput(MIMEType.TextHtml -> code)
htmlMagic.execute(code) should be (expected)
}
}
}
}
Example 5
| Source File: JavaScriptSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers}
import org.apache.toree.magic.CellMagicOutput
import org.apache.toree.kernel.protocol.v5.MIMEType
class JavaScriptSpec extends FunSpec with Matchers with MockitoSugar {
describe("JavaScript"){
describe("#execute") {
it("should return the entire cell's contents with the MIME type of text/javascript") {
val javaScriptMagic = new JavaScript
val code = "some code on a line\nmore code on another line"
val expected = CellMagicOutput(MIMEType.ApplicationJavaScript -> code)
javaScriptMagic.execute(code) should be (expected)
}
}
}
}
Example 6
| Source File: CodeCompleteHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.handler
import akka.actor._
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import org.apache.toree.Main
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content.CompleteRequest
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5Test._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, BeforeAndAfter, Matchers}
import org.mockito.Mockito._
import test.utils.MaxAkkaTestTimeout
class CodeCompleteHandlerSpec extends TestKit(
ActorSystem("CodeCompleteHandlerSpec", None, Some(Main.getClass.getClassLoader))
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar
with BeforeAndAfter {
var actorLoader: ActorLoader = _
var handlerActor: ActorRef = _
var kernelMessageRelayProbe: TestProbe = _
var interpreterProbe: TestProbe = _
var statusDispatchProbe: TestProbe = _
before {
actorLoader = mock[ActorLoader]
handlerActor = system.actorOf(Props(classOf[CodeCompleteHandler], actorLoader))
kernelMessageRelayProbe = TestProbe()
when(actorLoader.load(SystemActorType.KernelMessageRelay))
.thenReturn(system.actorSelection(kernelMessageRelayProbe.ref.path.toString))
interpreterProbe = new TestProbe(system)
when(actorLoader.load(SystemActorType.Interpreter))
.thenReturn(system.actorSelection(interpreterProbe.ref.path.toString))
statusDispatchProbe = new TestProbe(system)
when(actorLoader.load(SystemActorType.StatusDispatch))
.thenReturn(system.actorSelection(statusDispatchProbe.ref.path.toString))
}
def replyToHandlerWithOkAndResult() = {
val expectedClass = classOf[CompleteRequest]
interpreterProbe.expectMsgClass(expectedClass)
interpreterProbe.reply((0, List[String]()))
}
def replyToHandlerWithOkAndBadResult() = {
val expectedClass = classOf[CompleteRequest]
interpreterProbe.expectMsgClass(expectedClass)
interpreterProbe.reply("hello")
}
describe("CodeCompleteHandler (ActorLoader)") {
it("should send a CompleteRequest") {
handlerActor ! MockCompleteRequestKernelMessage
replyToHandlerWithOkAndResult()
kernelMessageRelayProbe.fishForMessage(MaxAkkaTestTimeout) {
case KernelMessage(_, _, header, _, _, _) =>
header.msg_type == MessageType.Outgoing.CompleteReply.toString
}
}
it("should throw an error for bad JSON") {
handlerActor ! MockKernelMessageWithBadJSON
var result = false
try {
replyToHandlerWithOkAndResult()
}
catch {
case t: Throwable => result = true
}
result should be (true)
}
it("should throw an error for bad code completion") {
handlerActor ! MockCompleteRequestKernelMessage
try {
replyToHandlerWithOkAndBadResult()
}
catch {
case error: Exception => error.getMessage should be ("Parse error in CodeCompleteHandler")
}
}
it("should send an idle message") {
handlerActor ! MockCompleteRequestKernelMessage
replyToHandlerWithOkAndResult()
statusDispatchProbe.fishForMessage(MaxAkkaTestTimeout) {
case Tuple2(status, _) =>
status == KernelStatusType.Idle
}
}
}
}
Example 7
| Source File: GenericSocketMessageHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.handler
import akka.actor.{ActorSystem, Props, ActorRef, ActorSelection}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5Test._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import test.utils.MaxAkkaTestTimeout
class GenericSocketMessageHandlerSpec extends TestKit(
ActorSystem(
"GenericSocketMessageHandlerSystem",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("GenericSocketMessageHandler( ActorLoader, SocketType )") {
// Create a mock ActorLoader for the Relay we are going to test
val actorLoader: ActorLoader = mock[ActorLoader]
// Create a probe for the ActorSelection that the ActorLoader will return
val selectionProbe: TestProbe = TestProbe()
val selection: ActorSelection = system.actorSelection(selectionProbe.ref.path.toString)
when(actorLoader.load(SocketType.Control)).thenReturn(selection)
// The Relay we are going to be testing against
val genericHandler: ActorRef = system.actorOf(
Props(classOf[GenericSocketMessageHandler], actorLoader, SocketType.Control)
)
describe("#receive( KernelMessage )") {
genericHandler ! MockKernelMessage
it("should send the message to the selected actor"){
selectionProbe.expectMsg(MaxAkkaTestTimeout, MockKernelMessage)
}
}
}
}
Example 8
| Source File: KernelInfoRequestHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.handler
import akka.actor.{ActorSelection, ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.Main
import org.apache.toree.kernel.protocol.v5.content.KernelInfoReply
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5._
import org.mockito.AdditionalMatchers.{not => mockNot}
import org.mockito.Matchers.{eq => mockEq}
import com.typesafe.config.ConfigFactory
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import play.api.libs.json.Json
import test.utils.MaxAkkaTestTimeout
object KernelInfoRequestHandlerSpec {
val config = """
akka {
loglevel = "WARNING"
}"""
}
class KernelInfoRequestHandlerSpec extends TestKit(
ActorSystem("KernelInfoRequestHandlerSpec",
ConfigFactory.parseString(KernelInfoRequestHandlerSpec.config),
Main.getClass.getClassLoader)
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val actorLoader: ActorLoader = mock[ActorLoader]
val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, LanguageInfo("test", "1.0.0", Some(".test"))))
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection =
system.actorSelection(relayProbe.ref.path)
when(actorLoader.load(SystemActorType.KernelMessageRelay))
.thenReturn(relaySelection)
when(actorLoader.load(mockNot(mockEq(SystemActorType.KernelMessageRelay))))
.thenReturn(system.actorSelection(""))
val header = Header("","","","","")
val kernelMessage = new KernelMessage(
Seq[Array[Byte]](), "test message", header, header, Metadata(), "{}"
)
describe("Kernel Info Request Handler") {
it("should return a KernelMessage containing kernel info response") {
actor ! kernelMessage
val reply = relayProbe.receiveOne(MaxAkkaTestTimeout).asInstanceOf[KernelMessage]
val kernelInfo = Json.parse(reply.contentString).as[KernelInfoReply]
kernelInfo.implementation should be ("spark")
}
}
}
Example 9
| Source File: ShellSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import java.nio.charset.Charset
import akka.actor.{ActorSelection, ActorRef, ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import akka.util.ByteString
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.kernel.{ActorLoader, Utilities}
import org.apache.toree.kernel.protocol.v5Test._
import Utilities._
import com.typesafe.config.ConfigFactory
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.MaxAkkaTestTimeout
object ShellSpec {
val config ="""
akka {
loglevel = "WARNING"
}"""
}
class ShellSpec extends TestKit(
ActorSystem(
"ShellActorSpec",
ConfigFactory.parseString(ShellSpec.config),
org.apache.toree.Main.getClass.getClassLoader
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("Shell") {
val socketFactory = mock[SocketFactory]
val actorLoader = mock[ActorLoader]
val socketProbe : TestProbe = TestProbe()
when(socketFactory.Shell(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(socketProbe.ref)
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection = system.actorSelection(relayProbe.ref.path)
when(actorLoader.load(SystemActorType.KernelMessageRelay)).thenReturn(relaySelection)
val shell = system.actorOf(Props(classOf[Shell], socketFactory, actorLoader))
describe("#receive") {
it("( KernelMessage ) should reply with a ZMQMessage via the socket") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
shell ! MockKernelMessage
socketProbe.expectMsg(MockZMQMessage)
}
it("( ZMQMessage ) should forward ZMQ Strings and KernelMessage to Relay") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
shell ! MockZMQMessage
// Should get the last four (assuming no buffer) strings in UTF-8
val zmqStrings = MockZMQMessage.frames.map((byteString: ByteString) =>
new String(byteString.toArray, Charset.forName("UTF-8"))
).takeRight(4)
val kernelMessage: KernelMessage = MockZMQMessage
relayProbe.expectMsg(MaxAkkaTestTimeout, (zmqStrings, kernelMessage))
}
}
}
}
Example 10
| Source File: IOPubSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import akka.actor.{ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5.kernel.Utilities
import org.apache.toree.kernel.protocol.v5Test._
import Utilities._
import com.typesafe.config.ConfigFactory
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.MaxAkkaTestTimeout
object IOPubSpec {
val config ="""
akka {
loglevel = "WARNING"
}"""
}
class IOPubSpec extends TestKit(
ActorSystem("IOPubActorSpec",
ConfigFactory.parseString(IOPubSpec.config),
org.apache.toree.Main.getClass.getClassLoader
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("IOPubActor") {
val socketFactory = mock[SocketFactory]
val probe : TestProbe = TestProbe()
when(socketFactory.IOPub(any(classOf[ActorSystem]))).thenReturn(probe.ref)
val socket = system.actorOf(Props(classOf[IOPub], socketFactory))
// TODO test that the response type changed
describe("#receive") {
it("should reply with a ZMQMessage") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
socket ! MockKernelMessage
probe.expectMsg(MaxAkkaTestTimeout, MockZMQMessage)
}
}
}
}
Example 11
| Source File: HeartbeatSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import akka.util.ByteString
import org.apache.toree.communication.ZMQMessage
import com.typesafe.config.ConfigFactory
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.MaxAkkaTestTimeout
object HeartbeatSpec {
val config = """
akka {
loglevel = "WARNING"
}"""
}
class HeartbeatSpec extends TestKit(
ActorSystem(
"HeartbeatActorSpec",
ConfigFactory.parseString(HeartbeatSpec.config),
org.apache.toree.Main.getClass.getClassLoader
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val SomeMessage: String = "some message"
val SomeZMQMessage: ZMQMessage = ZMQMessage(ByteString(SomeMessage.getBytes))
describe("HeartbeatActor") {
val socketFactory = mock[SocketFactory]
val probe : TestProbe = TestProbe()
when(socketFactory.Heartbeat(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(probe.ref)
val heartbeat = system.actorOf(Props(classOf[Heartbeat], socketFactory))
describe("send heartbeat") {
it("should receive and send same ZMQMessage") {
heartbeat ! SomeZMQMessage
probe.expectMsg(MaxAkkaTestTimeout, SomeZMQMessage)
}
}
}
}
Example 12
| Source File: StdinSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import java.nio.charset.Charset
import akka.actor.{Props, ActorSelection, ActorRef, ActorSystem}
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import akka.util.ByteString
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5.kernel.Utilities._
import org.apache.toree.kernel.protocol.v5Test._
import org.apache.toree.kernel.protocol.v5.{KernelMessage, SystemActorType}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import com.typesafe.config.ConfigFactory
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import org.mockito.Mockito._
import org.mockito.Matchers._
import test.utils.MaxAkkaTestTimeout
object StdinSpec {
val config ="""
akka {
loglevel = "WARNING"
}"""
}
class StdinSpec extends TestKit(ActorSystem(
"StdinActorSpec",
ConfigFactory.parseString(StdinSpec.config),
org.apache.toree.Main.getClass.getClassLoader
)) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("Stdin") {
val socketFactory = mock[SocketFactory]
val actorLoader = mock[ActorLoader]
val socketProbe : TestProbe = TestProbe()
when(socketFactory.Stdin(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(socketProbe.ref)
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection = system.actorSelection(relayProbe.ref.path)
when(actorLoader.load(SystemActorType.KernelMessageRelay)).thenReturn(relaySelection)
val stdin = system.actorOf(Props(classOf[Stdin], socketFactory, actorLoader))
describe("#receive") {
it("( KernelMessage ) should reply with a ZMQMessage via the socket") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
stdin ! MockKernelMessage
socketProbe.expectMsg(MockZMQMessage)
}
it("( ZMQMessage ) should forward ZMQ Strings and KernelMessage to Relay") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
stdin ! MockZMQMessage
// Should get the last four (assuming no buffer) strings in UTF-8
val zmqStrings = MockZMQMessage.frames.map((byteString: ByteString) =>
new String(byteString.toArray, Charset.forName("UTF-8"))
).takeRight(4)
val kernelMessage: KernelMessage = MockZMQMessage
relayProbe.expectMsg(MaxAkkaTestTimeout, (zmqStrings, kernelMessage))
}
}
}
}
Example 13
| Source File: ActorLoaderSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel
import akka.actor.{ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5.{MessageType, SocketType}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.TestProbeProxyActor
import test.utils.MaxAkkaTestTimeout
class ActorLoaderSpec extends TestKit(
ActorSystem(
"ActorLoaderSpecSystem",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("ActorLoader"){
describe("#load( MessageType )"){
it("should load an ActorSelection that has been loaded into the system"){
val testProbe: TestProbe = TestProbe()
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe),
MessageType.Outgoing.ClearOutput.toString)
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(MessageType.Outgoing.ClearOutput) ! "<Test Message>"
testProbe.expectMsg("<Test Message>")
}
it("should expect no message when there is no actor"){
val testProbe: TestProbe = TestProbe()
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(MessageType.Outgoing.CompleteReply) ! "<Test Message>"
testProbe.expectNoMessage(MaxAkkaTestTimeout)
// This is to test to see if there the messages go to the actor inbox or the dead mail inbox
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe),
MessageType.Outgoing.CompleteReply.toString)
testProbe.expectNoMessage(MaxAkkaTestTimeout)
}
}
describe("#load( SocketType )"){
it("should load an ActorSelection that has been loaded into the system"){
val testProbe: TestProbe = TestProbe()
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe), SocketType.Shell.toString)
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(SocketType.Shell) ! "<Test Message>"
testProbe.expectMsg("<Test Message>")
}
it("should expect no message when there is no actor"){
val testProbe: TestProbe = TestProbe()
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(SocketType.IOPub) ! "<Test Message>"
testProbe.expectNoMessage(MaxAkkaTestTimeout)
// This is to test to see if there the messages go to the actor inbox or the dead mail inbox
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe), SocketType.IOPub.toString)
testProbe.expectNoMessage(MaxAkkaTestTimeout)
}
}
}
}
Example 14
| Source File: StatusDispatchSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.dispatch
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content.KernelStatus
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, FunSpecLike, Matchers}
import play.api.libs.json.Json
import test.utils.MaxAkkaTestTimeout
class StatusDispatchSpec extends TestKit(
ActorSystem(
"StatusDispatchSystem",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
)
)
with FunSpecLike with Matchers with MockitoSugar with BeforeAndAfter{
var statusDispatchRef: ActorRef = _
var relayProbe: TestProbe = _
before {
// Mock the relay with a probe
relayProbe = TestProbe()
// Mock the ActorLoader
val mockActorLoader: ActorLoader = mock[ActorLoader]
when(mockActorLoader.load(SystemActorType.KernelMessageRelay))
.thenReturn(system.actorSelection(relayProbe.ref.path.toString))
statusDispatchRef = system.actorOf(Props(classOf[StatusDispatch],mockActorLoader))
}
describe("StatusDispatch") {
describe("#receive( KernelStatusType )") {
it("should send a status message to the relay") {
statusDispatchRef ! KernelStatusType.Busy
// Check the kernel message is the correct type
val statusMessage: KernelMessage = relayProbe.receiveOne(MaxAkkaTestTimeout).asInstanceOf[KernelMessage]
statusMessage.header.msg_type should be (MessageType.Outgoing.Status.toString)
// Check the status is what we sent
val status: KernelStatus = Json.parse(statusMessage.contentString).as[KernelStatus]
status.execution_state should be (KernelStatusType.Busy.toString)
}
}
describe("#receive( KernelStatusType, Header )") {
it("should send a status message to the relay") {
val tuple = Tuple2(KernelStatusType.Busy, mock[Header])
statusDispatchRef ! tuple
// Check the kernel message is the correct type
val statusMessage: KernelMessage = relayProbe.receiveOne(MaxAkkaTestTimeout).asInstanceOf[KernelMessage]
statusMessage.header.msg_type should be (MessageType.Outgoing.Status.toString)
// Check the status is what we sent
val status: KernelStatus = Json.parse(statusMessage.contentString).as[KernelStatus]
status.execution_state should be (KernelStatusType.Busy.toString)
}
}
}
}
Example 15
| Source File: KernelCommManagerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.comm
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content.CommContent
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
class KernelCommManagerSpec extends FunSpec with Matchers with BeforeAndAfter
with MockitoSugar
{
private val TestTargetName = "some target"
private var mockActorLoader: ActorLoader = _
private var mockKMBuilder: KMBuilder = _
private var mockCommRegistrar: CommRegistrar = _
private var kernelCommManager: KernelCommManager = _
private var generatedCommWriter: CommWriter = _
before {
mockActorLoader = mock[ActorLoader]
mockKMBuilder = mock[KMBuilder]
mockCommRegistrar = mock[CommRegistrar]
kernelCommManager = new KernelCommManager(
mockActorLoader,
mockKMBuilder,
mockCommRegistrar
) {
override protected def newCommWriter(commId: UUID): CommWriter = {
val commWriter = super.newCommWriter(commId)
generatedCommWriter = commWriter
val spyCommWriter = spy(commWriter)
doNothing().when(spyCommWriter)
.sendCommKernelMessage(any[KernelMessageContent with CommContent])
spyCommWriter
}
}
}
describe("KernelCommManager") {
describe("#open") {
it("should return a wrapped instance of KernelCommWriter") {
kernelCommManager.open(TestTargetName, v5.MsgData.Empty)
// Exposed hackishly for testing
generatedCommWriter shouldBe a [KernelCommWriter]
}
}
}
}
Example 16
| Source File: MultiOutputStreamSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import java.io.OutputStream
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
import org.mockito.Matchers._
import org.mockito.Mockito._
class MultiOutputStreamSpec
extends FunSpec with Matchers with MockitoSugar with BeforeAndAfter {
describe("MultiOutputStream") {
val listOfMockOutputStreams = List(mock[OutputStream], mock[OutputStream])
val multiOutputStream = MultiOutputStream(listOfMockOutputStreams)
describe("#close") {
it("should call #close on all internal output streams") {
multiOutputStream.close()
listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).close())
}
}
describe("#flush") {
it("should call #flush on all internal output streams") {
multiOutputStream.flush()
listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).flush())
}
}
describe("#write(int)") {
it("should call #write(int) on all internal output streams") {
multiOutputStream.write(anyInt())
listOfMockOutputStreams.foreach(
mockOutputStream => verify(mockOutputStream).write(anyInt()))
}
}
describe("#write(byte[])") {
it("should call #write(byte[]) on all internal output streams") {
multiOutputStream.write(any[Array[Byte]])
listOfMockOutputStreams.foreach(
mockOutputStream => verify(mockOutputStream).write(any[Array[Byte]]))
}
}
describe("#write(byte[], int, int)") {
it("should call #write(byte[], int, int) on all internal output streams") {
multiOutputStream.write(any[Array[Byte]], anyInt(), anyInt())
listOfMockOutputStreams.foreach(
mockOutputStream =>
verify(mockOutputStream).write(any[Array[Byte]], anyInt(), anyInt()))
}
}
}
}
Example 17
| Source File: ArgumentParsingSupportSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
import joptsimple.{OptionSet, OptionSpec, OptionParser}
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import collection.JavaConverters._
class ArgumentParsingSupportSpec extends FunSpec with Matchers
with BeforeAndAfter with MockitoSugar
{
private var mockOptions: OptionSet = _
private var mockParser: OptionParser = _
private var argumentParsingInstance: ArgumentParsingSupport = _
before {
mockOptions = mock[OptionSet]
mockParser = mock[OptionParser]
doReturn(mockOptions).when(mockParser).parse(anyVararg[String]())
argumentParsingInstance = new Object() with ArgumentParsingSupport {
override protected lazy val parser: OptionParser = mockParser
}
}
describe("ArgumentParsingSupport") {
describe("#parseArgs") {
it("should invoke the underlying parser's parse method") {
doReturn(Nil.asJava).when(mockOptions).nonOptionArguments()
argumentParsingInstance.parseArgs("")
verify(mockParser).parse(anyString())
}
it("should return an empty list if there are no non-option arguments") {
val expected = Nil
doReturn(expected.asJava).when(mockOptions).nonOptionArguments()
val actual = argumentParsingInstance.parseArgs((
"--transitive" :: expected
).mkString(" "))
actual should be (expected)
}
it("should return a list containing non-option arguments") {
val expected = "non-option" :: Nil
doReturn(expected.asJava).when(mockOptions).nonOptionArguments()
val actual = argumentParsingInstance.parseArgs((
"--transitive" :: expected
).mkString(" "))
actual should be (expected)
}
}
}
}
Example 18
| Source File: ConditionalOutputStreamSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import java.io.OutputStream
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{Matchers, FunSpec}
class ConditionalOutputStreamSpec extends FunSpec with Matchers with MockitoSugar {
describe("ConditionalOutputStream") {
describe("#()") {
it("should throw an exception if the output stream is null") {
intercept[IllegalArgumentException] {
new ConditionalOutputStream(null, true)
}
}
}
describe("#write") {
it("should call the underlying write if the condition is true") {
val mockOutputStream = mock[OutputStream]
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, true)
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream).write(expected)
}
it("should call the underlying write if the condition becomes true") {
val mockOutputStream = mock[OutputStream]
var condition = false
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, condition)
condition = true
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream).write(expected)
}
it("should not call the underlying write if the condition is false") {
val mockOutputStream = mock[OutputStream]
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, false)
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream, never()).write(any[Byte])
}
it("should not call the underlying write if the condition becomes false") {
val mockOutputStream = mock[OutputStream]
var condition = true
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, condition)
condition = false
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream, never()).write(any[Byte])
}
}
}
}
Example 19
| Source File: InternalClassLoaderSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar
class InternalClassLoaderSpec extends FunSpec with Matchers with MockitoSugar {
abstract class MockClassLoader extends ClassLoader(null) {
override def loadClass(name: String): Class[_] = null
}
describe("InternalClassLoader") {
describe("#loadClass") {
it("should invoke super loadClass with loader's package prepended") {
val expected = classOf[Class[_]]
val packageName = "org.apache.toree.magic"
val className = "SomeClass"
var parentLoadClassCorrectlyInvoked = false
val internalClassLoader = new InternalClassLoader(null) {
override private[magic] def parentLoadClass(name: String, resolve: Boolean): Class[_] = {
parentLoadClassCorrectlyInvoked =
name == s"$packageName.$className" && resolve
expected
}
}
internalClassLoader.loadClass(className, true) should be (expected)
parentLoadClassCorrectlyInvoked should be (true)
}
it("should use loader's package instead of provided package first") {
val expected = classOf[Class[_]]
val forcedPackageName = "org.apache.toree.magic"
val packageName = "some.other.package"
val className = "SomeClass"
var parentLoadClassCorrectlyInvoked = false
val internalClassLoader = new InternalClassLoader(null) {
override private[magic] def parentLoadClass(name: String, resolve: Boolean): Class[_] = {
parentLoadClassCorrectlyInvoked =
name == s"$forcedPackageName.$className" && resolve
expected
}
}
internalClassLoader.loadClass(s"$packageName.$className", true) should be (expected)
parentLoadClassCorrectlyInvoked should be (true)
}
it("should invoke super loadClass with given package if internal missing") {
val expected = classOf[Class[_]]
val packageName = "some.other.package"
val className = "SomeClass"
var parentLoadClassCorrectlyInvoked = false
var methodCalled = false
val internalClassLoader = new InternalClassLoader(null) {
override private[magic] def parentLoadClass(name: String, resolve: Boolean): Class[_] = {
if (!methodCalled) {
methodCalled = true
throw new ClassNotFoundException()
}
parentLoadClassCorrectlyInvoked =
name == s"$packageName.$className" && resolve
expected
}
}
internalClassLoader.loadClass(s"$packageName.$className", true) should
be (expected)
parentLoadClassCorrectlyInvoked should be (true)
}
}
}
}
Example 20
| Source File: BrokerProcessHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.interpreter.broker
import org.apache.commons.exec.ExecuteException
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers, OneInstancePerTest}
import org.mockito.Mockito._
import org.mockito.Matchers._
class BrokerProcessHandlerSpec extends FunSpec with Matchers
with OneInstancePerTest with MockitoSugar
{
private val mockBrokerBridge = mock[BrokerBridge]
private val brokerProcessHandler = new BrokerProcessHandler(
mockBrokerBridge,
restartOnFailure = true,
restartOnCompletion = true
)
describe("BrokerProcessHandler") {
describe("#onProcessFailed") {
it("should invoke the reset method") {
val mockResetMethod = mock[String => Unit]
brokerProcessHandler.setResetMethod(mockResetMethod)
brokerProcessHandler.onProcessFailed(mock[ExecuteException])
verify(mockResetMethod).apply(anyString())
}
it("should invoke the restart method if the proper flag is set to true") {
val mockRestartMethod = mock[() => Unit]
brokerProcessHandler.setRestartMethod(mockRestartMethod)
brokerProcessHandler.onProcessFailed(mock[ExecuteException])
verify(mockRestartMethod).apply()
}
}
describe("#onProcessComplete") {
it("should invoke the reset method") {
val mockResetMethod = mock[String => Unit]
brokerProcessHandler.setResetMethod(mockResetMethod)
brokerProcessHandler.onProcessComplete(0)
verify(mockResetMethod).apply(anyString())
}
it("should invoke the restart method if the proper flag is set to true") {
val mockRestartMethod = mock[() => Unit]
brokerProcessHandler.setRestartMethod(mockRestartMethod)
brokerProcessHandler.onProcessComplete(0)
verify(mockRestartMethod).apply()
}
}
}
}
Example 21
| Source File: BrokerBridgeSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.interpreter.broker
import org.apache.toree.kernel.api.KernelLike
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers, OneInstancePerTest}
class BrokerBridgeSpec extends FunSpec with Matchers with OneInstancePerTest
with MockitoSugar
{
private val mockBrokerState = mock[BrokerState]
private val mockKernel = mock[KernelLike]
private val brokerBridge = new BrokerBridge(
mockBrokerState,
mockKernel
)
describe("BrokerBridge") {
describe("#state") {
it("should return the broker state from the constructor") {
brokerBridge.state should be (mockBrokerState)
}
}
describe("#kernel") {
it("should return the kernel from the constructor") {
brokerBridge.kernel should be (mockKernel)
}
}
}
}
Example 22
| Source File: MarkdownReporterTest.scala From drunken-data-quality with Apache License 2.0 | 5 votes |
|
package de.frosner.ddq.reporters
import java.io.{ByteArrayOutputStream, PrintStream}
import de.frosner.ddq.constraints._
import de.frosner.ddq.core._
import de.frosner.ddq.testutils.{DummyConstraint, DummyConstraintResult}
import org.apache.spark.sql.DataFrame
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
class MarkdownReporterTest extends FlatSpec with Matchers with MockitoSugar {
"A Markdown reporter" should "produce correct output for a check with constraints" in {
val baos = new ByteArrayOutputStream()
val markdownReporter = new MarkdownReporter(new PrintStream(baos))
val df = mock[DataFrame]
val dfName = "myDf"
val dfColumns = Array("1", "2")
val dfCount = 5
when(df.columns).thenReturn(dfColumns)
val header = s"Checking $dfName"
val prologue = s"It has a total number of ${dfColumns.size} columns and $dfCount rows."
val message1 = "1"
val status1 = ConstraintSuccess
val constraint1 = DummyConstraint(message1, status1)
val result1 = constraint1.fun(df)
val message2 = "2"
val status2 = ConstraintFailure
val constraint2 = DummyConstraint(message2, status2)
val result2 = constraint2.fun(df)
val message3 = "3"
val status3 = ConstraintError(new IllegalArgumentException())
val constraint3 = DummyConstraint(message3, status3)
val result3 = DummyConstraintResult(constraint3, message3, status3)
val constraints = Map[Constraint, ConstraintResult[Constraint]](
constraint1 -> result1,
constraint2 -> result2,
constraint3 -> result3
)
val check = Check(df, Some(dfName), Option.empty, constraints.keys.toSeq)
markdownReporter.report(CheckResult(constraints, check, dfCount))
val expectedOutput = s"""**$header**
$prologue
- *SUCCESS*: ${result1.message}
- *FAILURE*: ${result2.message}
- *ERROR*: ${result3.message}
"""
baos.toString shouldBe expectedOutput
}
it should "produce correct output for a check without constraint" in {
val baos = new ByteArrayOutputStream()
val markdownReporter = new MarkdownReporter(new PrintStream(baos))
val df = mock[DataFrame]
val dfName = "myDf"
val dfColumns = Array("1", "2")
val dfCount = 5
when(df.columns).thenReturn(dfColumns)
val header = s"Checking $dfName"
val prologue = s"It has a total number of ${dfColumns.size} columns and $dfCount rows."
val check = Check(df, Some(dfName), Option.empty, Seq.empty)
markdownReporter.report(CheckResult(Map.empty, check, dfCount))
val expectedOutput = s"""**$header**
$prologue
Nothing to check!
"""
baos.toString shouldBe expectedOutput
}
}
Example 23
| Source File: ConsoleReporterTest.scala From drunken-data-quality with Apache License 2.0 | 5 votes |
|
package de.frosner.ddq.reporters
import java.io.{ByteArrayOutputStream, PrintStream}
import de.frosner.ddq.constraints._
import de.frosner.ddq.core._
import de.frosner.ddq.testutils.{DummyConstraint, DummyConstraintResult}
import org.apache.spark.sql.DataFrame
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
class ConsoleReporterTest extends FlatSpec with Matchers with MockitoSugar {
"A Console reporter" should "produce correct output for a check with constraints" in {
val baos = new ByteArrayOutputStream()
val consoleReporter = new ConsoleReporter(new PrintStream(baos))
val df = mock[DataFrame]
val displayName = "myDf"
val dfColumns = Array("1", "2")
val dfCount = 5
when(df.columns).thenReturn(dfColumns)
val header = s"Checking $displayName"
val prologue = s"It has a total number of ${dfColumns.size} columns and $dfCount rows."
val message1 = "1"
val status1 = ConstraintSuccess
val constraint1 = DummyConstraint(message1, status1)
val result1 = constraint1.fun(df)
val message2 = "2"
val status2 = ConstraintFailure
val constraint2 = DummyConstraint(message2, status2)
val result2 = constraint2.fun(df)
val message3 = "3"
val status3 = ConstraintError(new IllegalArgumentException())
val constraint3 = DummyConstraint(message3, status3)
val result3 = DummyConstraintResult(constraint3, message3, status3)
val constraints = Map[Constraint, ConstraintResult[Constraint]](
constraint1 -> result1,
constraint2 -> result2,
constraint3 -> result3
)
val check = Check(df, Some(displayName), Option.empty, constraints.keys.toSeq)
consoleReporter.report(CheckResult(constraints, check, dfCount))
val expectedOutput = s"""${Console.BLUE}$header${Console.RESET}
${Console.BLUE}$prologue${Console.RESET}
${Console.GREEN}- ${result1.message}${Console.RESET}
${Console.RED}- ${result2.message}${Console.RESET}
${Console.YELLOW}- ${result3.message}${Console.RESET}
"""
baos.toString shouldBe expectedOutput
}
it should "produce correct output for a check without constraint" in {
val baos = new ByteArrayOutputStream()
val consoleReporter = new ConsoleReporter(new PrintStream(baos))
val df = mock[DataFrame]
val displayName = "myDf"
val dfColumns = Array("1", "2")
val dfCount = 5
when(df.columns).thenReturn(dfColumns)
val header = s"Checking $displayName"
val prologue = s"It has a total number of ${dfColumns.size} columns and $dfCount rows."
val check = Check(df, Some(displayName), Option.empty, Seq.empty)
consoleReporter.report(CheckResult(Map.empty, check, dfCount))
val expectedOutput = s"""${Console.BLUE}$header${Console.RESET}
${Console.BLUE}$prologue${Console.RESET}
${Console.BLUE}Nothing to check!${Console.RESET}
"""
baos.toString shouldBe expectedOutput
}
}
Example 24
| Source File: RunnerTest.scala From drunken-data-quality with Apache License 2.0 | 5 votes |
|
package de.frosner.ddq.core
import de.frosner.ddq.constraints.{ConstraintFailure, ConstraintSuccess}
import de.frosner.ddq.reporters.Reporter
import de.frosner.ddq.testutils.DummyConstraint
import org.apache.spark.sql.DataFrame
import org.apache.spark.storage.StorageLevel
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
class RunnerTest extends FlatSpec with Matchers with MockitoSugar {
"A runner" should "run with multiple checks" in {
val df1 = mock[DataFrame]
val df2 = mock[DataFrame]
val message1 = "1"
val status1 = ConstraintSuccess
val constraint1 = DummyConstraint(message1, status1)
val result1 = constraint1.fun(df1)
val message2 = "2"
val status2 = ConstraintFailure
val constraint2 = DummyConstraint(message2, status2)
val result2 = constraint2.fun(df2)
val check1 = Check(df1, None, None, Seq(constraint1))
val check2 = Check(df2, None, None, Seq(constraint2))
val checkResults = Runner.run(List(check1, check2), List.empty)
checkResults.size shouldBe 2
val checkResult1 = checkResults(check1)
val checkResult2 = checkResults(check2)
checkResult1.check shouldBe check1
checkResult1.constraintResults shouldBe Map((constraint1, result1))
checkResult2.check shouldBe check2
checkResult2.constraintResults shouldBe Map((constraint2, result2))
}
it should "persist and unpersist the data frame if a persist method is specified" in {
val storageLevel = StorageLevel.MEMORY_AND_DISK
val df = mock[DataFrame]
when(df.persist(storageLevel)).thenReturn(df.asInstanceOf[df.type])
val check = Check(df, None, Some(storageLevel), Seq(DummyConstraint("test", ConstraintSuccess)))
val checkResult = Runner.run(List(check), List.empty)(check)
verify(df).persist(storageLevel)
verify(df).unpersist()
}
it should "not persist and unpersist the data frame if no persist method is specified" in {
val df = mock[DataFrame]
val check = Check(df, None, None, Seq(DummyConstraint("test", ConstraintSuccess)))
val checkResult = Runner.run(List(check), List.empty)(check)
verify(df, never()).persist()
verify(df, never()).unpersist()
}
it should "report to all reporters what it returns" in {
val df = mock[DataFrame]
val check = Check(df, None, None, Seq(DummyConstraint("test", ConstraintSuccess)))
val checkResult = Runner.run(List(check), List.empty)(check)
val reporter1 = mock[Reporter]
val reporter2 = mock[Reporter]
Runner.run(List(check), List(reporter1, reporter2))
verify(reporter1).report(checkResult)
verify(reporter2).report(checkResult)
}
}
Example 25
| Source File: KafkaJsonConsumerSpec.scala From coral with Apache License 2.0 | 5 votes |
|
package io.coral.lib
import java.util.Properties
import kafka.consumer._
import kafka.message.MessageAndMetadata
import org.json4s.JsonAST.{JNothing, JValue}
import org.json4s.jackson.JsonMethods._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, WordSpec}
class KafkaJsonConsumerSpec extends WordSpec with Matchers with MockitoSugar {
"KafkaJsonConsumer" should {
"provide a stream" in {
val consumer = KafkaJsonConsumer()
intercept[IllegalArgumentException] {
consumer.stream("abc", new Properties())
}
}
}
"KafkaJsonStream" should {
val fakeConnection = mock[ConsumerConnector]
doNothing.when(fakeConnection).commitOffsets
val fakeMessage = mock[MessageAndMetadata[Array[Byte], JValue]]
when(fakeMessage.key()).thenReturn("TestKey".getBytes)
when(fakeMessage.message()).thenReturn(parse( """{ "json": "test" }"""))
val fakeIterator = mock[ConsumerIterator[Array[Byte], JValue]]
when(fakeIterator.hasNext()).thenReturn(true).thenReturn(false)
when(fakeIterator.next()).thenReturn(fakeMessage)
val fakeStream = mock[KafkaStream[Array[Byte], JValue]]
when(fakeStream.iterator()).thenReturn(fakeIterator)
"provide a next value" in {
val kjs = new KafkaJsonStream(fakeConnection, fakeStream)
kjs.hasNextInTime shouldBe true
kjs.next shouldBe parse( """{ "json": "test" }""")
}
}
"JsonDecoder" should {
"convert bytes to Json object" in {
val jsonString = """{ "hello": "json" }"""
val bytes = jsonString.getBytes
val jsonValue = parse(jsonString)
JsonDecoder.fromBytes(bytes) shouldBe jsonValue
}
"return JNothing for invalid JSon" in {
val jsonString = """hello"""
val bytes = jsonString.getBytes
JsonDecoder.fromBytes(bytes) shouldBe JNothing
}
}
}
Example 26
| Source File: ScalastyleRunnerSpec.scala From sonar-scala with GNU Lesser General Public License v3.0 | 5 votes |
|
package com.ncredinburgh.sonar.scalastyle
import java.io.File
import java.nio.charset.StandardCharsets
import org.mockito.Mockito._
import org.scalastyle._
import org.scalastyle.StyleError
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers, PrivateMethodTester}
import org.sonar.api.profiles.RulesProfile
import org.sonar.api.rules.{Rule, RulePriority}
import scala.collection.JavaConversions._
class ScalastyleRunnerSpec extends FlatSpec with Matchers with MockitoSugar with PrivateMethodTester {
trait Fixture {
val checker1 = ConfigurationChecker("org.scalastyle.scalariform.MultipleStringLiteralsChecker", ErrorLevel, true, Map(), None, None)
val checker2 = ConfigurationChecker("org.scalastyle.file.HeaderMatchesChecker", ErrorLevel, true, Map("header" -> "// Expected Header Comment"), None, None)
val configuration = ScalastyleConfiguration("sonar", true, List(checker1, checker2))
val testeeSpy = spy(new ScalastyleRunner(mock[RulesProfile]))
doReturn(configuration).when(testeeSpy).config
val charset = StandardCharsets.UTF_8.name
}
"a scalastyle runner" should "report StyleError messages if there are rule violations" in new Fixture {
val files = List(new File("src/test/resources/ScalaFile1.scala"))
val messages = testeeSpy.run(charset, files).map(_.toString)
messages should contain ("StyleError key=header.matches args=List() lineNumber=Some(1) column=None customMessage=None")
}
it should "not report StyleError messages if there are no violations" in new Fixture {
val files = List(new File("src/test/resources/ScalaFile2.scala"))
val messages = testeeSpy.run(charset, files)
messages.length shouldEqual 0
}
it should "scan multiple files" in new Fixture {
val files = List(new File("src/test/resources/ScalaFile1.scala"), new File("src/test/resources/ScalaFile2.scala"))
val messages = testeeSpy.run(charset, files)
messages.length shouldEqual 1
}
it should "convert rules to checker" in {
val ruleToChecker = PrivateMethod[ConfigurationChecker]('ruleToChecker)
val profile = RulesProfile.create(Constants.ProfileName, Constants.ScalaKey)
val testee = new ScalastyleRunner(profile)
val key = "multiple.string.literals"
val className = "org.scalastyle.scalariform.MultipleStringLiteralsChecker"
val rule = Rule.create
rule.setRepositoryKey(Constants.RepositoryKey)
.setKey(className)
.setName(ScalastyleResources.label(key))
.setDescription(ScalastyleResources.description(key))
.setConfigKey(key)
.setSeverity(RulePriority.MAJOR)
rule.createParameter
.setKey("allowed")
.setDescription("")
.setType("integer")
.setDefaultValue("1")
rule.createParameter
.setKey("ignoreRegex")
.setDescription("")
.setType("integer")
.setDefaultValue("^""$")
// add synthetic parameter as reference to the class
rule.createParameter
.setKey(Constants.ClazzParam)
.setDescription("Scalastyle checker that validates the rule.")
.setType("string")
.setDefaultValue("org.scalastyle.scalariform.MultipleStringLiteralsChecker")
val activeRule = profile.activateRule(rule, rule.getSeverity)
activeRule.setParameter("allowed", "1")
activeRule.setParameter("ignoreRegex", "^""$")
activeRule.setParameter(Constants.ClazzParam, "org.scalastyle.scalariform.MultipleStringLiteralsChecker")
val checker = testee invokePrivate ruleToChecker(activeRule)
val expectedParameters = Map("allowed" -> "1", "ignoreRegex" -> "^""$", Constants.ClazzParam -> "org.scalastyle.scalariform.MultipleStringLiteralsChecker")
val expectedChecker = ConfigurationChecker(className, ErrorLevel, true, expectedParameters, None, Some(className))
checker shouldEqual expectedChecker
}
}
Example 27
| Source File: ScalastyleAdaptedQualityProfileSpec.scala From sonar-scala with GNU Lesser General Public License v3.0 | 5 votes |
|
package com.ncredinburgh.sonar.scalastyle
import com.ncredinburgh.sonar.scalastyle.testUtils.TestRuleFinderWithTemplates
import org.scalatest._
import org.scalatest.mock.MockitoSugar
import org.sonar.api.profiles.RulesProfile
import org.sonar.api.utils.ValidationMessages
import scala.collection.JavaConversions._
class ScalastyleQualityProfileSpec extends FlatSpec with Matchers with MockitoSugar {
trait Fixture {
val validationMessages = ValidationMessages.create
val testee = new ScalastyleQualityProfile(TestRuleFinderWithTemplates)
}
val rulesCount = 38
val parametersCount = 21
"a scalastyle quality profile" should "create a default profile" in new Fixture {
val rulesProfile = testee.createProfile(validationMessages)
rulesProfile.getClass shouldEqual classOf[RulesProfile]
rulesProfile.getName shouldEqual Constants.ProfileName
rulesProfile.getLanguage shouldEqual Constants.ScalaKey
}
"the default quality profile" should "have all the rules in default config" in new Fixture {
val rulesProfile = testee.createProfile(validationMessages)
rulesProfile.getActiveRules.size shouldBe rulesCount
}
it should "have all the parameters in default config" in new Fixture {
val totalParameters = parametersCount + (rulesCount * 1)
val rulesProfile = testee.createProfile(validationMessages)
rulesProfile.getActiveRules.flatMap(_.getActiveRuleParams).size shouldBe totalParameters
}
it should "have correct values for parameters" in new Fixture {
val ruleKey = "scalastyle_NumberOfMethodsInTypeChecker"
val rulesProfile = testee.createProfile(validationMessages)
val rule = rulesProfile.getActiveRule(Constants.RepositoryKey, ruleKey)
val param = rule.getActiveRuleParams.head
param.getKey shouldBe "maxMethods"
param.getValue shouldBe "30"
}
}
Example 28
| Source File: ScalastyleDefaultQualityProfileSpec.scala From sonar-scala with GNU Lesser General Public License v3.0 | 5 votes |
|
package com.ncredinburgh.sonar.scalastyle
import com.ncredinburgh.sonar.scalastyle.testUtils.TestRuleFinderWithTemplates
import org.scalatest._
import org.scalatest.mock.MockitoSugar
import org.sonar.api.profiles.RulesProfile
import org.sonar.api.utils.ValidationMessages
import scala.collection.JavaConversions._
import com.ncredinburgh.sonar.scalastyle.testUtils.TestRuleFinder
class ScalastyleDefaultQualityProfileSpec extends FlatSpec with Matchers with MockitoSugar {
trait Fixture {
val validationMessages = ValidationMessages.create
val testee = new ScalastyleQualityProfile(TestRuleFinder)
}
val rulesCount = 19 // rules without templates
"a scalastyle quality profile" should "create a default profile" in new Fixture {
val rulesProfile = testee.createProfile(validationMessages)
rulesProfile.getClass shouldEqual classOf[RulesProfile]
rulesProfile.getName shouldEqual Constants.ProfileName
rulesProfile.getLanguage shouldEqual Constants.ScalaKey
}
"the default quality profile" should "have all the rules in default config" in new Fixture {
val rulesProfile = testee.createProfile(validationMessages)
rulesProfile.getActiveRules.size shouldBe rulesCount
}
it should "have all the parameters in default config" in new Fixture {
val totalParameters = (rulesCount * 1)
val rulesProfile = testee.createProfile(validationMessages)
rulesProfile.getActiveRules.flatMap(_.getActiveRuleParams).size shouldBe totalParameters
}
}
Example 29
| Source File: CacheManagerSuite.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import org.mockito.Mockito._
import org.scalatest.BeforeAndAfter
import org.scalatest.mock.MockitoSugar
import org.apache.spark.executor.{DataReadMethod, TaskMetrics}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage._
// TODO: Test the CacheManager's thread-safety aspects
class CacheManagerSuite extends SparkFunSuite with LocalSparkContext with BeforeAndAfter
with MockitoSugar {
var blockManager: BlockManager = _
var cacheManager: CacheManager = _
var split: Partition = _
var rdd: RDD[Int] = _
var rdd2: RDD[Int] = _
var rdd3: RDD[Int] = _
before {
sc = new SparkContext("local", "test")
blockManager = mock[BlockManager]
cacheManager = new CacheManager(blockManager)
split = new Partition { override def index: Int = 0 }
rdd = new RDD[Int](sc, Nil) {
override def getPartitions: Array[Partition] = Array(split)
override val getDependencies = List[Dependency[_]]()
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
Array(1, 2, 3, 4).iterator
}
rdd2 = new RDD[Int](sc, List(new OneToOneDependency(rdd))) {
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
firstParent[Int].iterator(split, context)
}.cache()
rdd3 = new RDD[Int](sc, List(new OneToOneDependency(rdd2))) {
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
firstParent[Int].iterator(split, context)
}.cache()
}
test("get uncached rdd") {
// Do not mock this test, because attempting to match Array[Any], which is not covariant,
// in blockManager.put is a losing battle. You have been warned.
blockManager = sc.env.blockManager
cacheManager = sc.env.cacheManager
val context = TaskContext.empty()
val computeValue = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
val getValue = blockManager.get(RDDBlockId(rdd.id, split.index))
assert(computeValue.toList === List(1, 2, 3, 4))
assert(getValue.isDefined, "Block cached from getOrCompute is not found!")
assert(getValue.get.data.toList === List(1, 2, 3, 4))
}
test("get cached rdd") {
val result = new BlockResult(Array(5, 6, 7).iterator, DataReadMethod.Memory, 12)
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(Some(result))
val context = TaskContext.empty()
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(5, 6, 7))
}
test("get uncached local rdd") {
// Local computation should not persist the resulting value, so don't expect a put().
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(None)
val context = new TaskContextImpl(0, 0, 0, 0, null, null, Seq.empty, runningLocally = true)
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(1, 2, 3, 4))
}
test("verify task metrics updated correctly") {
cacheManager = sc.env.cacheManager
val context = TaskContext.empty()
cacheManager.getOrCompute(rdd3, split, context, StorageLevel.MEMORY_ONLY)
assert(context.taskMetrics.updatedBlocks.getOrElse(Seq()).size === 2)
}
}
Example 30
| Source File: KyuubiSessionPageSuite.scala From kyuubi with Apache License 2.0 | 5 votes |
|
package org.apache.spark.ui
import javax.servlet.http.HttpServletRequest
import scala.util.Try
import org.apache.spark.{KyuubiSparkUtil, SparkConf, SparkContext, SparkFunSuite}
import org.scalatest.mock.MockitoSugar
class KyuubiSessionPageSuite extends SparkFunSuite with MockitoSugar {
var sc: SparkContext = _
var user: String = _
var tab: KyuubiSessionTab = _
override def beforeAll(): Unit = {
val conf = new SparkConf(loadDefaults = true).setMaster("local").setAppName("test")
sc = new SparkContext(conf)
user = KyuubiSparkUtil.getCurrentUserName
tab = new KyuubiSessionTab(user, sc)
}
override def afterAll(): Unit = {
sc.stop()
}
test("render kyuubi session page") {
val page = new KyuubiSessionPage(tab)
val request = mock[HttpServletRequest]
assert(Try { page.render(request) }.isSuccess )
}
}
Example 31
| Source File: CacheManagerSuite.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import org.mockito.Mockito._
import org.scalatest.BeforeAndAfter
import org.scalatest.mock.MockitoSugar
import org.apache.spark.executor.{DataReadMethod, TaskMetrics}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage._
// TODO: Test the CacheManager's thread-safety aspects(测试线程安全方面)
class CacheManagerSuite extends SparkFunSuite with LocalSparkContext with BeforeAndAfter
with MockitoSugar {
var blockManager: BlockManager = _
var cacheManager: CacheManager = _
var split: Partition = _
var rdd: RDD[Int] = _
var rdd2: RDD[Int] = _
var rdd3: RDD[Int] = _
before {
sc = new SparkContext("local", "test")
blockManager = mock[BlockManager]//模拟BlockManager
//引用BlockManager
cacheManager = new CacheManager(blockManager)
split = new Partition { override def index: Int = 0 }
rdd = new RDD[Int](sc, Nil) {
override def getPartitions: Array[Partition] = Array(split)
override val getDependencies = List[Dependency[_]]()//获得依赖关系
override def compute(split: Partition, context: TaskContext): Iterator[Int] ={
//println(split.index+"=="+context.taskMetrics().hostname);
Array(1, 2, 3, 4).iterator//计算
}
}
rdd2 = new RDD[Int](sc, List(new OneToOneDependency(rdd))) {//依赖RDD
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
firstParent[Int].iterator(split, context)
}.cache()//缓存
rdd3 = new RDD[Int](sc, List(new OneToOneDependency(rdd2))) {//依赖RDD1
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
firstParent[Int].iterator(split, context)
}.cache()//缓存
}
test("get uncached rdd") {//得到未缓存的RDD
// Do not mock this test, because attempting to match Array[Any], which is not covariant,
// in blockManager.put is a losing battle(可能失败). You have been warned.
//不要模拟这个测试,因为试图匹配数组[任何],这不是协变的,blockManager插入可能失败,你被警告了
blockManager = sc.env.blockManager
cacheManager = sc.env.cacheManager
val context = TaskContext.empty()
val computeValue = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
val getValue = blockManager.get(RDDBlockId(rdd.id, split.index))
assert(computeValue.toList === List(1, 2, 3, 4))//获得计算值
//getValue BlockResult
//如果false,则块缓存从getorcompute没有被发现
assert(getValue.isDefined, "Block cached from getOrCompute is not found!")
assert(getValue.get.data.toList === List(1, 2, 3, 4))
}
test("get cached rdd") {//得到缓存的RDD
val result = new BlockResult(Array(5, 6, 7).iterator, DataReadMethod.Memory, 12)
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(Some(result))//然后返回
val context = TaskContext.empty()
val getValue = blockManager.get(RDDBlockId(rdd.id, split.index))
println(split.index+"==rddId=="+rdd.id+"==="+getValue.get)
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(5, 6, 7))
}
test("get uncached local rdd") {//得到未被缓存的本地RDD
// Local computation should not persist the resulting value, so don't expect a put().
//本地计算产生的值不持久化,所以不期望一个插入
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(None)//然后返回
val context = new TaskContextImpl(0, 0, 0, 0, null, null, Seq.empty, runningLocally = true)
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(1, 2, 3, 4))
}
test("verify task metrics updated correctly") {//验证任务度量的正确更新
cacheManager = sc.env.cacheManager
val context = TaskContext.empty()
cacheManager.getOrCompute(rdd3, split, context, StorageLevel.MEMORY_ONLY)
assert(context.taskMetrics.updatedBlocks.getOrElse(Seq()).size === 2)
}
}
Example 32
| Source File: MesosClusterSchedulerSuite.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.scheduler.mesos
import java.util.Date
import org.scalatest.mock.MockitoSugar
import org.apache.spark.deploy.Command
import org.apache.spark.deploy.mesos.MesosDriverDescription
import org.apache.spark.scheduler.cluster.mesos._
import org.apache.spark.{LocalSparkContext, SparkConf, SparkFunSuite}
class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext with MockitoSugar {
private val command = new Command("mainClass", Seq("arg"), null, null, null, null)
test("can queue drivers") {//可以排队的Drive
val conf = new SparkConf()
conf.setMaster("mesos://localhost:5050")
conf.setAppName("spark mesos")
val scheduler = new MesosClusterScheduler(
new BlackHoleMesosClusterPersistenceEngineFactory, conf) {
override def start(): Unit = { ready = true }
}
scheduler.start()
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", 1000, 1, true,
command, Map[String, String](), "s1", new Date()))
assert(response.success)
val response2 =
scheduler.submitDriver(new MesosDriverDescription(
"d1", "jar", 1000, 1, true, command, Map[String, String](), "s2", new Date()))
assert(response2.success)
val state = scheduler.getSchedulerState()
val queuedDrivers = state.queuedDrivers.toList
assert(queuedDrivers(0).submissionId == response.submissionId)
assert(queuedDrivers(1).submissionId == response2.submissionId)
}
test("can kill queued drivers") {//可以杀死排队的驱动程序
val conf = new SparkConf()
conf.setMaster("mesos://localhost:5050")
conf.setAppName("spark mesos")
val scheduler = new MesosClusterScheduler(
new BlackHoleMesosClusterPersistenceEngineFactory, conf) {
override def start(): Unit = { ready = true }
}
scheduler.start()
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", 1000, 1, true,
command, Map[String, String](), "s1", new Date()))
assert(response.success)
val killResponse = scheduler.killDriver(response.submissionId)
assert(killResponse.success)
val state = scheduler.getSchedulerState()
assert(state.queuedDrivers.isEmpty)
}
}
Example 33
| Source File: UtilsTest.scala From sbt-dynamodb with MIT License | 5 votes |
|
package com.teambytes.sbt.dynamodb
import org.scalatest.mock.MockitoSugar
import org.scalatest.{MustMatchers, WordSpec}
class UtilsTest extends WordSpec with MustMatchers with MockitoSugar {
"Utils" should {
"extract PID correctly" in {
val jpsOutput =
"""
|16706 QuorumPeerMain
|60405 Boot
|59022 DynamoDBLocal.jar
|60479 Jps
|51449
""".stripMargin
Utils.extractDyanmoDBPid(jpsOutput) must equal(Some("59022"))
}
}
}
Example 34
| Source File: RecordIOOutputFormatTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package com.amazonaws.services.sagemaker.sparksdk.protobuf
import java.io.ByteArrayOutputStream
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path}
import org.apache.hadoop.io.{BytesWritable, NullWritable}
import org.apache.hadoop.mapreduce.TaskAttemptContext
import org.mockito.Matchers.any
import org.mockito.Mockito.{verify, when}
import org.scalatest.{BeforeAndAfter, FlatSpec}
import org.scalatest.mock.MockitoSugar
import com.amazonaws.services.sagemaker.sparksdk.protobuf.RecordIOOutputFormat.SageMakerProtobufRecordWriter
class RecordIOOutputFormatTests extends FlatSpec with MockitoSugar with BeforeAndAfter {
var sagemakerProtobufRecordWriter: SageMakerProtobufRecordWriter = _
var mockOutputStream : FSDataOutputStream = _
var byteArrayOutputStream: ByteArrayOutputStream = _
var mockTaskAttemptContext: TaskAttemptContext = _
var mockPath: Path = _
var mockFileSystem: FileSystem = _
before {
byteArrayOutputStream = new ByteArrayOutputStream()
mockOutputStream = mock[FSDataOutputStream]
sagemakerProtobufRecordWriter = new SageMakerProtobufRecordWriter(mockOutputStream)
mockTaskAttemptContext = mock[TaskAttemptContext]
mockPath = mock[Path]
mockFileSystem = mock[FileSystem]
}
it should "write an empty array of bytes" in {
val bytesWritable = new BytesWritable(byteArrayOutputStream.toByteArray)
val bytes = ProtobufConverter.byteArrayToRecordIOEncodedByteArray(bytesWritable.getBytes)
sagemakerProtobufRecordWriter.write(NullWritable.get(), bytesWritable)
verify(mockOutputStream).write(bytes, 0, bytes.length)
}
it should "write an array of bytes" in {
val byteArray = Array[Byte](0, 0, 0, 0)
byteArrayOutputStream.write(byteArray)
val bytesWritable = new BytesWritable(byteArrayOutputStream.toByteArray)
val bytes = ProtobufConverter.byteArrayToRecordIOEncodedByteArray(bytesWritable.getBytes)
sagemakerProtobufRecordWriter.write(NullWritable.get(), bytesWritable)
verify(mockOutputStream).write(bytes, 0, bytes.length)
}
it should "write an array of bytes, padding as necessary" in {
byteArrayOutputStream.write(5)
val bytesWritable = new BytesWritable(byteArrayOutputStream.toByteArray)
val bytes = ProtobufConverter.byteArrayToRecordIOEncodedByteArray(bytesWritable.getBytes)
sagemakerProtobufRecordWriter.write(NullWritable.get(), bytesWritable)
verify(mockOutputStream).write(bytes, 0, bytes.length)
}
it should "write an array of bytes, padding only as much as necessary" in {
byteArrayOutputStream.write(Array[Byte](0, 0, 0, 0, 0))
val bytesWritable = new BytesWritable(byteArrayOutputStream.toByteArray)
val bytes = ProtobufConverter.byteArrayToRecordIOEncodedByteArray(bytesWritable.getBytes)
sagemakerProtobufRecordWriter.write(NullWritable.get(), bytesWritable)
verify(mockOutputStream).write(bytes, 0, bytes.length)
}
it should "create a record writer from a FSDataOutputStream created by the filesystem" in {
val mockTaskAttemptContext = mock[TaskAttemptContext]
val mockPath = mock[Path]
val mockFileSystem = mock[FileSystem]
when(mockPath.getFileSystem(any[Configuration])).thenReturn(mockFileSystem)
new RecordIOOutputFormat() {
override def getDefaultWorkFile(context: TaskAttemptContext, extension: String): Path = {
mockPath
}
}.getRecordWriter(mockTaskAttemptContext)
verify(mockFileSystem).create(mockPath, true)
}
}
Example 35
| Source File: NamePolicyTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package com.amazonaws.services.sagemaker.sparksdk;
import org.scalatest.FlatSpec
import org.scalatest.Matchers
import org.scalatest.mock.MockitoSugar
class NamePolicyTests extends FlatSpec with Matchers with MockitoSugar {
"RandomNamePolicy" should "have correct names" in {
val policy = RandomNamePolicy()
assert(policy.trainingJobName.contains("trainingJob"))
assert(policy.modelName.contains("model"))
assert(policy.endpointConfigName.contains("endpointConfig"))
assert(policy.endpointName.contains("endpoint"))
}
"RandomNamePolicy" should "have correct prefix" in {
val policy = RandomNamePolicy("prefix")
assert(policy.trainingJobName.startsWith("prefix"))
assert(policy.modelName.startsWith("prefix"))
assert(policy.endpointConfigName.startsWith("prefix"))
assert(policy.endpointName.startsWith("prefix"))
}
"RandomNamePolicyFactory" should "return a random name policy with correct prefixes" in {
val factory = new RandomNamePolicyFactory("prefix")
val policy = factory.createNamePolicy
assert(policy.isInstanceOf[RandomNamePolicy])
assert(policy.trainingJobName.startsWith("prefix"))
assert(policy.modelName.startsWith("prefix"))
assert(policy.endpointConfigName.startsWith("prefix"))
assert(policy.endpointName.startsWith("prefix"))
}
"CustomNamePolicy" should "have custom names" in {
val policy = new CustomNamePolicy("jobName", "modelName", "endpointConfig", "endpointName")
assert(policy.trainingJobName.equals("jobName"))
assert(policy.modelName.equals("modelName"))
assert(policy.endpointConfigName.equals("endpointConfig"))
assert(policy.endpointName.startsWith("endpointName"))
}
"CustomNamePolicyWithTimeStampSuffix" should "have custom names with proper suffix" in {
val policy = new CustomNamePolicyWithTimeStampSuffix("jobName", "modelName",
"endpointConfig", "endpointName")
assert(!policy.trainingJobName.equals("jobName"))
assert(!policy.modelName.equals("modelName"))
assert(!policy.endpointConfigName.equals("endpointConfig"))
assert(!policy.endpointName.equals("endpointName"))
assert(policy.trainingJobName.startsWith("jobName"))
assert(policy.modelName.startsWith("modelName"))
assert(policy.endpointConfigName.startsWith("endpointConfig"))
assert(policy.endpointName.startsWith("endpointName"))
}
"CustomNamePolicyFactory" should "return a custom name policy with correct params" in {
val jobName = "jobName"
val modelName = "modelName"
val endpointConfigName = "endpointConfigName"
val endpointName = "endpointName"
val factory = new CustomNamePolicyFactory(jobName, modelName, endpointConfigName, endpointName)
val policy = factory.createNamePolicy
assert(policy.isInstanceOf[CustomNamePolicy])
assert(policy.trainingJobName.equals(jobName))
assert(policy.modelName.equals(modelName))
assert(policy.endpointConfigName.equals(endpointConfigName))
assert(policy.endpointName.equals(endpointName))
}
"CustomNamePolicyWithTimeStampSuffixFactory" should
"return a custom name policy with timestamp suffix" in {
val jobName = "jobName"
val modelName = "modelName"
val endpointConfigName = "endpointConfigName"
val endpointName = "endpointName"
val factory = new CustomNamePolicyWithTimeStampSuffixFactory(jobName, modelName,
endpointConfigName, endpointName)
val policy = factory.createNamePolicy
assert(policy.isInstanceOf[CustomNamePolicyWithTimeStampSuffix])
assert(!policy.trainingJobName.equals(jobName))
assert(!policy.modelName.equals(modelName))
assert(!policy.endpointConfigName.equals(endpointConfigName))
assert(!policy.endpointName.equals(endpointName))
assert(policy.trainingJobName.startsWith(jobName))
assert(policy.modelName.startsWith(modelName))
assert(policy.endpointConfigName.startsWith(endpointConfigName))
assert(policy.endpointName.startsWith(endpointName))
}
}
Example 36
| Source File: LibSVMTransformationLocalFunctionalTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package com.amazonaws.services.sagemaker.sparksdk.transformation
import java.io.{File, FileWriter}
import collection.JavaConverters._
import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers}
import org.scalatest.mock.MockitoSugar
import org.apache.spark.sql.SparkSession
import com.amazonaws.services.sagemaker.sparksdk.transformation.deserializers.LibSVMResponseRowDeserializer
import com.amazonaws.services.sagemaker.sparksdk.transformation.serializers.LibSVMRequestRowSerializer
class LibSVMTransformationLocalFunctionalTests extends FlatSpec with Matchers with MockitoSugar
with BeforeAndAfter {
val spark = SparkSession.builder
.master("local")
.appName("spark session")
.getOrCreate()
var libsvmDataFile : File = _
val libsvmdata =
"1.0 1:1.5 2:3.0 28:-39.935 55:0.01\n" +
"0.0 2:3.0 28:-39.935 55:0.01\n" +
"-1.0 23:-39.935 55:0.01\n" +
"3.0 1:1.5 2:3.0"
before {
libsvmDataFile = File.createTempFile("temp", "temp")
val fw = new FileWriter(libsvmDataFile)
fw.write(libsvmdata)
fw.close()
}
"LibSVMSerialization" should "serialize Spark loaded libsvm file to same contents" in {
import spark.implicits._
val df = spark.read.format("libsvm").load(libsvmDataFile.getPath)
val libsvmSerializer = new LibSVMRequestRowSerializer(Some(df.schema))
val result = df.map(row => new String(libsvmSerializer.serializeRow(row))).collect().mkString
assert (libsvmdata.trim == result.trim)
}
"LibSVMDeserialization" should "deserialize serialized lib svm records" in {
val libsvmdata =
"1.0 1:1.5 2:3.0 28:-39.935 55:0.01\n" +
"0.0 2:3.0 28:-39.935 55:0.01\n" +
"-1.0 23:-39.935 55:0.01\n" +
"3.0 1:1.5 2:3.0"
val libsvmDeserializer = new LibSVMResponseRowDeserializer (55)
val rowList = libsvmDeserializer.deserializeResponse(libsvmdata.getBytes).toBuffer.asJava
val deserializedDataFrame = spark.createDataFrame(rowList, libsvmDeserializer.schema)
val sparkProducedDataFrame = spark.read.format("libsvm").load(libsvmDataFile.getPath)
val deserializedRows = deserializedDataFrame.collectAsList()
val sparkRows = sparkProducedDataFrame.collectAsList()
assert (deserializedRows == sparkRows)
}
}
Example 37
| Source File: LibSVMResponseRowDeserializerTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package com.amazonaws.services.sagemaker.sparksdk.transformation.deserializers
import org.scalatest._
import org.scalatest.mock.MockitoSugar
import scala.collection.mutable.ListBuffer
import org.apache.spark.ml.linalg.SparseVector
import org.apache.spark.sql._
class LibSVMResponseRowDeserializerTests extends FlatSpec with Matchers with MockitoSugar {
"LibSVMResponseRowDeserializer" should "deserialize a single record with a two features" in {
val rrd = new LibSVMResponseRowDeserializer(3)
val responseIterator =
rrd.deserializeResponse(createLibSVMRecord(1, Array(1, 2), Array(1.0, 2.0)).getBytes)
assert(responseIterator.next == Row(1, new SparseVector(3, Array(1, 2), Array(1.0, 2.0))))
}
it should "deserialize a single record with no values" in {
val rrd = new LibSVMResponseRowDeserializer(0)
val responseIterator = rrd.deserializeResponse(
createLibSVMRecord(1, Seq[Int]().toArray, Seq[Double]().toArray).getBytes)
assert(responseIterator.next ==
Row(1, new SparseVector(0, Seq[Int]().toArray, Seq[Double]().toArray)))
}
it should "deserialize multiple records with multiple features" in {
val dim = 100
val rrd = new LibSVMResponseRowDeserializer(dim)
val sb = new StringBuilder
val rows = new ListBuffer[Row]
for (i <- Range(0, dim)) {
val label = i.asInstanceOf[Double]
val indices = Range (0, i)
val values = Range(0, i) map( a => (a - 10) * a) map (a => a.asInstanceOf[Double])
sb ++= createLibSVMRecord(label, indices.toArray, values.toArray)
rows += Row(label, new SparseVector(dim, indices.toArray, values.toArray))
sb ++= "\n"
}
assert(List() ++ rrd.deserializeResponse(sb.mkString.getBytes) == rows.toList)
}
it should "throw on invalid dimension" in {
intercept[IllegalArgumentException] {
new LibSVMResponseRowDeserializer(-1)
}
}
it should "fail on invalid label" in {
val rrd = new LibSVMResponseRowDeserializer(3)
intercept[RuntimeException] {
val responseIterator = rrd.deserializeResponse("XXX 1:1".getBytes)
}
}
it should "fail on invalid value" in {
val rrd = new LibSVMResponseRowDeserializer(3)
intercept[RuntimeException] {
rrd.deserializeResponse("1.0 1:Elizabeth".getBytes)
}
}
it should "fail on invalid index" in {
val rrd = new LibSVMResponseRowDeserializer(3)
intercept[RuntimeException] {
rrd.deserializeResponse("1.0 BLAH:1.3421".getBytes)
}
}
it should "fail on missing index" in {
val rrd = new LibSVMResponseRowDeserializer(3)
intercept[RuntimeException] {
rrd.deserializeResponse("1.0 :1.3421".getBytes)
}
}
it should "fail on missing value" in {
val rrd = new LibSVMResponseRowDeserializer(3)
intercept[RuntimeException] {
rrd.deserializeResponse("1.0 1:".getBytes)
}
}
it should "fail on index out of bounds" in {
val rrd = new LibSVMResponseRowDeserializer(2)
intercept[RuntimeException] {
rrd.deserializeResponse("1.0 3:2.0".getBytes)
}
}
private def createLibSVMRecord(label : Double, indices : Array[Int], values : Array[Double])
: String = {
val sb = new StringBuilder(label.toString)
val x = indices zip values
for((index, value) <- x) {
sb ++= s" ${index + 1}:$value"
}
sb.mkString
}
}
Example 38
| Source File: ProtobufRequestRowSerializerTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package com.amazonaws.services.sagemaker.sparksdk.transformation.serializers
import org.scalatest.{FlatSpec, Matchers}
import org.scalatest.mock.MockitoSugar
import org.apache.spark.ml.linalg.{DenseVector, SparseVector, SQLDataTypes}
import org.apache.spark.ml.linalg.SQLDataTypes.VectorType
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import org.apache.spark.sql.types.{DoubleType, StringType, StructField, StructType}
import com.amazonaws.services.sagemaker.sparksdk.protobuf.ProtobufConverter
class ProtobufRequestRowSerializerTests extends FlatSpec with Matchers with MockitoSugar {
val labelColumnName = "label"
val featuresColumnName = "features"
val schema = StructType(Array(StructField(labelColumnName, DoubleType), StructField(
featuresColumnName, VectorType)))
it should "serialize a dense vector" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
val row = new GenericRowWithSchema(values = Seq(1.0, vec).toArray, schema = schema)
val rrs = new ProtobufRequestRowSerializer(Some(schema))
val protobuf = ProtobufConverter.rowToProtobuf(row, featuresColumnName, Option.empty)
val serialized = rrs.serializeRow(row)
val protobufIterator = ProtobufConverter.recordIOByteArrayToProtobufs(serialized)
val protobufFromRecordIO = protobufIterator.next
assert(!protobufIterator.hasNext)
assert(protobuf.equals(protobufFromRecordIO))
}
it should "serialize a sparse vector" in {
val vec = new SparseVector(100, Seq[Int](0, 10).toArray, Seq[Double](-100.0, 100.1).toArray)
val row = new GenericRowWithSchema(values = Seq(1.0, vec).toArray, schema = schema)
val rrs = new ProtobufRequestRowSerializer(Some(schema))
val protobuf = ProtobufConverter.rowToProtobuf(row, featuresColumnName, Option.empty)
val serialized = rrs.serializeRow(row)
val protobufIterator = ProtobufConverter.recordIOByteArrayToProtobufs(serialized)
val protobufFromRecordIO = protobufIterator.next
assert(!protobufIterator.hasNext)
assert(protobuf.equals(protobufFromRecordIO))
}
it should "fail to set schema on invalid features name" in {
val vec = new SparseVector(100, Seq[Int](0, 10).toArray, Seq[Double](-100.0, 100.1).toArray)
val row = new GenericRowWithSchema(values = Seq(1.0, vec).toArray, schema = schema)
intercept[IllegalArgumentException] {
val rrs = new ProtobufRequestRowSerializer(Some(schema), featuresColumnName = "doesNotExist")
}
}
it should "fail on invalid types" in {
val schemaWithInvalidFeaturesType = StructType(Array(
StructField("label", DoubleType, nullable = false),
StructField("features", StringType, nullable = false)))
intercept[RuntimeException] {
new ProtobufRequestRowSerializer(Some(schemaWithInvalidFeaturesType))
}
}
it should "validate correct schema" in {
val validSchema = StructType(Array(
StructField("features", SQLDataTypes.VectorType, nullable = false)))
new ProtobufRequestRowSerializer(Some(validSchema))
}
}
Example 39
| Source File: UnlabeledLibSVMRequestRowSerializerTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package com.amazonaws.services.sagemaker.sparksdk.transformation.serializers
import org.scalatest.{FlatSpec, Matchers}
import org.scalatest.mock.MockitoSugar
import org.apache.spark.ml.linalg.{DenseVector, SparseVector, SQLDataTypes}
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import org.apache.spark.sql.types.{StringType, StructField, StructType}
class UnlabeledLibSVMRequestRowSerializerTests extends FlatSpec with Matchers with MockitoSugar {
val schema = StructType(Array(StructField("features", SQLDataTypes.VectorType, nullable = false)))
"UnlabeledLibSVMRequestRowSerializer" should "serialize sparse vector" in {
val vec = new SparseVector(100, Seq[Int](0, 10).toArray, Seq[Double](-100.0, 100.1).toArray)
val row = new GenericRowWithSchema(values = Seq(vec).toArray, schema = schema)
val rrs = new UnlabeledLibSVMRequestRowSerializer()
val serialized = new String(rrs.serializeRow(row))
assert ("0.0 1:-100.0 11:100.1\n" == serialized)
}
it should "serialize dense vector" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
val row = new GenericRowWithSchema(values = Seq(vec).toArray, schema = schema)
val rrs = new UnlabeledLibSVMRequestRowSerializer()
val serialized = new String(rrs.serializeRow(row))
assert("0.0 1:10.0 2:-100.0 3:2.0\n" == serialized)
}
it should "fail on invalid features column name" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
val row = new GenericRowWithSchema(values = Seq(1.0, vec).toArray, schema = schema)
val rrs =
new UnlabeledLibSVMRequestRowSerializer(featuresColumnName = "mangoes are not features")
intercept[RuntimeException] {
rrs.serializeRow(row)
}
}
it should "fail on invalid features type" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
val row =
new GenericRowWithSchema(values = Seq(1.0, "FEATURESSSSSZ!1!").toArray, schema = schema)
val rrs = new UnlabeledLibSVMRequestRowSerializer()
intercept[RuntimeException] {
rrs.serializeRow(row)
}
}
it should "validate correct schema" in {
val validSchema = StructType(Array(
StructField("features", SQLDataTypes.VectorType, nullable = false)))
val rrs = new UnlabeledLibSVMRequestRowSerializer(Some(validSchema))
}
it should "fail to validate incorrect schema" in {
val invalidSchema = StructType(Array(
StructField("features", StringType, nullable = false)))
intercept[IllegalArgumentException] {
new UnlabeledLibSVMRequestRowSerializer(Some(invalidSchema))
}
}
}
Example 40
| Source File: LibSVMRequestRowSerializerTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package com.amazonaws.services.sagemaker.sparksdk.transformation.serializers
import org.scalatest._
import org.scalatest.{FlatSpec, Matchers}
import org.scalatest.mock.MockitoSugar
import org.apache.spark.ml.linalg.{DenseVector, SparseVector, SQLDataTypes}
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import org.apache.spark.sql.types.{DoubleType, StringType, StructField, StructType}
import com.amazonaws.services.sagemaker.sparksdk.transformation.deserializers.LibSVMResponseRowDeserializer
class LibSVMRequestRowSerializerTests extends FlatSpec with Matchers with MockitoSugar {
val schema = new LibSVMResponseRowDeserializer(10).schema
"LibSVMRequestRowSerializer" should "serialize sparse vector" in {
val vec = new SparseVector(100, Seq[Int](0, 10).toArray, Seq[Double](-100.0, 100.1).toArray)
val row = new GenericRowWithSchema(values = Seq(1.0, vec).toArray, schema = schema)
val rrs = new LibSVMRequestRowSerializer(Some(schema))
val serialized = new String(rrs.serializeRow(row))
assert ("1.0 1:-100.0 11:100.1\n" == serialized)
}
it should "serialize dense vector" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
val row = new GenericRowWithSchema(values = Seq(1.0, vec).toArray, schema = schema)
val rrs = new LibSVMRequestRowSerializer(Some(schema))
val serialized = new String(rrs.serializeRow(row))
assert("1.0 1:10.0 2:-100.0 3:2.0\n" == serialized)
}
it should "ignore other columns" in {
val schemaWithExtraColumns = StructType(Array(
StructField("name", StringType, nullable = false),
StructField("label", DoubleType, nullable = false),
StructField("features", SQLDataTypes.VectorType, nullable = false),
StructField("favorite activity", StringType, nullable = false)))
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
val row = new GenericRowWithSchema(values = Seq("Elizabeth", 1.0, vec, "Crying").toArray,
schema = schemaWithExtraColumns)
val rrs = new LibSVMRequestRowSerializer(Some(schemaWithExtraColumns))
val serialized = new String(rrs.serializeRow(row))
assert("1.0 1:10.0 2:-100.0 3:2.0\n" == serialized)
}
it should "fail on invalid features column name" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
intercept[RuntimeException] {
new LibSVMRequestRowSerializer(Some(schema), featuresColumnName = "i do not exist dear sir!")
}
}
it should "fail on invalid label column name" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
intercept[RuntimeException] {
new LibSVMRequestRowSerializer(Some(schema),
labelColumnName = "Sir! I must protest! I do not exist!")
}
}
it should "fail on invalid types" in {
val schemaWithInvalidLabelType = StructType(Array(
StructField("label", StringType, nullable = false),
StructField("features", SQLDataTypes.VectorType, nullable = false)))
intercept[RuntimeException] {
new LibSVMRequestRowSerializer(Some(schemaWithInvalidLabelType))
}
val schemaWithInvalidFeaturesType = StructType(Array(
StructField("label", DoubleType, nullable = false),
StructField("features", StringType, nullable = false)))
intercept[RuntimeException] {
new LibSVMRequestRowSerializer(Some(schemaWithInvalidFeaturesType))
}
}
it should "validate correct schema" in {
val validSchema = StructType(Array(
StructField("label", DoubleType, nullable = false),
StructField("features", SQLDataTypes.VectorType, nullable = false)))
new LibSVMRequestRowSerializer(Some(validSchema))
}
}
Example 41
| Source File: UnlabeledCSVRequestRowSerializerTests.scala From sagemaker-spark with Apache License 2.0 | 5 votes |
|
package unit.com.amazonaws.services.sagemaker.sparksdk.transformation.serializers
import org.scalatest.{FlatSpec, Matchers}
import org.scalatest.mock.MockitoSugar
import org.apache.spark.ml.linalg.{DenseVector, SparseVector, SQLDataTypes}
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import org.apache.spark.sql.types.{StructField, StructType}
import com.amazonaws.services.sagemaker.sparksdk.transformation.serializers.UnlabeledCSVRequestRowSerializer
class UnlabeledCSVRequestRowSerializerTests extends FlatSpec with Matchers with MockitoSugar {
val schema: StructType =
StructType(Array(StructField("features", SQLDataTypes.VectorType, nullable = false)))
it should "serialize sparse vector" in {
val vec = new SparseVector(100, Seq[Int](0, 10).toArray, Seq[Double](-100.0, 100.1).toArray)
val row = new GenericRowWithSchema(values = Seq(vec).toArray, schema = schema)
val rrs = new UnlabeledCSVRequestRowSerializer(Some(schema))
val serialized = new String(rrs.serializeRow(row))
val sparseString = "-100.0," + "0.0," * 9 + "100.1," + "0.0," * 88 + "0.0\n"
assert (sparseString == serialized)
}
it should "serialize dense vector" in {
val vec = new DenseVector(Seq(10.0, -100.0, 2.0).toArray)
val row = new GenericRowWithSchema(values = Seq(vec).toArray, schema = schema)
val rrs = new UnlabeledCSVRequestRowSerializer(Some(schema))
val serialized = new String(rrs.serializeRow(row))
assert("10.0,-100.0,2.0\n" == serialized)
}
}
Example 42
| Source File: InMemoryShellNotificationManagerTest.scala From shellbase with Apache License 2.0 | 5 votes |
|
package com.sumologic.shellbase.notifications
import com.sumologic.shellbase.CommonWordSpec
import org.junit.runner.RunWith
import org.mockito.Mockito._
import org.scalatest.BeforeAndAfterEach
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
@RunWith(classOf[JUnitRunner])
class InMemoryShellNotificationManagerTest extends CommonWordSpec with BeforeAndAfterEach with MockitoSugar {
"InMemoryShellNotificationManager" should {
"provide notification names" in {
val sut = new InMemoryShellNotificationManager("", Seq(notification1, notification2))
sut.notifierNames should be(Seq(firstName, secondName))
}
"know if a notification is enabled by default" in {
val sut = new InMemoryShellNotificationManager("", Seq(notification1, notification2), enabledByDefault = false)
sut.notificationEnabled(firstName) should be(false)
sut.notificationEnabled(secondName) should be(false)
sut.notificationEnabled("madeUp") should be(false)
val sut2 = new InMemoryShellNotificationManager("", Seq(notification1, notification2), enabledByDefault = true)
sut2.notificationEnabled(firstName) should be(true)
sut2.notificationEnabled(secondName) should be(true)
sut2.notificationEnabled("madeUp") should be(true)
}
"support enabling and disabling notifications" in {
val sut = new InMemoryShellNotificationManager("", Seq(notification1, notification2))
sut.notificationEnabled(firstName) should be(false)
sut.notificationEnabled(secondName) should be(false)
sut.enable(firstName)
sut.notificationEnabled(firstName) should be(true)
sut.notificationEnabled(secondName) should be(false)
sut.enable(secondName)
sut.notificationEnabled(firstName) should be(true)
sut.notificationEnabled(secondName) should be(true)
sut.disable(firstName)
sut.notificationEnabled(firstName) should be(false)
sut.notificationEnabled(secondName) should be(true)
sut.disable(secondName)
sut.notificationEnabled(firstName) should be(false)
sut.notificationEnabled(secondName) should be(false)
}
"only notify enabled notifications" in {
val notificationString = "test"
val sut = new InMemoryShellNotificationManager("", Seq(notification1, notification2))
sut.notify(notificationString)
verify(notification1, times(0)).notify("", notificationString)
verify(notification2, times(0)).notify("", notificationString)
sut.enable(firstName)
sut.notify(notificationString)
verify(notification1, times(1)).notify("", notificationString)
verify(notification2, times(0)).notify("", notificationString)
sut.enable(secondName)
sut.notify(notificationString)
verify(notification1, times(2)).notify("", notificationString)
verify(notification2, times(1)).notify("", notificationString)
sut.disable(firstName)
sut.notify(notificationString)
verify(notification1, times(2)).notify("", notificationString)
verify(notification2, times(2)).notify("", notificationString)
}
}
private val firstName = "first"
private val secondName = "second"
private var notification1: ShellNotification = _
private var notification2: ShellNotification = _
override protected def beforeEach(): Unit = {
notification1 = mock[ShellNotification]
notification2 = mock[ShellNotification]
when(notification1.name).thenReturn(firstName)
when(notification2.name).thenReturn(secondName)
}
}
Example 43
| Source File: KyuubiSessionSubPageSuite.scala From kyuubi with Apache License 2.0 | 5 votes |
|
package org.apache.spark.ui
import javax.servlet.http.HttpServletRequest
import scala.util.Try
import org.apache.spark.{KyuubiSparkUtil, SparkConf, SparkContext, SparkFunSuite}
import org.mockito.Mockito.when
import org.scalatest.mock.MockitoSugar
import yaooqinn.kyuubi.ui.{ExecutionInfo, KyuubiServerListener, SessionInfo}
class KyuubiSessionSubPageSuite extends SparkFunSuite with MockitoSugar {
var sc: SparkContext = _
var user: String = _
var tab: KyuubiSessionTab = _
override def beforeAll(): Unit = {
val conf = new SparkConf(loadDefaults = true).setMaster("local").setAppName("test")
sc = new SparkContext(conf)
user = KyuubiSparkUtil.getCurrentUserName
tab = new KyuubiSessionTab(user, sc)
}
override def afterAll(): Unit = {
sc.stop()
}
test("render kyuubi session page") {
val page = new KyuubiSessionSubPage(tab)
val request = mock[HttpServletRequest]
intercept[IllegalArgumentException](page.render(request))
val id = "id1"
when(request.getParameter("id")).thenReturn(id)
intercept[IllegalArgumentException](page.render(request))
val sessionInfo = mock[SessionInfo]
val tab1 = mock[KyuubiSessionTab]
when(request.getParameter("id")).thenReturn(id)
val listener = mock[KyuubiServerListener]
when(tab1.listener).thenReturn(listener)
when(listener.getSession(id)).thenReturn(Some(sessionInfo))
when(sessionInfo.sessionId).thenReturn("1")
when(listener.getExecutionList).thenReturn(Seq[ExecutionInfo]())
when(tab1.appName).thenReturn("name")
when(tab1.headerTabs).thenReturn(Seq[WebUITab]())
val page2 = new KyuubiSessionSubPage(tab1)
assert(Try { page2.render(request) }.isSuccess )
}
}
Example 44
| Source File: MesosClusterSchedulerSuite.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.scheduler.mesos
import java.util.Date
import org.scalatest.mock.MockitoSugar
import org.apache.spark.deploy.Command
import org.apache.spark.deploy.mesos.MesosDriverDescription
import org.apache.spark.scheduler.cluster.mesos._
import org.apache.spark.{LocalSparkContext, SparkConf, SparkFunSuite}
class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext with MockitoSugar {
private val command = new Command("mainClass", Seq("arg"), null, null, null, null)
test("can queue drivers") {
val conf = new SparkConf()
conf.setMaster("mesos://localhost:5050")
conf.setAppName("spark mesos")
val scheduler = new MesosClusterScheduler(
new BlackHoleMesosClusterPersistenceEngineFactory, conf) {
override def start(): Unit = { ready = true }
}
scheduler.start()
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", 1000, 1, true,
command, Map[String, String](), "s1", new Date()))
assert(response.success)
val response2 =
scheduler.submitDriver(new MesosDriverDescription(
"d1", "jar", 1000, 1, true, command, Map[String, String](), "s2", new Date()))
assert(response2.success)
val state = scheduler.getSchedulerState()
val queuedDrivers = state.queuedDrivers.toList
assert(queuedDrivers(0).submissionId == response.submissionId)
assert(queuedDrivers(1).submissionId == response2.submissionId)
}
test("can kill queued drivers") {
val conf = new SparkConf()
conf.setMaster("mesos://localhost:5050")
conf.setAppName("spark mesos")
val scheduler = new MesosClusterScheduler(
new BlackHoleMesosClusterPersistenceEngineFactory, conf) {
override def start(): Unit = { ready = true }
}
scheduler.start()
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", 1000, 1, true,
command, Map[String, String](), "s1", new Date()))
assert(response.success)
val killResponse = scheduler.killDriver(response.submissionId)
assert(killResponse.success)
val state = scheduler.getSchedulerState()
assert(state.queuedDrivers.isEmpty)
}
}
Example 45
| Source File: TestHelper.scala From diffy with GNU Affero General Public License v3.0 | 5 votes |
|
package ai.diffy
import java.net.InetSocketAddress
import ai.diffy.analysis._
import ai.diffy.compare.Difference
import ai.diffy.proxy._
import ai.diffy.util.ServiceInstance
import com.twitter.util.{Duration, StorageUnit}
import org.scalatest.mock.MockitoSugar
object TestHelper extends MockitoSugar {
lazy val testSettings = Settings(
datacenter = "test",
servicePort = new InetSocketAddress(9999),
candidate = "candidate",
primary = "primary",
secondary = "secondary",
protocol = "test",
clientId = "test",
pathToThriftJar = "test",
serviceClass = "test",
serviceName = "test",
apiRoot = "test",
enableThriftMux = false,
relativeThreshold = 0.0,
absoluteThreshold = 0.0,
teamEmail = "test",
emailDelay = Duration.fromSeconds(0),
rootUrl = "test",
allowHttpSideEffects = true,
excludeHttpHeadersComparison = true,
skipEmailsWhenNoErrors = false,
httpsPort = "443",
useFramedThriftTransport = false,
responseMode = ServiceInstance.Primary,
maxHeaderSize = StorageUnit.fromKilobytes(32),
maxResponseSize = StorageUnit.fromMegabytes(5)
)
def makeEmptyJoinedDifferences = {
val rawCounter = RawDifferenceCounter(new InMemoryDifferenceCounter())
val noiseCounter = NoiseDifferenceCounter(new InMemoryDifferenceCounter())
JoinedDifferences(rawCounter, noiseCounter)
}
def makePopulatedJoinedDifferences(endpoint : String, diffs : Map[String, Difference]) = {
val rawCounter = RawDifferenceCounter(new InMemoryDifferenceCounter())
val noiseCounter = NoiseDifferenceCounter(new InMemoryDifferenceCounter())
val data = new InMemoryEndpointMetadata()
data.add(diffs)
rawCounter.counter.asInstanceOf[InMemoryDifferenceCounter].endpointsMap += (endpoint -> data)
JoinedDifferences(rawCounter, noiseCounter)
}
}
Example 46
| Source File: XMLParserXMLExtractNamespaceSpec.scala From akka-xml-parser with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.akka.xml
import akka.stream.scaladsl.{Keep, Source}
import akka.util.ByteString
import org.scalatest.{FlatSpec, Matchers}
import org.scalatest.concurrent.{Eventually, ScalaFutures}
import org.scalatest.mock.MockitoSugar
import org.scalatest.time.{Millis, Seconds, Span}
class XMLParserXMLExtractNamespaceSpec extends FlatSpec
with Matchers
with ScalaFutures
with MockitoSugar
with Eventually
with XMLParserFixtures {
val f = fixtures
implicit override val patienceConfig =
PatienceConfig(timeout = Span(5, Seconds), interval = Span(5, Millis))
import f._
behavior of "CompleteChunkStage#parser"
it should "Parse and extract several non-default namespaces" in {
val testXMLX =
<ns5:GovTalkMessage
xmlns:ns0="http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/13-14/2"
xmlns:ns2="http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/15-16/1"
xmlns:ns5="http://www.govtalk.gov.uk/CM/envelope"
xmlns:ns1="http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/14-15/1"
xmlns:ns3="http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/16-17/1"
xmlns:ns4="http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/17-18/1"
xmlns="">
<ns5:EnvelopeVersion>2.0</ns5:EnvelopeVersion>
<ns5:Header></ns5:Header>
<ns5:GovTalkDetails></ns5:GovTalkDetails>
</ns5:GovTalkMessage>
val source = Source(List(ByteString(testXMLX.toString())))
val paths = Seq[XMLInstruction](
XMLExtract(Seq("GovTalkMessage"), Map("xmlns:ns2" -> "http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/15-16/1")),
XMLExtract(Seq("GovTalkMessage"), Map("xmlns:BLABLA" -> "http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/13-14/2")),
XMLExtract(Seq("GovTalkMessage"), Map("xmlns" -> "http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/17-18/1")),
XMLExtract(Seq("GovTalkMessage"), Map("xmlns" -> "http://www.govtalk.gov.uk/CM/envelope"))
)
val expected = Set(
XMLElement(List("GovTalkMessage"), Map("xmlns:ns2" -> "http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/15-16/1"), Some("")),
XMLElement(List("GovTalkMessage"), Map("xmlns:ns0" -> "http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/13-14/2"), Some("")),
XMLElement(List("GovTalkMessage"), Map("xmlns:ns4" -> "http://www.govtalk.gov.uk/taxation/PAYE/RTI/EmployerPaymentSummary/17-18/1"), Some("")),
XMLElement(List("GovTalkMessage"), Map("xmlns:ns5" -> "http://www.govtalk.gov.uk/CM/envelope"), Some("")),
XMLElement(List(), Map(CompleteChunkStage.STREAM_SIZE -> "681"), Some(CompleteChunkStage.STREAM_SIZE))
)
whenReady(source.runWith(parseToXMLElements(paths))) { r =>
r shouldBe expected
}
whenReady(source.runWith(parseToByteString(paths))) { r =>
whenReady(source.toMat(collectByteString)(Keep.right).run()) { t =>
r shouldBe t
}
}
}
}
Example 47
| Source File: XMLParsingStopSpec.scala From akka-xml-parser with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.akka.xml
import akka.stream.scaladsl.{Keep, Source}
import org.scalatest.{FlatSpec, Matchers}
import org.scalatest.concurrent.{Eventually, ScalaFutures}
import org.scalatest.mock.MockitoSugar
import org.scalatest.time.{Millis, Seconds, Span}
class XMLParsingStopSpec extends FlatSpec
with Matchers
with ScalaFutures
with MockitoSugar
with Eventually
with XMLParserFixtures {
val f = fixtures
implicit override val patienceConfig =
PatienceConfig(timeout = Span(5, Seconds), interval = Span(5, Millis))
import f._
it should "Stop parsing when the passed in xPath is encountered" in {
val source = Source(ParserTestHelpers.getBrokenMessage(ParserTestHelpers.sa100.toString, 100))
val paths = Seq[XMLInstruction](
XMLExtract(Seq("GovTalkMessage", "Header", "MessageDetails", "Class")),
XMLExtract(Seq("GovTalkMessage", "Header", "MessageDetails", "Qualifier")),
XMLExtract(Seq("GovTalkMessage", "Header", "MessageDetails", "Function")),
XMLExtract(Seq("GovTalkMessage", "Body", "IRenvelope", "MTR", "SA100", "YourPersonalDetails", "NationalInsuranceNumber")), //This is in the body, will not be parsed
XMLStopParsing(Seq("GovTalkMessage", "Body"))
)
val expected = Set(
XMLElement(List("GovTalkMessage", "Header", "MessageDetails", "Class"), Map(), Some("HMRC-SA-SA100")),
XMLElement(List("GovTalkMessage", "Header", "MessageDetails", "Function"), Map(), Some("submit")),
XMLElement(List("GovTalkMessage", "Header", "MessageDetails", "Qualifier"), Map(), Some("request"))
)
whenReady(source.runWith(parseToXMLElements(paths))) { r =>
r.filterNot(a => a.value == Some(FastParsingStage.STREAM_SIZE)) shouldBe expected
}
whenReady(source.runWith(parseToByteString(paths))) { r =>
whenReady(source.toMat(collectByteString)(Keep.right).run()) { t =>
r shouldBe t
}
}
}
it should "Notify if the payload exceeded the maximum allowed size" in {
val source = Source(ParserTestHelpers.getBrokenMessage(ParserTestHelpers.sa100.toString, 100))
val paths = Seq[XMLInstruction](XMLExtract(Seq("GovTalkMessage", "Header", "MessageDetails", "Class")))
val expected = Set(
XMLElement(List("GovTalkMessage", "Header", "MessageDetails", "Class"), Map(), Some("HMRC-SA-SA100")),
XMLElement(List(), Map(), Some("Stream max size"))
)
whenReady(source.runWith(parseToXMLElements(paths, Some(200)))) { r =>
r.filterNot(a => a.value == Some(FastParsingStage.STREAM_SIZE)) shouldBe expected
}
whenReady(source.runWith(parseToByteString(paths))) { r =>
whenReady(source.toMat(collectByteString)(Keep.right).run()) { t =>
r shouldBe t
}
}
}
}
Example 48
| Source File: JeroMQSocketSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.communication.socket
import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer
import org.scalatest.{Matchers, BeforeAndAfter, OneInstancePerTest, FunSpec}
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.zeromq.ZMsg
class JeroMQSocketSpec extends FunSpec with MockitoSugar
with OneInstancePerTest with BeforeAndAfter with Matchers
{
private val runnable = mock[ZeroMQSocketRunnable]
@volatile private var running = true
// Mock the running of the runnable for the tests
doAnswer(new Answer[Unit] {
override def answer(invocation: InvocationOnMock): Unit = while (running) {
Thread.sleep(1)
}
}).when(runnable).run()
// Mock the close of the runnable to shutdown
doAnswer(new Answer[Unit] {
override def answer(invocation: InvocationOnMock): Unit = running = false
}).when(runnable).close()
private val socket: JeroMQSocket = new JeroMQSocket(runnable)
after {
running = false
}
describe("JeroMQSocket") {
describe("#send") {
it("should offer a message to the runnable") {
val message: String = "Some Message"
val expected = ZMsg.newStringMsg(message)
socket.send(message.getBytes)
verify(runnable).offer(expected)
}
it("should thrown and AssertionError when socket is no longer alive") {
socket.close()
intercept[AssertionError] {
socket.send("".getBytes)
}
}
}
describe("#close") {
it("should close the runnable") {
socket.close()
verify(runnable).close()
}
it("should close the socket thread") {
socket.close()
socket.isAlive should be (false)
}
}
describe("#isAlive") {
it("should evaluate to true when the socket thread is alive") {
socket.isAlive should be (true)
}
it("should evaluate to false when the socket thread is dead") {
socket.close()
socket.isAlive should be (false)
}
}
}
}
Example 49
| Source File: OrderedSupportSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.communication.utils
import akka.actor._
import akka.testkit.{ImplicitSender, TestKit}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
case class OrderedType()
case class NotOrderedType()
case class FinishProcessingMessage()
case class ReceiveMessageCount(count: Int)
class TestOrderedSupport extends OrderedSupport {
var receivedCounter = 0
override def orderedTypes(): Seq[Class[_]] = Seq(classOf[OrderedType])
override def receive: Receive = {
case OrderedType() =>
startProcessing()
receivedCounter = receivedCounter + 1
sender ! ReceiveMessageCount(receivedCounter)
case NotOrderedType() =>
receivedCounter = receivedCounter + 1
sender ! ReceiveMessageCount(receivedCounter)
case FinishProcessingMessage() =>
finishedProcessing()
}
}
class OrderedSupportSpec extends TestKit(ActorSystem("OrderedSupportSystem"))
with ImplicitSender with Matchers with FunSpecLike
with MockitoSugar {
describe("OrderedSupport"){
describe("#waiting"){
it("should wait for types defined in orderedTypes"){
val testOrderedSupport = system.actorOf(Props[TestOrderedSupport])
// Send a message having a type in orderedTypes
// Starts processing and is handled with receive()
testOrderedSupport ! new OrderedType
// This message should be handled with waiting()
testOrderedSupport ! new OrderedType
// Verify receive was not called for the second OrderedType
expectMsg(ReceiveMessageCount(1))
}
it("should process types not defined in orderedTypes"){
val testOrderedSupport = system.actorOf(Props[TestOrderedSupport])
// Send a message that starts the processing
testOrderedSupport ! new OrderedType
// Send a message having a type not in orderedTypes
testOrderedSupport ! new NotOrderedType
// Verify receive did get called for NotOrderedType
expectMsg(ReceiveMessageCount(1))
expectMsg(ReceiveMessageCount(2))
}
}
describe("#finishedProcessing"){
it("should switch actor to receive method"){
val testOrderedSupport = system.actorOf(Props[TestOrderedSupport])
// Switch actor to waiting mode
testOrderedSupport ! new OrderedType
// Call finishedProcessing
testOrderedSupport ! new FinishProcessingMessage
// Sending something that would match in receive, and is in orderedTypes
testOrderedSupport ! new OrderedType
expectMsg(ReceiveMessageCount(1))
expectMsg(ReceiveMessageCount(2))
}
}
}
}
Example 50
| Source File: JVMReprSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package integration.interpreter.scala
import java.util
import java.io.ByteArrayOutputStream
import jupyter.{Displayer, Displayers, MIMETypes}
import org.apache.toree.global.StreamState
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.interpreter.Results.Success
import org.apache.toree.kernel.api.{DisplayMethodsLike, KernelLike}
import org.apache.toree.kernel.interpreter.scala.ScalaInterpreter
import org.mockito.Mockito.doReturn
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
import org.scalatest.mock.MockitoSugar
import scala.util.Random
class JVMReprSpec extends FunSpec with Matchers with MockitoSugar with BeforeAndAfter {
private val outputResult = new ByteArrayOutputStream()
private var interpreter: Interpreter = _
before {
val mockKernel = mock[KernelLike]
val mockDisplayMethods = mock[DisplayMethodsLike]
doReturn(mockDisplayMethods).when(mockKernel).display
interpreter = new ScalaInterpreter().init(mockKernel)
StreamState.setStreams(outputStream = outputResult)
}
after {
interpreter.stop()
outputResult.reset()
}
describe("ScalaInterpreter") {
describe("#interpret") {
it("should display Scala int as a text representation") {
val (result, outputOrError) = interpreter.interpret("val a = 12")
result should be(Success)
outputOrError.isLeft should be(true)
outputOrError.left.get should be(Map(MIMETypes.TEXT -> "12"))
}
it("should display Scala Some(str) as a text representation") {
val (result, outputOrError) = interpreter.interpret("""val a = Some("str")""")
result should be(Success)
outputOrError.isLeft should be(true)
outputOrError.left.get should be(Map(MIMETypes.TEXT -> "Some(str)"))
}
ignore("should use the Jupyter REPR API for display representation") {
Displayers.register(classOf[DisplayerTest], new Displayer[DisplayerTest] {
override def display(t: DisplayerTest): util.Map[String, String] = {
val output = new util.HashMap[String, String]()
output.put("text/plain", s"test object: ${t.id}")
output.put("application/json", s"""{"id": ${t.id}""")
output
}
})
val inst = DisplayerTest()
interpreter.bind("inst", classOf[DisplayerTest].getName, inst, List())
val (result, outputOrError) = interpreter.interpret("""inst""")
result should be(Success)
outputOrError.isLeft should be(true)
outputOrError.left.get should be(Map(
MIMETypes.TEXT -> s"test object: ${inst.id}",
"application/json" -> s"""{"id": ${inst.id}"""
))
}
}
}
}
case class DisplayerTest(id: Long = new Random().nextLong())
Example 51
| Source File: ClientCommManagerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.comm
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.client.ActorLoader
import org.apache.toree.kernel.protocol.v5.content.CommContent
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
class ClientCommManagerSpec extends FunSpec with Matchers with BeforeAndAfter
with MockitoSugar
{
private val TestTargetName = "some target"
private var mockActorLoader: ActorLoader = _
private var mockKMBuilder: KMBuilder = _
private var mockCommRegistrar: CommRegistrar = _
private var clientCommManager: ClientCommManager = _
private var generatedCommWriter: CommWriter = _
before {
mockActorLoader = mock[ActorLoader]
mockKMBuilder = mock[KMBuilder]
mockCommRegistrar = mock[CommRegistrar]
clientCommManager = new ClientCommManager(
mockActorLoader,
mockKMBuilder,
mockCommRegistrar
) {
override protected def newCommWriter(commId: UUID): CommWriter = {
val commWriter = super.newCommWriter(commId)
generatedCommWriter = commWriter
val spyCommWriter = spy(commWriter)
doNothing().when(spyCommWriter)
.sendCommKernelMessage(any[KernelMessageContent with CommContent])
spyCommWriter
}
}
}
describe("ClientCommManager") {
describe("#open") {
it("should return a wrapped instance of ClientCommWriter") {
clientCommManager.open(TestTargetName, v5.MsgData.Empty)
// Exposed hackishly for testing
generatedCommWriter shouldBe a [ClientCommWriter]
}
}
}
}
Example 52
| Source File: ShellClientSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.client.socket
import java.util.UUID
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.communication.security.SecurityActorType
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.client.ActorLoader
import org.apache.toree.kernel.protocol.v5.content.ExecuteRequest
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import org.mockito.Mockito._
import org.mockito.Matchers._
import play.api.libs.json.Json
class ShellClientSpec extends TestKit(ActorSystem("ShellActorSpec"))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
private val SignatureEnabled = true
describe("ShellClientActor") {
val socketFactory = mock[SocketFactory]
val mockActorLoader = mock[ActorLoader]
val probe : TestProbe = TestProbe()
when(socketFactory.ShellClient(
any(classOf[ActorSystem]), any(classOf[ActorRef])
)).thenReturn(probe.ref)
val signatureManagerProbe = TestProbe()
doReturn(system.actorSelection(signatureManagerProbe.ref.path.toString))
.when(mockActorLoader).load(SecurityActorType.SignatureManager)
val shellClient = system.actorOf(Props(
classOf[ShellClient], socketFactory, mockActorLoader, SignatureEnabled
))
describe("send execute request") {
it("should send execute request") {
val request = ExecuteRequest(
"foo", false, true, UserExpressions(), true
)
val header = Header(
UUID.randomUUID().toString, "spark",
UUID.randomUUID().toString, MessageType.Incoming.ExecuteRequest.toString,
"5.0"
)
val kernelMessage = KernelMessage(
Seq[Array[Byte]](), "",
header, HeaderBuilder.empty,
Metadata(), Json.toJson(request).toString
)
shellClient ! kernelMessage
// Echo back the kernel message sent to have a signature injected
signatureManagerProbe.expectMsgClass(classOf[KernelMessage])
signatureManagerProbe.reply(kernelMessage)
probe.expectMsgClass(classOf[ZMQMessage])
}
}
}
}
Example 53
| Source File: HeartbeatClientSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.client.socket
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5.client.ActorLoader
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import org.mockito.Matchers._
import org.mockito.Mockito._
class HeartbeatClientSpec extends TestKit(ActorSystem("HeartbeatActorSpec"))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("HeartbeatClientActor") {
val socketFactory = mock[SocketFactory]
val mockActorLoader = mock[ActorLoader]
val probe : TestProbe = TestProbe()
when(socketFactory.HeartbeatClient(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(probe.ref)
val heartbeatClient = system.actorOf(Props(
classOf[HeartbeatClient], socketFactory, mockActorLoader, true
))
describe("send heartbeat") {
it("should send ping ZMQMessage") {
heartbeatClient ! HeartbeatMessage
probe.expectMsgClass(classOf[ZMQMessage])
}
}
}
}
Example 54
| Source File: SparkKernelClientSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.client
import akka.actor.ActorSystem
import akka.testkit.{TestKit, TestProbe}
import org.apache.toree.comm.{CommCallbacks, CommStorage, CommRegistrar}
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.client.execution.ExecuteRequestTuple
import scala.concurrent.duration._
import org.mockito.Mockito._
import org.mockito.Matchers.{eq => mockEq, _}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, FunSpecLike, Matchers}
class SparkKernelClientSpec
extends TestKit(ActorSystem("SparkKernelClientActorSystem"))
with Matchers with MockitoSugar with FunSpecLike with BeforeAndAfter
{
private val TestTargetName = "some target"
private var mockActorLoader: ActorLoader = _
private var mockCommRegistrar: CommRegistrar = _
private var sparkKernelClient: SparkKernelClient = _
private var executeRequestProbe: TestProbe = _
private var shellClientProbe: TestProbe = _
before {
mockActorLoader = mock[ActorLoader]
mockCommRegistrar = mock[CommRegistrar]
executeRequestProbe = TestProbe()
when(mockActorLoader.load(MessageType.Incoming.ExecuteRequest))
.thenReturn(system.actorSelection(executeRequestProbe.ref.path.toString))
shellClientProbe = TestProbe()
when(mockActorLoader.load(SocketType.ShellClient))
.thenReturn(system.actorSelection(shellClientProbe.ref.path.toString))
sparkKernelClient = new SparkKernelClient(
mockActorLoader, system, mockCommRegistrar)
}
describe("SparkKernelClient") {
describe("#execute") {
it("should send an ExecuteRequest message") {
val func = (x: Any) => println(x)
sparkKernelClient.execute("val foo = 2")
executeRequestProbe.expectMsgClass(classOf[ExecuteRequestTuple])
}
}
}
}
Example 55
| Source File: DataFrameConverterSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{DataFrame, Row}
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfterAll, FunSpec, Matchers}
import play.api.libs.json.{JsArray, JsString, Json}
import test.utils.SparkContextProvider
import scala.collection.mutable
class DataFrameConverterSpec extends FunSpec with MockitoSugar with Matchers with BeforeAndAfterAll {
lazy val spark = SparkContextProvider.sparkContext
override protected def afterAll(): Unit = {
spark.stop()
super.afterAll()
}
val dataFrameConverter: DataFrameConverter = new DataFrameConverter
val mockDataFrame = mock[DataFrame]
val mockRdd = spark.parallelize(Seq(Row(new mutable.WrappedArray.ofRef(Array("test1", "test2")), 2, null)))
val mockStruct = mock[StructType]
val columns = Seq("foo", "bar").toArray
doReturn(mockStruct).when(mockDataFrame).schema
doReturn(columns).when(mockStruct).fieldNames
doReturn(mockRdd).when(mockDataFrame).rdd
describe("DataFrameConverter") {
describe("#convert") {
it("should convert to a valid JSON object") {
val someJson = dataFrameConverter.convert(mockDataFrame, "json")
val jsValue = Json.parse(someJson.get)
jsValue \ "columns" should be (JsArray(Seq(JsString("foo"), JsString("bar"))))
jsValue \ "rows" should be (JsArray(Seq(
JsArray(Seq(JsString("[test1, test2]"), JsString("2"), JsString("null")))
)))
}
it("should convert to csv") {
val csv = dataFrameConverter.convert(mockDataFrame, "csv").get
val values = csv.split("\n")
values(0) shouldBe "foo,bar"
values(1) shouldBe "[test1, test2],2,null"
}
it("should convert to html") {
val html = dataFrameConverter.convert(mockDataFrame, "html").get
html.contains("<th>foo</th>") should be(true)
html.contains("<th>bar</th>") should be(true)
html.contains("<td>[test1, test2]</td>") should be(true)
html.contains("<td>2</td>") should be(true)
html.contains("<td>null</td>") should be(true)
}
it("should convert limit the selection") {
val someLimited = dataFrameConverter.convert(mockDataFrame, "csv", 1)
val limitedLines = someLimited.get.split("\n")
limitedLines.length should be(2)
}
it("should return a Failure for invalid types") {
val result = dataFrameConverter.convert(mockDataFrame, "Invalid Type")
result.isFailure should be(true)
}
}
}
}
Example 56
| Source File: PartiallyUnrolledIteratorSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.storage
import org.mockito.Matchers
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.apache.spark.SparkFunSuite
import org.apache.spark.memory.MemoryMode.ON_HEAP
import org.apache.spark.storage.memory.{MemoryStore, PartiallyUnrolledIterator}
class PartiallyUnrolledIteratorSuite extends SparkFunSuite with MockitoSugar {
test("join two iterators") {
val unrollSize = 1000
val unroll = (0 until unrollSize).iterator
val restSize = 500
val rest = (unrollSize until restSize + unrollSize).iterator
val memoryStore = mock[MemoryStore]
val joinIterator = new PartiallyUnrolledIterator(memoryStore, ON_HEAP, unrollSize, unroll, rest)
// Firstly iterate over unrolling memory iterator
(0 until unrollSize).foreach { value =>
assert(joinIterator.hasNext)
assert(joinIterator.hasNext)
assert(joinIterator.next() == value)
}
joinIterator.hasNext
joinIterator.hasNext
verify(memoryStore, times(1))
.releaseUnrollMemoryForThisTask(Matchers.eq(ON_HEAP), Matchers.eq(unrollSize.toLong))
// Secondly, iterate over rest iterator
(unrollSize until unrollSize + restSize).foreach { value =>
assert(joinIterator.hasNext)
assert(joinIterator.hasNext)
assert(joinIterator.next() == value)
}
joinIterator.close()
// MemoryMode.releaseUnrollMemoryForThisTask is called only once
verifyNoMoreInteractions(memoryStore)
}
}
Example 57
| Source File: TestSFObjectWriter.scala From spark-salesforce with Apache License 2.0 | 5 votes |
|
package com.springml.spark.salesforce
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{ FunSuite, BeforeAndAfterEach}
import com.springml.salesforce.wave.api.BulkAPI
import org.apache.spark.{ SparkConf, SparkContext}
import com.springml.salesforce.wave.model.{ JobInfo, BatchInfo}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{ Row, DataFrame, SQLContext}
import org.apache.spark.sql.types.{ StructType, StringType, StructField}
class TestSFObjectWriter extends FunSuite with MockitoSugar with BeforeAndAfterEach {
val contact = "Contact";
val jobId = "750B0000000WlhtIAC";
val batchId = "751B0000000scSHIAY";
val data = "Id,Description\n003B00000067Rnx,123456\n003B00000067Rnw,7890";
val bulkAPI = mock[BulkAPI](withSettings().serializable())
val writer = mock[SFObjectWriter]
var sparkConf: SparkConf = _
var sc: SparkContext = _
override def beforeEach() {
val jobInfo = new JobInfo
jobInfo.setId(jobId)
when(bulkAPI.createJob(contact)).thenReturn(jobInfo)
val batchInfo = new BatchInfo
batchInfo.setId(batchId)
batchInfo.setJobId(jobId)
when(bulkAPI.addBatch(jobId, data)).thenReturn(batchInfo)
when(bulkAPI.closeJob(jobId)).thenReturn(jobInfo)
when(bulkAPI.isCompleted(jobId)).thenReturn(true)
sparkConf = new SparkConf().setMaster("local").setAppName("Test SF Object Update")
sc = new SparkContext(sparkConf)
}
private def sampleDF() : DataFrame = {
val rowArray = new Array[Row](2)
val fieldArray = new Array[String](2)
fieldArray(0) = "003B00000067Rnx"
fieldArray(1) = "Desc1"
rowArray(0) = Row.fromSeq(fieldArray)
val fieldArray1 = new Array[String](2)
fieldArray1(0) = "001B00000067Rnx"
fieldArray1(1) = "Desc2"
rowArray(1) = Row.fromSeq(fieldArray1)
val rdd = sc.parallelize(rowArray)
val schema = StructType(
StructField("id", StringType, true) ::
StructField("desc", StringType, true) :: Nil)
val sqlContext = new SQLContext(sc)
sqlContext.createDataFrame(rdd, schema)
}
test ("Write Object to Salesforce") {
val df = sampleDF();
val csvHeader = Utils.csvHeadder(df.schema)
writer.writeData(df.rdd)
sc.stop()
}
}
Example 58
| Source File: MockIdentityVerificationHttp.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.helpers
import org.mockito.ArgumentMatchers
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import play.api.http.Status
import play.api.libs.json.Json
import scala.concurrent.Future
import scala.io.Source
import uk.gov.hmrc.http.{ HttpGet, HttpResponse }
object MockIdentityVerificationHttp extends MockitoSugar {
val mockHttp = mock[HttpGet]
val possibleJournies = Map(
"success-journey-id" -> "test/resources/identity-verification/success.json",
"incomplete-journey-id" -> "test/resources/identity-verification/incomplete.json",
"failed-matching-journey-id" -> "test/resources/identity-verification/failed-matching.json",
"insufficient-evidence-journey-id" -> "test/resources/identity-verification/insufficient-evidence.json",
"locked-out-journey-id" -> "test/resources/identity-verification/locked-out.json",
"user-aborted-journey-id" -> "test/resources/identity-verification/user-aborted.json",
"timeout-journey-id" -> "test/resources/identity-verification/timeout.json",
"technical-issue-journey-id" -> "test/resources/identity-verification/technical-issue.json",
"precondition-failed-journey-id" -> "test/resources/identity-verification/precondition-failed.json",
"invalid-journey-id" -> "test/resources/identity-verification/invalid-result.json",
"invalid-fields-journey-id" -> "test/resources/identity-verification/invalid-fields.json",
"failed-iv-journey-id" -> "test/resources/identity-verification/failed-iv.json"
)
def mockJourneyId(journeyId: String): Unit = {
val fileContents = Source.fromFile(possibleJournies(journeyId)).mkString
when(mockHttp.GET[HttpResponse](ArgumentMatchers.contains(journeyId))(ArgumentMatchers.any(), ArgumentMatchers.any(),ArgumentMatchers.any())).
thenReturn(Future.successful(HttpResponse(Status.OK, responseJson = Some(Json.parse(fileContents)))))
}
possibleJournies.keys.foreach(mockJourneyId)
}
Example 59
| Source File: MockCitizenDetailsHttp.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.helpers
import org.mockito.ArgumentMatchers
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import uk.gov.hmrc.domain.Nino
import uk.gov.hmrc.play.http._
import uk.gov.hmrc.play.test.UnitSpec
import scala.concurrent.Future
import uk.gov.hmrc.http.{BadRequestException, HttpGet, HttpResponse, NotFoundException, Upstream5xxResponse}
object MockCitizenDetailsHttp extends UnitSpec with MockitoSugar {
val mockHttp = mock[HttpGet]
val ninos = List(
TestAccountBuilder.regularNino,
TestAccountBuilder.blankNino,
TestAccountBuilder.fullUserNino,
TestAccountBuilder.contractedOutBTestNino,
TestAccountBuilder.mqpNino,
TestAccountBuilder.forecastOnlyNino,
TestAccountBuilder.noNameNino,
TestAccountBuilder.abroadNino,
TestAccountBuilder.mqpAbroadNino,
TestAccountBuilder.hrpNino,
TestAccountBuilder.fillGapSingle,
TestAccountBuilder.fillGapsMultiple,
TestAccountBuilder.noQualifyingYears,
TestAccountBuilder.backendNotFound,
TestAccountBuilder.urBannerNino,
TestAccountBuilder.noUrBannerNino,
TestAccountBuilder.excludedAll,
TestAccountBuilder.excludedAllButDead,
TestAccountBuilder.excludedAllButDeadMCI,
TestAccountBuilder.excludedDissonanceIomMwrreAbroad,
TestAccountBuilder.excludedIomMwrreAbroad,
TestAccountBuilder.excludedMwrreAbroad,
TestAccountBuilder.excludedAbroad,
TestAccountBuilder.excludedMwrre,
TestAccountBuilder.spaUnderConsiderationExclusionAmountDisNino,
TestAccountBuilder.spaUnderConsiderationExclusionIoMNino,
TestAccountBuilder.spaUnderConsiderationExclusionMwrreNino,
TestAccountBuilder.spaUnderConsiderationExclusionOverSpaNino,
TestAccountBuilder.spaUnderConsiderationExclusionMultipleNino,
TestAccountBuilder.spaUnderConsiderationExclusionNoFlagNino
)
def createMockedURL(nino: Nino, response: Future[HttpResponse]): Unit =
when(mockHttp.GET[HttpResponse](ArgumentMatchers.endsWith(s"citizen-details/$nino/designatory-details"))(ArgumentMatchers.any(), ArgumentMatchers.any(),ArgumentMatchers.any())).thenReturn(response)
private def setupCitizenDetailsMocking(nino: Nino) = createMockedURL(nino, TestAccountBuilder.jsonResponse(nino, "citizen-details"))
ninos.foreach(setupCitizenDetailsMocking)
createMockedURL(TestAccountBuilder.notFoundNino, Future.failed(new NotFoundException("")))
createMockedURL(TestAccountBuilder.nonExistentNino, Future.failed(new NotFoundException("")))
createMockedURL(TestAccountBuilder.blankNino, Future.failed(new BadRequestException("")))
createMockedURL(TestAccountBuilder.internalServerError, Future.failed(new Upstream5xxResponse("CRITICAL FAILURE", 500, 500)))
}
Example 60
| Source File: MockTemplateRenderer.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.utils
import org.scalatest.mock.MockitoSugar
import play.api.Mode.Mode
import play.api.i18n.Messages
import play.api.{Configuration, Play}
import play.twirl.api.Html
import uk.gov.hmrc.nisp.config.{LocalTemplateRenderer, WsAllMethods}
import scala.concurrent.duration._
object MockTemplateRenderer extends LocalTemplateRenderer {
override lazy val templateServiceBaseUrl = "http://example.com/template/mustache"
override val refreshAfter = 10 minutes
override val wsHttp = MockitoSugar.mock[WsAllMethods]
override def renderDefaultTemplate(path:String, content: Html, extraArgs: Map[String, Any])(implicit messages: Messages) = {
Html(
"<title>" + extraArgs("pageTitle") + "</title>"
+ "<sidebar>"+extraArgs("sidebar")+"</sidebar>"
+ "<navLinks>"+extraArgs("navLinks")+"</navLinks>"
+ displayUrBanner(extraArgs) +
"<mainContentHeader>" +extraArgs("mainContentHeader")+ "</mainContentHeader>"
+ content)
}
def displayUrBanner(extraArgs: Map[String, Any]): String ={
if(extraArgs.contains("fullWidthBannerTitle")){
"<div id=full-width-banner>" + "<div class = \"full-width-banner__title\">" + extraArgs("fullWidthBannerTitle") + "</div>" + "<div id = fullWidthBannerLink>" + extraArgs("fullWidthBannerLink") + "</div>"+ "<div>" + extraArgs("fullWidthBannerText")+ "</div>"+ "<div id = fullWidthBannerDismissText>"+extraArgs("fullWidthBannerDismissText")+"</div>"
}
else ""
}
override protected def mode: Mode = Play.current.mode
override protected def runModeConfiguration: Configuration = Play.current.configuration
}
Example 61
| Source File: CitizenDetailsServiceSpec.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.services
import org.joda.time.LocalDate
import org.scalatest.BeforeAndAfter
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.mock.MockitoSugar
import org.scalatestplus.play.OneAppPerSuite
import uk.gov.hmrc.domain.Nino
import uk.gov.hmrc.http.{HeaderCarrier, Upstream5xxResponse}
import uk.gov.hmrc.nisp.helpers.{MockCitizenDetailsService, TestAccountBuilder}
import uk.gov.hmrc.nisp.models.citizen.{Address, Citizen, CitizenDetailsError, CitizenDetailsResponse}
import uk.gov.hmrc.play.test.UnitSpec
import scala.concurrent.Future
class CitizenDetailsServiceSpec extends UnitSpec with MockitoSugar with BeforeAndAfter with ScalaFutures with OneAppPerSuite {
val nino: Nino = TestAccountBuilder.regularNino
val noNameNino: Nino = TestAccountBuilder.noNameNino
val nonExistentNino: Nino = TestAccountBuilder.nonExistentNino
val badRequestNino: Nino = TestAccountBuilder.blankNino
"CitizenDetailsService" should {
"return something for valid NINO" in {
val person: Future[Either[CitizenDetailsError, CitizenDetailsResponse]] = MockCitizenDetailsService.retrievePerson(nino)(new HeaderCarrier())
whenReady(person) {p =>
p.isRight shouldBe true
}
}
"return None for bad NINO" in {
val person: Future[Either[CitizenDetailsError, CitizenDetailsResponse]] = MockCitizenDetailsService.retrievePerson(nonExistentNino)(new HeaderCarrier())
whenReady(person) {p =>
p.isLeft shouldBe true
}
}
"return None for bad request" in {
val person: Future[Either[CitizenDetailsError, CitizenDetailsResponse]] = MockCitizenDetailsService.retrievePerson(badRequestNino)(new HeaderCarrier())
whenReady(person) {p =>
p.isLeft shouldBe true
}
}
"return a Failed Future for a 5XX error" in {
val person: Future[Either[CitizenDetailsError, CitizenDetailsResponse]] = MockCitizenDetailsService.retrievePerson(TestAccountBuilder.internalServerError)(new HeaderCarrier())
whenReady(person.failed) { ex =>
ex shouldBe a [Upstream5xxResponse]
}
}
"return correct name and Date of Birth for NINO" in {
val person: Future[Either[CitizenDetailsError, CitizenDetailsResponse]] = MockCitizenDetailsService.retrievePerson(nino)(new HeaderCarrier())
whenReady(person) {p =>
p.right.map(_.person.copy(nino = nino)) shouldBe Right(Citizen(nino, Some("AHMED"), Some("BRENNAN"), new LocalDate(1954, 3, 9)))
p.right.get.person.getNameFormatted shouldBe Some("AHMED BRENNAN")
}
}
"return formatted name of None if Citizen returns without a name" in {
val person: Future[Either[CitizenDetailsError, CitizenDetailsResponse]] = MockCitizenDetailsService.retrievePerson(noNameNino)(new HeaderCarrier())
whenReady(person) {p =>
p shouldBe Right(CitizenDetailsResponse(Citizen(noNameNino, None, None, new LocalDate(1954, 3, 9)), Some(Address(Some("GREAT BRITAIN")))))
p.right.get.person.getNameFormatted shouldBe None
}
}
}
}
Example 62
| Source File: NispFrontendControllerSpec.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.controllers
import org.slf4j.{Logger => Slf4JLogger}
import org.mockito.Mockito.{verify, when}
import org.mockito.ArgumentMatchers._
import org.scalatest.mock.MockitoSugar
import org.scalatestplus.play.OneAppPerSuite
import play.api.Logger
import play.api.http.Status
import play.api.mvc.Result
import play.api.test.FakeRequest
import uk.gov.hmrc.nisp.config.{ApplicationGlobal, ApplicationGlobalTrait}
import uk.gov.hmrc.nisp.helpers.{MockApplicationGlobal, MockCachedStaticHtmlPartialRetriever}
import uk.gov.hmrc.nisp.utils.MockTemplateRenderer
import uk.gov.hmrc.play.test.UnitSpec
import uk.gov.hmrc.renderer.TemplateRenderer
class NispFrontendControllerSpec extends UnitSpec with MockitoSugar with OneAppPerSuite {
val mockLogger: Slf4JLogger = mock[Slf4JLogger]
when(mockLogger.isErrorEnabled).thenReturn(true)
def controller = new NispFrontendController {
override val logger = new Logger(mockLogger)
val cachedStaticHtmlPartialRetriever = MockCachedStaticHtmlPartialRetriever
override implicit val templateRenderer: TemplateRenderer = MockTemplateRenderer
override val applicationGlobal:ApplicationGlobalTrait = MockApplicationGlobal
}
implicit val request = FakeRequest()
"onError" should {
"should log error details" in {
val result: Result = controller.onError(new Exception())
verify(mockLogger).error(anyString(), any[Exception])
}
"should return an Internal Server Error (500)" in {
val result: Result = controller.onError(new Exception())
status(result) shouldBe Status.INTERNAL_SERVER_ERROR
}
}
}
Example 63
| Source File: GARedirectControllerSpec.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.controllers
import org.scalatest.mock.MockitoSugar
import org.scalatestplus.play.{OneAppPerSuite, PlaySpec}
import play.api.http._
import play.api.i18n.Lang
import play.api.test.FakeRequest
import play.api.test.Helpers._
import uk.gov.hmrc.nisp.config.wiring.NispFormPartialRetriever
import uk.gov.hmrc.nisp.helpers.MockCachedStaticHtmlPartialRetriever
import uk.gov.hmrc.nisp.utils.MockTemplateRenderer
import uk.gov.hmrc.play.partials.CachedStaticHtmlPartialRetriever
import uk.gov.hmrc.renderer.TemplateRenderer
class GARedirectControllerSpec extends PlaySpec with MockitoSugar with OneAppPerSuite {
private implicit val fakeRequest = FakeRequest("GET", "/redirect")
private implicit val lang = Lang("en")
private implicit val retriever = MockCachedStaticHtmlPartialRetriever
implicit val formPartialRetriever: uk.gov.hmrc.play.partials.FormPartialRetriever = NispFormPartialRetriever
implicit val templateRenderer: TemplateRenderer = MockTemplateRenderer
val testGARedirectController = new GARedirectController {
override implicit val cachedStaticHtmlPartialRetriever: CachedStaticHtmlPartialRetriever = retriever
override implicit val templateRenderer: TemplateRenderer = MockTemplateRenderer
}
"GET /redirect" should {
"return 200" in {
val result = testGARedirectController.show(fakeRequest)
status(result) mustBe Status.OK
}
}
}
Example 64
| Source File: BackendConnectorSpec.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.connectors
import org.mockito.Mockito.when
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.mock.MockitoSugar
import play.api.libs.json.Json
import uk.gov.hmrc.http.cache.client.SessionCache
import uk.gov.hmrc.nisp.helpers.{MockMetricsService, MockSessionCache}
import uk.gov.hmrc.nisp.models.NationalInsuranceRecord
import uk.gov.hmrc.nisp.models.enums.APIType
import uk.gov.hmrc.nisp.services.MetricsService
import uk.gov.hmrc.nisp.utils.JsonDepersonaliser
import uk.gov.hmrc.play.test.UnitSpec
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.{Failure, Success}
import uk.gov.hmrc.http.{HeaderCarrier, HttpGet, HttpResponse}
class BackendConnectorSpec extends UnitSpec with MockitoSugar with ScalaFutures {
val mockHttp: HttpGet = mock[HttpGet]
object BackendConnectorImpl extends BackendConnector {
override def http: HttpGet = mockHttp
override def sessionCache: SessionCache = MockSessionCache
override def serviceUrl: String = "national-insurance"
override val metricsService: MetricsService = MockMetricsService
def getNationalInsurance()(implicit headerCarrier: HeaderCarrier): Future[NationalInsuranceRecord] = {
val urlToRead = s"$serviceUrl/ni"
retrieveFromCache[NationalInsuranceRecord](APIType.NationalInsurance, urlToRead)(headerCarrier, NationalInsuranceRecord.formats)
}
}
implicit val headerCarrier = HeaderCarrier(extraHeaders = Seq("Accept" -> "application/vnd.hmrc.1.0+json"))
"connectToMicroservice" should {
"should return depersonalised JSON" in {
val json = Json.obj(
"qualifyingYearsPriorTo1975" -> 0,
"numberOfGaps" -> 6,
"numberOfGapsPayable" -> 4,
"dateOfEntry" -> "1975-08-01",
"homeResponsibilitiesProtection" -> false,
"earningsIncludedUpTo" -> "2016-04-05",
"_embedded" -> Json.obj(
"taxYears" -> Json.arr()
)
)
val depersonalisedJson = JsonDepersonaliser.depersonalise(json) match {
case Success(s) => s
case Failure(_) => fail()
}
val Ok = 200
val response = Future(HttpResponse(Ok, Option.apply(json)))
when(mockHttp.GET[HttpResponse]("national-insurance/ni")).thenReturn(response)
val future: Future[NationalInsuranceRecord] = BackendConnectorImpl.getNationalInsurance()
whenReady(future.failed) {
t: Throwable =>
t.getMessage.contains(depersonalisedJson) shouldBe true
t.getMessage.contains("2016-04-05") shouldBe false
}
}
}
}
Example 65
| Source File: MockitoSparkContext.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import com.sap.spark.WithSparkContext
import org.mockito.Mockito._
import org.scalatest.Suite
import org.scalatest.mock.MockitoSugar
trait MockitoSparkContext extends WithSparkContext with MockitoSugar {
self: Suite =>
private var _sparkConf: SparkConf = _
private var _sc: SparkContext = _
override def sc: SparkContext = _sc
protected def mockSparkConf: SparkConf = _sparkConf
override protected def setUpSparkContext(): Unit = {
_sparkConf = sparkConf
_sc = mock[SparkContext](withSettings().stubOnly())
when(_sc.conf).thenReturn(_sparkConf)
when(_sc.getConf).thenReturn(_sparkConf)
when(_sc.ui).thenReturn(None)
}
override protected def tearDownSparkContext(): Unit = {
_sc.stop()
_sc = null
}
}
Example 66
| Source File: DescribeTableUsingSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql
import org.apache.spark.util.SqlContextConfigurationUtils
import org.scalatest.FunSuite
import org.mockito.Mockito._
import DatasourceResolver.withResolver
import org.apache.spark.sql.sources.{DatasourceCatalog, RelationInfo}
import org.scalatest.mock.MockitoSugar
class DescribeTableUsingSuite
extends FunSuite
with SqlContextConfigurationUtils
with GlobalSapSQLContext
with MockitoSugar {
test("DESCRIBE existing table") {
// simulate existing relation in the data source's catalog.
com.sap.spark.dstest.DefaultSource.addRelation("t1")
val actual = sqlContext.sql(s"DESCRIBE TABLE t1 USING com.sap.spark.dstest").collect
assertResult(Seq(Row("t1", "<DDL statement>")))(actual)
}
test("DESCRIBE non-existing table") {
val actual = sqlContext.sql(s"DESCRIBE TABLE nonExisting USING com.sap.spark.dstest").collect
assertResult(Seq(Row("", "")))(actual)
}
test("DESCRIBE TABLE works with case insensitivity") {
val provider = mock[DatasourceCatalog]
when(provider.getRelation(sqlc, Seq("FOO"), Map.empty))
.thenReturn(Some(RelationInfo("foo", isTemporary = false, "TABLE", Some("ddl"), "test")))
val resolver = mock[DatasourceResolver]
when(resolver.newInstanceOfTyped[DatasourceCatalog]("test"))
.thenReturn(provider)
withResolver(sqlc, resolver) {
withConf(SQLConf.CASE_SENSITIVE.key, "false") {
val values = sqlc.sql("DESCRIBE TABLE FOO USING test").collect().toSet
assertResult(Set(Row("foo", "ddl")))(values)
}
}
}
test("DESCRIBE TABLE works with case sensitivity") {
val provider = mock[DatasourceCatalog]
when(provider.getRelation(sqlc, Seq("FoO"), Map.empty))
.thenReturn(Some(RelationInfo("foo", isTemporary = false, "TABLE", Some("ddl"), "test")))
val resolver = mock[DatasourceResolver]
when(resolver.newInstanceOfTyped[DatasourceCatalog]("test"))
.thenReturn(provider)
withResolver(sqlc, resolver) {
withConf(SQLConf.CASE_SENSITIVE.key, "true") {
val values = sqlc.sql("DESCRIBE TABLE FoO USING test").collect().toSet
assertResult(Set(Row("foo", "ddl")))(values)
}
}
}
}
Example 67
| Source File: DropPartitionFunctionSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.execution.datasources
import org.apache.spark.sql.DatasourceResolver._
import org.apache.spark.sql._
import org.apache.spark.sql.sources._
import org.mockito.Mockito._
import org.mockito.internal.stubbing.answers.Returns
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class DropPartitionFunctionSuite extends FunSuite
with GlobalSapSQLContext
with MockitoSugar {
test("PartitionCatalog drop") {
val pfName = "pf"
val catalog = mock[PartitionCatalog]
val resolver = mock[DatasourceResolver]
when(resolver.newInstanceOf("foo")).thenAnswer(new Returns(catalog))
withResolver(sqlContext, resolver) {
val values =
sqlc.sql(
s"""DROP PARTITION FUNCTION $pfName
|USING foo""".stripMargin)
.collect()
verify(catalog, times(1)).dropPartitionFunction(pfName)
}
}
test("PartitioningFunctionProvider drop") {
val pfName = "pf"
val catalog = mock[PartitioningFunctionProvider]
val resolver = mock[DatasourceResolver]
when(resolver.newInstanceOf("foo")).thenAnswer(new Returns(catalog))
withResolver(sqlContext, resolver) {
val values =
sqlc.sql(
s"""DROP PARTITION FUNCTION $pfName
|USING foo""".stripMargin)
.collect()
verify(catalog, times(1)).dropPartitioningFunction(sqlContext, Map.empty, pfName, false)
}
}
}
Example 68
| Source File: CollapseExpandSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.catalyst.analysis.CollapseExpandSuite.SqlLikeCatalystSourceRelation
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions.{Attribute, Literal}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.sources.sql.SqlLikeRelation
import org.apache.spark.sql.sources.{BaseRelation, CatalystSource, Table}
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.util.PlanComparisonUtils._
import org.apache.spark.sql.{GlobalSapSQLContext, Row}
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class CollapseExpandSuite extends FunSuite with MockitoSugar with GlobalSapSQLContext {
case object Leaf extends LeafNode {
override def output: Seq[Attribute] = Seq.empty
}
test("Expansion with a single sequence of projections is correctly collapsed") {
val expand =
Expand(
Seq(Seq('a.string, Literal(1))),
Seq('a.string, 'gid.int),
Leaf)
val collapsed = CollapseExpand(expand)
assertResult(normalizeExprIds(Project(Seq('a.string, Literal(1) as "gid"), Leaf)))(
normalizeExprIds(collapsed))
}
test("Expansion with multiple projections is correctly collapsed") {
val expand =
Expand(
Seq(
Seq('a.string, Literal(1)),
Seq('b.string, Literal(1))),
Seq('a.string, 'gid1.int, 'b.string, 'gid2.int),
Leaf)
val collapsed = CollapseExpand(expand)
assertResult(
normalizeExprIds(
Project(Seq(
'a.string,
Literal(1) as "gid1",
'b.string,
Literal(1) as "gid2"),
Leaf)))(normalizeExprIds(collapsed))
}
test("Expand pushdown integration") {
val relation = mock[SqlLikeCatalystSourceRelation]
when(relation.supportsLogicalPlan(any[Expand]))
.thenReturn(true)
when(relation.isMultiplePartitionExecution(any[Seq[CatalystSource]]))
.thenReturn(true)
when(relation.schema)
.thenReturn(StructType(StructField("foo", StringType) :: Nil))
when(relation.relationName)
.thenReturn("t")
when(relation.logicalPlanToRDD(any[LogicalPlan]))
.thenReturn(sc.parallelize(Seq(Row("a", 1), Row("b", 1), Row("a", 1))))
sqlc.baseRelationToDataFrame(relation).registerTempTable("t")
val dataFrame = sqlc.sql("SELECT COUNT(DISTINCT foo) FROM t")
val Seq(Row(ct)) = dataFrame.collect().toSeq
assertResult(2)(ct)
}
}
object CollapseExpandSuite {
abstract class SqlLikeCatalystSourceRelation
extends BaseRelation
with Table
with SqlLikeRelation
with CatalystSource
}
Example 69
| Source File: UseAliasesForAggregationsInGroupingsSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.plans.logical.Aggregate
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.sources.BaseRelation
import org.apache.spark.sql.types._
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class UseAliasesForAggregationsInGroupingsSuite extends FunSuite with MockitoSugar {
val br1 = new BaseRelation {
override def sqlContext: SQLContext = mock[SQLContext]
override def schema: StructType = StructType(Seq(
StructField("name", StringType),
StructField("age", IntegerType)
))
}
val lr1 = LogicalRelation(br1)
val nameAtt = lr1.output.find(_.name == "name").get
val ageAtt = lr1.output.find(_.name == "age").get
test("replace functions in group by") {
val avgExpr = avg(ageAtt)
val avgAlias = avgExpr as 'avgAlias
assertResult(
lr1.groupBy(avgAlias.toAttribute)(avgAlias)
)(UseAliasesForFunctionsInGroupings(
lr1.groupBy(avgExpr)(avgAlias))
)
assertResult(
lr1.select(ageAtt)
)(UseAliasesForFunctionsInGroupings(
lr1.select(ageAtt))
)
intercept[RuntimeException](
UseAliasesForFunctionsInGroupings(Aggregate(Seq(avgExpr), Seq(ageAtt), lr1))
)
}
}
Example 70
| Source File: RemoveNestedAliasesSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.analysis
import com.sap.spark.PlanTest
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions.Alias
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.sources.BaseRelation
import org.apache.spark.sql.types._
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class RemoveNestedAliasesSuite extends FunSuite with MockitoSugar with PlanTest {
val br1 = new BaseRelation {
override def sqlContext: SQLContext = mock[SQLContext]
override def schema: StructType = StructType(Seq(
StructField("name", StringType),
StructField("age", IntegerType)
))
}
val lr1 = LogicalRelation(br1)
val nameAtt = lr1.output.find(_.name == "name").get
val ageAtt = lr1.output.find(_.name == "age").get
test("Replace alias into aliases") {
val avgExpr = avg(ageAtt)
val avgAlias = avgExpr as 'avgAlias
val aliasAlias = avgAlias as 'aliasAlias
val aliasAliasAlias = aliasAlias as 'aliasAliasAlias
val copiedAlias = Alias(avgExpr, aliasAlias.name)(
exprId = aliasAlias.exprId
)
val copiedAlias2 = Alias(avgExpr, aliasAliasAlias.name)(
exprId = aliasAliasAlias.exprId
)
assertResult(
lr1.groupBy(avgAlias.toAttribute)(avgAlias)
)(RemoveNestedAliases(lr1.groupBy(avgAlias.toAttribute)(avgAlias)))
assertResult(
lr1.groupBy(copiedAlias.toAttribute)(copiedAlias)
)(RemoveNestedAliases(lr1.groupBy(aliasAlias.toAttribute)(aliasAlias)))
assertResult(
lr1.groupBy(copiedAlias2.toAttribute)(copiedAlias2)
)(RemoveNestedAliases(lr1.groupBy(aliasAliasAlias.toAttribute)(aliasAliasAlias)))
}
test("Replace alias into expressions") {
val ageAlias = ageAtt as 'ageAlias
val avgExpr = avg(ageAlias) as 'avgAlias
val correctedAvgExpr = avg(ageAtt) as 'avgAlias
comparePlans(
lr1.groupBy(correctedAvgExpr.toAttribute)(correctedAvgExpr),
RemoveNestedAliases(lr1.groupBy(avgExpr.toAttribute)(avgExpr))
)
}
}
Example 71
| Source File: ResolveHierarchySuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions.{Attribute, EqualTo}
import org.apache.spark.sql.catalyst.plans.logical.{AdjacencyListHierarchySpec, Hierarchy}
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.sources.BaseRelation
import org.apache.spark.sql.types._
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class ResolveHierarchySuite extends FunSuite with MockitoSugar {
val br1 = new BaseRelation {
override def sqlContext: SQLContext = mock[SQLContext]
override def schema: StructType = StructType(Seq(
StructField("id", IntegerType),
StructField("parent", IntegerType)
))
}
val lr1 = LogicalRelation(br1)
val idAtt = lr1.output.find(_.name == "id").get
val parentAtt = lr1.output.find(_.name == "parent").get
test("Check parenthood expression has no conflicting expression IDs and qualifiers") {
val source = SimpleAnalyzer.execute(lr1.select('id, 'parent).subquery('u))
assert(source.resolved)
val hierarchy = Hierarchy(
AdjacencyListHierarchySpec(source, "v",
UnresolvedAttribute("u" :: "id" :: Nil) === UnresolvedAttribute("v" :: "id" :: Nil),
Some('id.isNull), Nil),
'node
)
val resolveHierarchy = ResolveHierarchy(SimpleAnalyzer)
val resolveReferences = ResolveReferencesWithHierarchies(SimpleAnalyzer)
val resolvedHierarchy = (0 to 10).foldLeft(hierarchy: Hierarchy) { (h, _) =>
SimpleAnalyzer.ResolveReferences(
resolveReferences(resolveHierarchy(h))
).asInstanceOf[Hierarchy]
}
assert(resolvedHierarchy.node.resolved)
val resolvedSpec = resolvedHierarchy.spec.asInstanceOf[AdjacencyListHierarchySpec]
assert(resolvedSpec.parenthoodExp.resolved)
assert(resolvedSpec.startWhere.forall(_.resolved))
assert(resolvedHierarchy.childrenResolved)
assert(resolvedHierarchy.resolved)
val parenthoodExpression = resolvedSpec.parenthoodExp.asInstanceOf[EqualTo]
assertResult("u" :: Nil)(parenthoodExpression.left.asInstanceOf[Attribute].qualifiers)
assertResult("v" :: Nil)(parenthoodExpression.right.asInstanceOf[Attribute].qualifiers)
assert(parenthoodExpression.right.asInstanceOf[Attribute].exprId !=
source.output.find(_.name == "id").get.exprId)
}
}
Example 72
| Source File: ResolveAnnotationsSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical.Project
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.sources.BaseRelation
import org.apache.spark.sql.types._
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
import org.apache.spark.sql.catalyst.dsl.plans._
class ResolveAnnotationsSuite extends FunSuite with MockitoSugar {
// scalastyle:off magic.number
val annotatedRel1 = new BaseRelation {
override def sqlContext: SQLContext = mock[SQLContext]
override def schema: StructType = StructType(Seq(
StructField("id1.1", IntegerType, metadata =
new MetadataBuilder().putLong("key1.1", 11L).build()),
StructField("id1.2", IntegerType, metadata =
new MetadataBuilder()
.putLong("key1.2", 12L)
.putLong("key1.3", 13).build()))
)
}
val lr1 = LogicalRelation(annotatedRel1)
val id11Att = lr1.output.find(_.name == "id1.1").get
val id12Att = lr1.output.find(_.name == "id1.2").get
val id11AnnotatedAtt = AnnotatedAttribute(id11Att)(
Map("key1.1" -> Literal.create(100L, LongType), // override the old key
"newkey" -> Literal.create(200L, LongType))) // define a new key
val simpleAnnotatedSelect = lr1.select(id11AnnotatedAtt)
}
Example 73
| Source File: ResolveCountDistinctStarSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference}
import org.apache.spark.sql.catalyst.plans.logical.Aggregate
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.sources.BaseRelation
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.scalatest.FunSuite
import org.scalatest.Inside._
import org.scalatest.mock.MockitoSugar
import org.apache.spark.sql.catalyst.dsl.plans.DslLogicalPlan
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete, Count}
import scala.collection.mutable.ArrayBuffer
class ResolveCountDistinctStarSuite extends FunSuite with MockitoSugar {
val persons = new LogicalRelation(new BaseRelation {
override def sqlContext: SQLContext = mock[SQLContext]
override def schema: StructType = StructType(Seq(
StructField("age", IntegerType),
StructField("name", StringType)
))
})
test("Count distinct star is resolved correctly") {
val projection = persons.select(UnresolvedAlias(
AggregateExpression(Count(UnresolvedStar(None) :: Nil), Complete, true)))
val stillNotCompletelyResolvedAggregate = SimpleAnalyzer.execute(projection)
val resolvedAggregate = ResolveCountDistinctStar(SimpleAnalyzer)
.apply(stillNotCompletelyResolvedAggregate)
inside(resolvedAggregate) {
case Aggregate(Nil,
ArrayBuffer(Alias(AggregateExpression(Count(expressions), Complete, true), _)), _) =>
assert(expressions.collect {
case a:AttributeReference => a.name
}.toSet == Set("name", "age"))
}
assert(resolvedAggregate.resolved)
}
}
Example 74
| Source File: DatasourceResolverSuite.scala From HANAVora-Extensions with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import com.sap.spark.dstest.DefaultSource
import org.apache.spark.sql.{DatasourceResolver, DefaultDatasourceResolver, GlobalSapSQLContext}
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class DatasourceResolverSuite extends FunSuite with GlobalSapSQLContext with MockitoSugar {
test("DataSourceResolver looks up classes correclty") {
val clazz = new DefaultDatasourceResolver().lookup("com.sap.spark.dstest")
assertResult(classOf[DefaultSource])(clazz)
}
test("DefaultDatasourceResolver creates a new instance of a given provider correclty") {
val instance =
new DefaultDatasourceResolver().newInstanceOfTyped[DefaultSource]("com.sap.spark.dstest")
assert(instance.isInstanceOf[DefaultSource])
}
test("withResolver uses the correct resolver") {
val resolver = mock[DatasourceResolver]
DatasourceResolver.withResolver(sqlContext, resolver) {
assertResult(DatasourceResolver.resolverFor(sqlContext))(resolver)
}
assertResult(DatasourceResolver.resolverFor(sqlContext))(DefaultDatasourceResolver)
}
test("An exception is thrown if the given class does not exist") {
intercept[ClassNotFoundException] {
DatasourceResolver.resolverFor(sqlc).newInstanceOf("should.definitely.not.exist")
}
}
}
Example 75
| Source File: JarsHelpersTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.core.helpers
import java.io.File
import org.junit.runner.RunWith
import org.mockito.Mockito._
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
import scala.collection.mutable
@RunWith(classOf[JUnitRunner])
class JarsHelpersTest extends FlatSpec with Matchers with MockitoSugar {
val file = mock[File]
when(file.exists).thenReturn(true)
when(file.listFiles()).thenReturn(Array(
new File("first.jar"),
new File("second.jar"),
new File("sparta-driver.jar")))
it should "find the driver jar" in {
val seqofJars = JarsHelper.findDriverByPath(
file)
seqofJars should be (mutable.ArraySeq(new File("sparta-driver.jar")))
}
val fileNoSpartaDriver = mock[File]
when(fileNoSpartaDriver.exists).thenReturn(true)
when(fileNoSpartaDriver.listFiles()).thenReturn(Array(
new File("sparta.jar"),
new File("driver.jar"),
new File("sparta-driver.txt"))
)
it should "return an empty sequence" in {
val retrievedDrivers = JarsHelper.findDriverByPath(fileNoSpartaDriver)
retrievedDrivers should equal(Seq.empty)
}
}
Example 76
| Source File: PolicyHelperTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.core.helpers
import com.stratio.sparta.serving.core.models.policy.{OutputFieldsModel, PolicyModel, UserJar}
import com.stratio.sparta.serving.core.utils.BaseUtilsTest
import org.junit.runner.RunWith
import org.mockito.Mockito._
import org.scalatest.junit.JUnitRunner
import org.scalatest._
import org.scalatest.mock.MockitoSugar
@RunWith(classOf[JUnitRunner])
class PolicyHelperTest extends WordSpecLike with Matchers with MockitoSugar {
val aggModel: PolicyModel = mock[PolicyModel]
"PolicyHelper" should {
"return files" in {
when(aggModel.userPluginsJars).thenReturn(Seq(UserJar("path1"), UserJar("path2")))
val files = PolicyHelper.jarsFromPolicy(aggModel)
files.map(_.getName) shouldBe Seq("path1", "path2")
}
"return empty Seq" in {
when(aggModel.userPluginsJars).thenReturn(Seq(UserJar("")))
val files = PolicyHelper.jarsFromPolicy(aggModel)
files.size shouldBe 0
}
}
}
Example 77
| Source File: HdfsUtilsTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.core.utils
import java.io.{FileNotFoundException, InputStream}
import org.apache.hadoop.fs.{FileSystem, _}
import org.junit.runner.RunWith
import org.mockito.Mockito._
import org.scalatest._
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import scala.util.{Failure, Try}
@RunWith(classOf[JUnitRunner])
class HdfsUtilsTest extends FlatSpec with ShouldMatchers with MockitoSugar {
val fileSystem: FileSystem = mock[FileSystem]
val utils = new HdfsUtils(fileSystem, "stratio")
"hdfs utils" should "getfiles from a path" in {
val expected = Array(mock[FileStatus])
when(fileSystem.listStatus(new Path("myTestPath"))).thenReturn(expected)
val result = utils.getFiles("myTestPath")
result should be(expected)
}
it should "return single file as inputStream" in {
val expected: InputStream = mock[FSDataInputStream]
when(fileSystem.open(new Path("testFile"))).thenReturn(expected.asInstanceOf[FSDataInputStream])
val result: InputStream = utils.getFile("testFile")
result should be(expected)
}
it should "write" in {
val result = Try(utils.write("from", "to", true)) match {
case Failure(ex: Throwable) => ex
}
result.isInstanceOf[FileNotFoundException] should be(true)
}
it should "write without override" in {
val result = Try(utils.write("from", "to", false)) match {
case Failure(ex: Throwable) => ex
}
result.isInstanceOf[FileNotFoundException] should be(true)
}
}
Example 78
| Source File: RawStageTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.driver.test.stage
import akka.actor.ActorSystem
import akka.testkit.TestKit
import com.stratio.sparta.driver.stage.{LogError, RawDataStage}
import com.stratio.sparta.sdk.pipeline.autoCalculations.AutoCalculatedField
import com.stratio.sparta.sdk.properties.JsoneyString
import com.stratio.sparta.serving.core.models.policy.writer.{AutoCalculatedFieldModel, WriterModel}
import com.stratio.sparta.serving.core.models.policy.{PolicyModel, RawDataModel}
import org.junit.runner.RunWith
import org.mockito.Mockito.when
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpecLike, ShouldMatchers}
@RunWith(classOf[JUnitRunner])
class RawStageTest
extends TestKit(ActorSystem("RawStageTest"))
with FlatSpecLike with ShouldMatchers with MockitoSugar {
case class TestRawData(policy: PolicyModel) extends RawDataStage with LogError
def mockPolicy: PolicyModel = {
val policy = mock[PolicyModel]
when(policy.id).thenReturn(Some("id"))
policy
}
"rawDataStage" should "Generate a raw data" in {
val field = "field"
val timeField = "time"
val tableName = Some("table")
val outputs = Seq("output")
val partitionBy = Some("field")
val autocalculateFields = Seq(AutoCalculatedFieldModel())
val configuration = Map.empty[String, JsoneyString]
val policy = mockPolicy
val rawData = mock[RawDataModel]
val writerModel = mock[WriterModel]
when(policy.rawData).thenReturn(Some(rawData))
when(rawData.dataField).thenReturn(field)
when(rawData.timeField).thenReturn(timeField)
when(rawData.writer).thenReturn(writerModel)
when(writerModel.tableName).thenReturn(tableName)
when(writerModel.outputs).thenReturn(outputs)
when(writerModel.partitionBy).thenReturn(partitionBy)
when(writerModel.autoCalculatedFields).thenReturn(autocalculateFields)
when(rawData.configuration).thenReturn(configuration)
val result = TestRawData(policy).rawDataStage()
result.timeField should be(timeField)
result.dataField should be(field)
result.writerOptions.tableName should be(tableName)
result.writerOptions.partitionBy should be(partitionBy)
result.configuration should be(configuration)
result.writerOptions.outputs should be(outputs)
}
"rawDataStage" should "Fail with bad table name" in {
val field = "field"
val timeField = "time"
val tableName = None
val outputs = Seq("output")
val partitionBy = Some("field")
val configuration = Map.empty[String, JsoneyString]
val policy = mockPolicy
val rawData = mock[RawDataModel]
val writerModel = mock[WriterModel]
when(policy.rawData).thenReturn(Some(rawData))
when(rawData.dataField).thenReturn(field)
when(rawData.timeField).thenReturn(timeField)
when(rawData.writer).thenReturn(writerModel)
when(writerModel.tableName).thenReturn(tableName)
when(writerModel.outputs).thenReturn(outputs)
when(writerModel.partitionBy).thenReturn(partitionBy)
when(rawData.configuration).thenReturn(configuration)
the[IllegalArgumentException] thrownBy {
TestRawData(policy).rawDataStage()
} should have message "Something gone wrong saving the raw data. Please re-check the policy."
}
}
Example 79
| Source File: DriverActorTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.api.actor
import java.nio.file.{Files, Path}
import akka.actor.{ActorSystem, Props}
import akka.testkit.{DefaultTimeout, ImplicitSender, TestKit}
import akka.util.Timeout
import com.stratio.sparta.serving.api.actor.DriverActor.UploadDrivers
import com.stratio.sparta.serving.core.config.{SpartaConfig, SpartaConfigFactory}
import com.stratio.sparta.serving.core.models.SpartaSerializer
import com.stratio.sparta.serving.core.models.files.{SpartaFile, SpartaFilesResponse}
import com.typesafe.config.{Config, ConfigFactory}
import org.junit.runner.RunWith
import org.scalatest._
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import spray.http.BodyPart
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.{Failure, Success}
@RunWith(classOf[JUnitRunner])
class DriverActorTest extends TestKit(ActorSystem("PluginActorSpec"))
with DefaultTimeout
with ImplicitSender
with WordSpecLike
with Matchers
with BeforeAndAfterAll
with BeforeAndAfterEach
with MockitoSugar with SpartaSerializer {
val tempDir: Path = Files.createTempDirectory("test")
tempDir.toFile.deleteOnExit()
val localConfig: Config = ConfigFactory.parseString(
s"""
|sparta{
| api {
| host = local
| port= 7777
| }
|}
|
|sparta.config.driverPackageLocation = "$tempDir"
""".stripMargin)
val fileList = Seq(BodyPart("reference.conf", "file"))
override def beforeEach(): Unit = {
SpartaConfig.initMainConfig(Option(localConfig), SpartaConfigFactory(localConfig))
SpartaConfig.initApiConfig()
}
override def afterAll: Unit = {
shutdown()
}
override implicit val timeout: Timeout = Timeout(15 seconds)
"DriverActor " must {
"Not save files with wrong extension" in {
val driverActor = system.actorOf(Props(new DriverActor()))
driverActor ! UploadDrivers(fileList)
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.isEmpty shouldBe true
}
}
"Not upload empty files" in {
val driverActor = system.actorOf(Props(new DriverActor()))
driverActor ! UploadDrivers(Seq.empty)
expectMsgPF() {
case SpartaFilesResponse(Failure(f)) => f.getMessage shouldBe "At least one file is expected"
}
}
"Save a file" in {
val driverActor = system.actorOf(Props(new DriverActor()))
driverActor ! UploadDrivers(Seq(BodyPart("reference.conf", "file.jar")))
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.head.fileName.endsWith("file.jar") shouldBe true
}
}
}
}
Example 80
| Source File: PluginActorTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.api.actor
import java.nio.file.{Files, Path}
import akka.actor.{ActorSystem, Props}
import akka.testkit.{DefaultTimeout, ImplicitSender, TestKit}
import akka.util.Timeout
import com.stratio.sparta.serving.api.actor.PluginActor.{PluginResponse, UploadPlugins}
import com.stratio.sparta.serving.api.constants.HttpConstant
import com.stratio.sparta.serving.core.config.{SpartaConfig, SpartaConfigFactory}
import com.stratio.sparta.serving.core.models.SpartaSerializer
import com.stratio.sparta.serving.core.models.files.{SpartaFile, SpartaFilesResponse}
import com.typesafe.config.{Config, ConfigFactory}
import org.junit.runner.RunWith
import org.scalatest._
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import spray.http.BodyPart
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.{Failure, Success}
@RunWith(classOf[JUnitRunner])
class PluginActorTest extends TestKit(ActorSystem("PluginActorSpec"))
with DefaultTimeout
with ImplicitSender
with WordSpecLike
with Matchers
with BeforeAndAfterAll
with BeforeAndAfterEach
with MockitoSugar with SpartaSerializer {
val tempDir: Path = Files.createTempDirectory("test")
tempDir.toFile.deleteOnExit()
val localConfig: Config = ConfigFactory.parseString(
s"""
|sparta{
| api {
| host = local
| port= 7777
| }
|}
|
|sparta.config.pluginPackageLocation = "$tempDir"
""".stripMargin)
val fileList = Seq(BodyPart("reference.conf", "file"))
override def beforeEach(): Unit = {
SpartaConfig.initMainConfig(Option(localConfig), SpartaConfigFactory(localConfig))
SpartaConfig.initApiConfig()
}
override def afterAll: Unit = {
shutdown()
}
override implicit val timeout: Timeout = Timeout(15 seconds)
"PluginActor " must {
"Not save files with wrong extension" in {
val pluginActor = system.actorOf(Props(new PluginActor()))
pluginActor ! UploadPlugins(fileList)
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.isEmpty shouldBe true
}
}
"Not upload empty files" in {
val pluginActor = system.actorOf(Props(new PluginActor()))
pluginActor ! UploadPlugins(Seq.empty)
expectMsgPF() {
case SpartaFilesResponse(Failure(f)) => f.getMessage shouldBe "At least one file is expected"
}
}
"Save a file" in {
val pluginActor = system.actorOf(Props(new PluginActor()))
pluginActor ! UploadPlugins(Seq(BodyPart("reference.conf", "file.jar")))
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.head.fileName.endsWith("file.jar") shouldBe true
}
}
}
}
Example 81
| Source File: CassandraOutputTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.plugin.output.cassandra
import java.io.{Serializable => JSerializable}
import com.datastax.spark.connector.cql.CassandraConnector
import com.stratio.sparta.sdk._
import com.stratio.sparta.sdk.properties.JsoneyString
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
@RunWith(classOf[JUnitRunner])
class CassandraOutputTest extends FlatSpec with Matchers with MockitoSugar with AnswerSugar {
val s = "sum"
val properties = Map(("connectionHost", "127.0.0.1"), ("connectionPort", "9042"))
"getSparkConfiguration" should "return a Seq with the configuration" in {
val configuration = Map(("connectionHost", "127.0.0.1"), ("connectionPort", "9042"))
val cass = CassandraOutput.getSparkConfiguration(configuration)
cass should be(List(("spark.cassandra.connection.host", "127.0.0.1"), ("spark.cassandra.connection.port", "9042")))
}
"getSparkConfiguration" should "return all cassandra-spark config" in {
val config: Map[String, JSerializable] = Map(
("sparkProperties" -> JsoneyString(
"[{\"sparkPropertyKey\":\"spark.cassandra.input.fetch.size_in_rows\",\"sparkPropertyValue\":\"2000\"}," +
"{\"sparkPropertyKey\":\"spark.cassandra.input.split.size_in_mb\",\"sparkPropertyValue\":\"64\"}]")),
("anotherProperty" -> "true")
)
val sparkConfig = CassandraOutput.getSparkConfiguration(config)
sparkConfig.exists(_ == ("spark.cassandra.input.fetch.size_in_rows" -> "2000")) should be(true)
sparkConfig.exists(_ == ("spark.cassandra.input.split.size_in_mb" -> "64")) should be(true)
sparkConfig.exists(_ == ("anotherProperty" -> "true")) should be(false)
}
"getSparkConfiguration" should "not return cassandra-spark config" in {
val config: Map[String, JSerializable] = Map(
("hadoopProperties" -> JsoneyString(
"[{\"sparkPropertyKey\":\"spark.cassandra.input.fetch.size_in_rows\",\"sparkPropertyValue\":\"2000\"}," +
"{\"sparkPropertyKey\":\"spark.cassandra.input.split.size_in_mb\",\"sparkPropertyValue\":\"64\"}]")),
("anotherProperty" -> "true")
)
val sparkConfig = CassandraOutput.getSparkConfiguration(config)
sparkConfig.exists(_ == ("spark.cassandra.input.fetch.size_in_rows" -> "2000")) should be(false)
sparkConfig.exists(_ == ("spark.cassandra.input.split.size_in_mb" -> "64")) should be(false)
sparkConfig.exists(_ == ("anotherProperty" -> "true")) should be(false)
}
}
Example 82
| Source File: MessageHandlerTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.plugin.input.rabbitmq.handler
import com.rabbitmq.client.QueueingConsumer.Delivery
import com.stratio.sparta.plugin.input.rabbitmq.handler.MessageHandler.{ByteArrayMessageHandler, StringMessageHandler}
import org.junit.runner.RunWith
import org.mockito.Mockito._
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, WordSpec}
@RunWith(classOf[JUnitRunner])
class MessageHandlerTest extends WordSpec with Matchers with MockitoSugar {
val message = "This is the message for testing"
"RabbitMQ MessageHandler Factory " should {
"Get correct handler for string with a map " in {
MessageHandler(Map(MessageHandler.KeyDeserializer -> "")) should matchPattern { case StringMessageHandler => }
MessageHandler(Map.empty[String, String]) should matchPattern { case StringMessageHandler => }
MessageHandler(Map(MessageHandler.KeyDeserializer -> "badInput")) should
matchPattern { case StringMessageHandler => }
MessageHandler(Map(MessageHandler.KeyDeserializer -> "arraybyte")) should
matchPattern { case ByteArrayMessageHandler => }
}
"Get correct handler for string " in {
val result = MessageHandler("string")
result should matchPattern { case StringMessageHandler => }
}
"Get correct handler for arraybyte " in {
val result = MessageHandler("arraybyte")
result should matchPattern { case ByteArrayMessageHandler => }
}
"Get correct handler for empty input " in {
val result = MessageHandler("")
result should matchPattern { case StringMessageHandler => }
}
"Get correct handler for bad input " in {
val result = MessageHandler("badInput")
result should matchPattern { case StringMessageHandler => }
}
}
"StringMessageHandler " should {
"Handle strings" in {
val delivery = mock[Delivery]
when(delivery.getBody).thenReturn(message.getBytes)
val result = StringMessageHandler.handler(delivery)
verify(delivery, times(1)).getBody
result.getString(0) shouldBe message
}
}
"ByteArrayMessageHandler " should {
"Handle bytes" in {
val delivery = mock[Delivery]
when(delivery.getBody).thenReturn(message.getBytes)
val result = ByteArrayMessageHandler.handler(delivery)
verify(delivery, times(1)).getBody
result.getAs[Array[Byte]](0) shouldBe message.getBytes
}
}
}
Example 83
| Source File: JsonPathExtractorTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.plugin.transformation.json
import com.jayway.jsonpath.PathNotFoundException
import org.scalatest._
import org.scalatest.mock.MockitoSugar
class JsonPathExtractorTest extends FlatSpec with ShouldMatchers with MockitoSugar {
val JSON = """{ "store": {
| "book": [
| { "category": "reference",
| "author": "Nigel Rees",
| "title": "Sayings of the Century",
| "price": 8.95
| },
| { "category": "fiction",
| "author": "Evelyn Waugh",
| "title": "Sword of Honour",
| "price": 12.99
| },
| { "category": "fiction",
| "author": "Herman Melville",
| "title": "Moby Dick",
| "isbn": "0-553-21311-3",
| "price": 8.99
| },
| { "category": "fiction",
| "author": "J. R. R. Tolkien",
| "title": "The Lord of the Rings",
| "isbn": "0-395-19395-8",
| "price": 22.99
| }
| ],
| "bicycle": {
| "color": "red",
| "price": 19.95
| }
| }
|}""".stripMargin
it should "return bicycle color with dot-notation query" in {
val query = "$.store.bicycle.color"
val result = new JsonPathExtractor(JSON, false).query(query)
result.asInstanceOf[String] should be("red")
}
it should "return bicycle color with bracket-notation query" in {
val query = "$['store']['bicycle']['color']"
val result = new JsonPathExtractor(JSON, false).query(query)
result.asInstanceOf[String] should be("red")
}
it should "return bicycle price" in {
val query = "$.store.bicycle.price"
val result = new JsonPathExtractor(JSON, false).query(query)
result.asInstanceOf[Double] should be(19.95)
}
it should "return null with leaf" in {
val query = "$.store.bicycle.bad"
val result = new JsonPathExtractor(JSON, true).query(query)
result.asInstanceOf[String] should be(null)
}
it should "return exception without leaf" in {
val query = "$.store.bicycle.bad"
an[PathNotFoundException] should be thrownBy new JsonPathExtractor(JSON, false).query(query)
}
}
Example 84
| Source File: TwitterSourceSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.twitter
import java.time.Instant
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.twitter.TwitterSource.{Factory, MessageListener}
import org.mockito.Mockito._
import org.scalacheck.{Arbitrary, Gen}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, PropSpec}
import org.scalatest.prop.PropertyChecks
import twitter4j.{FilterQuery, TwitterStream}
class TwitterSourceSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
implicit val arbQuery: Arbitrary[Option[FilterQuery]] = Arbitrary {
Gen.oneOf(None, Some(new FilterQuery()))
}
property("TwitterSource should properly setup, poll message and teardown") {
forAll {
(query: Option[FilterQuery], startTime: Long) =>
val factory = mock[Factory]
val stream = mock[TwitterStream]
val listener = mock[MessageListener]
when(factory.getTwitterStream).thenReturn(stream)
val twitterSource = new TwitterSource(factory, query, listener)
twitterSource.open(MockUtil.mockTaskContext, Instant.ofEpochMilli(startTime))
verify(stream).addListener(listener)
query match {
case Some(q) =>
verify(stream).filter(q)
case None =>
verify(stream).sample()
}
twitterSource.read()
verify(listener).poll()
twitterSource.close()
verify(stream).shutdown()
}
}
}
Example 85
| Source File: HadoopCheckpointStoreIntegrationSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.hadoop
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.hadoop.lib.HadoopUtil
import org.apache.gearpump.streaming.hadoop.lib.rotation.FileSizeRotation
import org.apache.gearpump.streaming.task.TaskId
class HadoopCheckpointStoreIntegrationSpec
extends PropSpec with PropertyChecks with MockitoSugar with Matchers {
property("HadoopCheckpointStore should persist and recover checkpoints") {
val fileSizeGen = Gen.chooseNum[Int](100, 1000)
forAll(fileSizeGen) { (fileSize: Int) =>
val userConfig = UserConfig.empty
val taskContext = MockUtil.mockTaskContext
val hadoopConfig = new Configuration()
when(taskContext.appId).thenReturn(0)
when(taskContext.taskId).thenReturn(TaskId(0, 0))
val rootDirName = "test"
val rootDir = new Path(rootDirName + Path.SEPARATOR +
s"v${HadoopCheckpointStoreFactory.VERSION}")
val subDirName = "app0-task0_0"
val subDir = new Path(rootDir, subDirName)
val fs = HadoopUtil.getFileSystemForPath(rootDir, hadoopConfig)
fs.delete(rootDir, true)
fs.exists(rootDir) shouldBe false
val checkpointStoreFactory = new HadoopCheckpointStoreFactory(
rootDirName, hadoopConfig, new FileSizeRotation(fileSize))
val checkpointStore = checkpointStoreFactory.getCheckpointStore(subDirName)
checkpointStore.persist(0L, Array(0.toByte))
val tempFile = new Path(subDir, "checkpoints-0.store")
fs.exists(tempFile) shouldBe true
checkpointStore.persist(1L, Array.fill(fileSize)(0.toByte))
fs.exists(tempFile) shouldBe false
fs.exists(new Path(subDir, "checkpoints-0-1.store")) shouldBe true
checkpointStore.persist(2L, Array(0.toByte))
val newTempFile = new Path(subDir, "checkpoints-2.store")
fs.exists(newTempFile) shouldBe true
for (i <- 0 to 2) {
val optCp = checkpointStore.recover(i)
optCp should not be empty
}
fs.exists(newTempFile) shouldBe false
fs.exists(new Path(subDir, "checkpoints-2-2.store")) shouldBe true
checkpointStore.close()
fs.delete(rootDir, true)
fs.close()
}
}
}
Example 86
| Source File: HBaseSinkSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.external.hbase
import akka.actor.ActorSystem
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.external.hbase.HBaseSink.{HBaseWriter, HBaseWriterFactory}
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.task.TaskContext
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.TableName
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.hbase.util.Bytes
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
class HBaseSinkSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("HBaseSink should invoke HBaseWriter for writing message to HBase") {
val hbaseWriter = mock[HBaseWriter]
val hbaseWriterFactory = mock[HBaseWriterFactory]
implicit val system: ActorSystem = MockUtil.system
val userConfig = UserConfig.empty
val tableName = "hbase"
when(hbaseWriterFactory.getHBaseWriter(userConfig, tableName))
.thenReturn(hbaseWriter)
val hbaseSink = new HBaseSink(userConfig, tableName, hbaseWriterFactory)
hbaseSink.open(MockUtil.mockTaskContext)
forAll(Gen.alphaStr) { (value: String) =>
val message = Message(value)
hbaseSink.write(message)
verify(hbaseWriter, atLeastOnce()).put(value)
}
hbaseSink.close()
verify(hbaseWriter).close()
}
property("HBaseWriter should insert a row successfully") {
val table = mock[Table]
val config = mock[Configuration]
val connection = mock[Connection]
val taskContext = mock[TaskContext]
val map = Map[String, String]("HBASESINK" -> "hbasesink", "TABLE_NAME" -> "hbase.table.name",
"COLUMN_FAMILY" -> "hbase.table.column.family", "COLUMN_NAME" -> "hbase.table.column.name",
"HBASE_USER" -> "hbase.user", "GEARPUMP_KERBEROS_PRINCIPAL" -> "gearpump.kerberos.principal",
"GEARPUMP_KEYTAB_FILE" -> "gearpump.keytab.file"
)
val userConfig = new UserConfig(map)
val tableName = "hbase"
val row = "row"
val group = "group"
val name = "name"
val value = "3.0"
when(connection.getTable(TableName.valueOf(tableName))).thenReturn(table)
val put = new Put(Bytes.toBytes(row))
put.addColumn(Bytes.toBytes(group), Bytes.toBytes(name), Bytes.toBytes(value))
val hbaseWriter = new HBaseWriter(connection, tableName)
hbaseWriter.insert(Bytes.toBytes(row), Bytes.toBytes(group), Bytes.toBytes(name),
Bytes.toBytes(value))
verify(table).put(MockUtil.argMatch[Put](_.getRow sameElements Bytes.toBytes(row)))
}
}
Example 87
| Source File: KuduSinkSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.external.kudu
import akka.actor.ActorSystem
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.external.kudu.KuduSink.{KuduWriter, KuduWriterFactory}
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.task.TaskContext
import org.apache.kudu.client._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
class KuduSinkSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("KuduSink should invoke KuduWriter for writing message to Kudu") {
val kuduWriter = mock[KuduWriter]
val kuduWriterFactory = mock[KuduWriterFactory]
implicit val system: ActorSystem = MockUtil.system
val userConfig = UserConfig.empty
val tableName = "kudu"
when(kuduWriterFactory.getKuduWriter(userConfig, tableName))
.thenReturn(kuduWriter)
val kuduSink = new KuduSink(userConfig, tableName, kuduWriterFactory)
kuduSink.open(MockUtil.mockTaskContext)
val value = ("key", "value")
val message = Message(value)
kuduSink.write(message)
verify(kuduWriter, atLeastOnce()).put(message.value)
kuduSink.close()
verify(kuduWriter).close()
}
property("KuduWriter should insert a row successfully") {
val table = mock[KuduTable]
val kuduClient = mock[KuduClient]
val taskContext = mock[TaskContext]
val map = Map[String, String]("KUDUSINK" -> "kudusink", "TABLE_NAME" -> "kudu.table.name",
"COLUMN_FAMILY" -> "kudu.table.column.family", "COLUMN_NAME" -> "kudu.table.column.name",
"KUDU_USER" -> "kudu.user", "GEARPUMP_KERBEROS_PRINCIPAL" -> "gearpump.kerberos.principal",
"GEARPUMP_KEYTAB_FILE" -> "gearpump.keytab.file"
)
val userConfig = new UserConfig(map)
val tableName = "kudu"
val key = "key"
val value = "value"
when(kuduClient.openTable(tableName)).thenReturn(table)
}
}
Example 88
| Source File: KafkaSinkSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.kafka
import java.util.Properties
import com.twitter.bijection.Injection
import org.apache.gearpump.streaming.kafka.lib.sink.AbstractKafkaSink.KafkaProducerFactory
import org.apache.gearpump.streaming.kafka.util.KafkaConfig
import org.apache.gearpump.streaming.kafka.util.KafkaConfig.KafkaConfigFactory
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import org.apache.gearpump.Message
import org.apache.gearpump.streaming.MockUtil
class KafkaSinkSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
val dataGen = for {
topic <- Gen.alphaStr
key <- Gen.alphaStr
msg <- Gen.alphaStr
} yield (topic, Injection[String, Array[Byte]](key), Injection[String, Array[Byte]](msg))
property("KafkaSink write should send producer record") {
forAll(dataGen) {
(data: (String, Array[Byte], Array[Byte])) =>
val props = mock[Properties]
val producer = mock[KafkaProducer[Array[Byte], Array[Byte]]]
val producerFactory = mock[KafkaProducerFactory]
val configFactory = mock[KafkaConfigFactory]
val config = mock[KafkaConfig]
when(configFactory.getKafkaConfig(props)).thenReturn(config)
when(producerFactory.getKafkaProducer(config)).thenReturn(producer)
val (topic, key, msg) = data
val kafkaSink = new KafkaSink(topic, props, configFactory, producerFactory)
kafkaSink.write(Message((key, msg)))
verify(producer).send(MockUtil.argMatch[ProducerRecord[Array[Byte], Array[Byte]]](
r => r.topic == topic && (r.key sameElements key) && (r.value sameElements msg)))
kafkaSink.write(Message(msg))
verify(producer).send(MockUtil.argMatch[ProducerRecord[Array[Byte], Array[Byte]]](
r => r.topic() == topic && (r.key == null) && (r.value() sameElements msg)
))
kafkaSink.close()
}
}
property("KafkaSink close should close kafka producer") {
val props = mock[Properties]
val producer = mock[KafkaProducer[Array[Byte], Array[Byte]]]
val producerFactory = mock[KafkaProducerFactory]
val configFactory = mock[KafkaConfigFactory]
val config = mock[KafkaConfig]
when(configFactory.getKafkaConfig(props)).thenReturn(config)
when(producerFactory.getKafkaProducer(config)).thenReturn(producer)
val kafkaSink = new KafkaSink("topic", props, configFactory, producerFactory)
kafkaSink.close()
verify(producer).close()
}
}
Example 89
| Source File: KafkaConsumerSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.kafka.lib.source.consumer
import com.twitter.bijection.Injection
import kafka.api.OffsetRequest
import kafka.common.TopicAndPartition
import kafka.consumer.SimpleConsumer
import kafka.message.{Message, MessageAndOffset}
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
class KafkaConsumerSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
val messageGen = Gen.alphaStr map (msg => new Message(Injection[String, Array[Byte]](msg)))
val messageNumGen = Gen.choose[Int](0, 1000)
val topicAndPartitionGen = for {
topic <- Gen.alphaStr
partition <- Gen.choose[Int](0, Int.MaxValue)
} yield (topic, partition)
property("KafkaConsumer should iterate MessageAndOffset calling hasNext and next") {
forAll(messageGen, messageNumGen, topicAndPartitionGen) {
(message: Message, num: Int, topicAndPartition: (String, Int)) =>
val (topic, partition) = topicAndPartition
val consumer = mock[SimpleConsumer]
when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
OffsetRequest.EarliestTime, -1)).thenReturn(0)
val iterator = 0.until(num).map(index => MessageAndOffset(message, index.toLong)).iterator
val getIterator = (offset: Long) => iterator
val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
0.until(num).foreach { i =>
kafkaConsumer.hasNext shouldBe true
val kafkaMessage = kafkaConsumer.next
kafkaMessage.offset shouldBe i.toLong
kafkaMessage.key shouldBe None
}
kafkaConsumer.hasNext shouldBe false
}
}
val startOffsetGen = Gen.choose[Long](1L, 1000L)
property("KafkaConsumer setStartOffset should reset internal iterator") {
forAll(topicAndPartitionGen, startOffsetGen) {
(topicAndPartition: (String, Int), startOffset: Long) =>
val (topic, partition) = topicAndPartition
val consumer = mock[SimpleConsumer]
val getIterator = mock[Long => Iterator[MessageAndOffset]]
when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
OffsetRequest.EarliestTime, -1)).thenReturn(0)
val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
kafkaConsumer.setStartOffset(startOffset)
verify(getIterator).apply(startOffset)
}
}
property("KafkaConsumer close should close SimpleConsumer") {
forAll(topicAndPartitionGen) {
(topicAndPartition: (String, Int)) =>
val (topic, partition) = topicAndPartition
val consumer = mock[SimpleConsumer]
when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
OffsetRequest.EarliestTime, -1)).thenReturn(0)
val getIterator = mock[Long => Iterator[MessageAndOffset]]
val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
kafkaConsumer.close()
verify(consumer).close()
}
}
}
Example 90
| Source File: StormSpoutOutputCollectorSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.producer
import backtype.storm.spout.ISpout
import backtype.storm.utils.Utils
import org.apache.gearpump.experiments.storm.util.StormOutputCollector
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import scala.collection.JavaConverters._
class StormSpoutOutputCollectorSpec
extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("StormSpoutOutputCollector should call StormOutputCollector") {
val valGen = Gen.oneOf(Gen.alphaStr, Gen.alphaChar, Gen.chooseNum[Int](0, 1000))
val valuesGen = Gen.listOf[AnyRef](valGen)
forAll(valuesGen) { (values: List[AnyRef]) =>
val collector = mock[StormOutputCollector]
val spout = mock[ISpout]
val streamId = Utils.DEFAULT_STREAM_ID
val spoutCollector = new StormSpoutOutputCollector(collector, spout, false)
spoutCollector.emit(streamId, values.asJava, null)
verify(collector).emit(streamId, values.asJava)
}
}
}
Example 91
| Source File: StormProducerSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.producer
import java.time.Instant
import akka.testkit.TestProbe
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.experiments.storm.topology.GearpumpStormComponent.GearpumpSpout
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.source.Watermark
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, WordSpec}
class StormProducerSpec extends WordSpec with Matchers with MockitoSugar {
"StormProducer" should {
"start GearpumpSpout onStart" in {
val startTime = Instant.EPOCH
val gearpumpSpout = mock[GearpumpSpout]
when(gearpumpSpout.getMessageTimeout).thenReturn(None)
val taskContext = MockUtil.mockTaskContext
implicit val actorSystem = taskContext.system
val taskActor = TestProbe()
when(taskContext.self).thenReturn(taskActor.ref)
val userConfig = UserConfig.empty
val stormProducer = new StormProducer(gearpumpSpout, taskContext, userConfig)
stormProducer.onStart(startTime)
verify(gearpumpSpout).start(startTime)
taskActor.expectMsgType[Watermark]
}
"pass message to GearpumpBolt onNext" in {
val message = mock[Message]
val gearpumpSpout = mock[GearpumpSpout]
val timeout = 5L
when(gearpumpSpout.getMessageTimeout).thenReturn(Some(timeout))
val taskContext = MockUtil.mockTaskContext
implicit val actorSystem = taskContext.system
val taskActor = TestProbe()
when(taskContext.self).thenReturn(taskActor.ref)
val userConfig = UserConfig.empty
val stormProducer = new StormProducer(gearpumpSpout, taskContext, userConfig)
stormProducer.onNext(message)
verify(gearpumpSpout).next(message)
taskActor.expectMsgType[Watermark]
stormProducer.onNext(StormProducer.TIMEOUT)
verify(gearpumpSpout).timeout(timeout)
}
}
}
Example 92
| Source File: StormProcessorSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.processor
import java.time.Instant
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.experiments.storm.topology.GearpumpStormComponent.GearpumpBolt
import org.apache.gearpump.streaming.MockUtil
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, WordSpec}
class StormProcessorSpec extends WordSpec with Matchers with MockitoSugar {
"StormProcessor" should {
"start GearpumpSpout onStart" in {
val startTime = Instant.EPOCH
val gearpumpBolt = mock[GearpumpBolt]
when(gearpumpBolt.getTickFrequency).thenReturn(None)
val taskContext = MockUtil.mockTaskContext
val userConfig = UserConfig.empty
val stormProcessor = new StormProcessor(gearpumpBolt, taskContext, userConfig)
stormProcessor.onStart(startTime)
verify(gearpumpBolt).start(startTime)
}
"pass message to GearpumpBolt onNext" in {
val message = mock[Message]
val gearpumpBolt = mock[GearpumpBolt]
val freq = 5
when(gearpumpBolt.getTickFrequency).thenReturn(Some(freq))
val taskContext = MockUtil.mockTaskContext
val userConfig = UserConfig.empty
val stormProcessor = new StormProcessor(gearpumpBolt, taskContext, userConfig)
stormProcessor.onNext(message)
verify(gearpumpBolt).next(message)
stormProcessor.onNext(StormProcessor.TICK)
verify(gearpumpBolt).tick(freq)
}
}
}
Example 93
| Source File: StormBoltOutputCollectorSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.processor
import backtype.storm.tuple.Tuple
import backtype.storm.utils.Utils
import org.apache.gearpump.experiments.storm.util.StormOutputCollector
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import scala.collection.JavaConverters._
class StormBoltOutputCollectorSpec
extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("StormBoltOutputCollector should call StormOutputCollector") {
val valGen = Gen.oneOf(Gen.alphaStr, Gen.alphaChar, Gen.chooseNum[Int](0, 1000))
val valuesGen = Gen.listOf[AnyRef](valGen)
forAll(valuesGen) { (values: List[AnyRef]) =>
val collector = mock[StormOutputCollector]
val boltCollector = new StormBoltOutputCollector(collector)
val streamId = Utils.DEFAULT_STREAM_ID
boltCollector.emit(streamId, null, values.asJava)
verify(collector).emit(streamId, values.asJava)
}
}
property("StormBoltOutputCollector should throw on fail") {
val collector = mock[StormOutputCollector]
val tuple = mock[Tuple]
val boltCollector = new StormBoltOutputCollector(collector)
an[Exception] should be thrownBy boltCollector.fail(tuple)
}
}
Example 94
| Source File: StormSerializerPoolSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.util
import java.util.{HashMap => JHashMap, List => JList, Map => JMap}
import scala.collection.JavaConverters._
import akka.actor.ExtendedActorSystem
import backtype.storm.utils.Utils
import com.esotericsoftware.kryo.Kryo
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.experiments.storm.topology.GearpumpTuple
import org.apache.gearpump.experiments.storm.util.StormConstants._
import org.apache.gearpump.streaming.MockUtil
class StormSerializerPoolSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("StormSerializerPool should create and manage StormSerializer") {
val taskContext = MockUtil.mockTaskContext
val serializerPool = new StormSerializationFramework
val system = taskContext.system.asInstanceOf[ExtendedActorSystem]
implicit val actorSystem = system
val stormConfig = Utils.readDefaultConfig.asInstanceOf[JMap[AnyRef, AnyRef]]
val config = UserConfig.empty.withValue[JMap[AnyRef, AnyRef]](STORM_CONFIG, stormConfig)
serializerPool.init(system, config)
serializerPool.get shouldBe a[StormSerializer]
}
property("StormSerializer should serialize and deserialize GearpumpTuple") {
val tupleGen = for {
values <- Gen.listOf[String](Gen.alphaStr).map(_.asJava.asInstanceOf[JList[AnyRef]])
sourceTaskId <- Gen.chooseNum[Int](0, Int.MaxValue)
sourceStreamId <- Gen.alphaStr
} yield new GearpumpTuple(values, new Integer(sourceTaskId), sourceStreamId, null)
val kryo = new Kryo
forAll(tupleGen) { (tuple: GearpumpTuple) =>
val serializer = new StormSerializer(kryo)
serializer.deserialize(serializer.serialize(tuple)) shouldBe tuple
}
}
}
Example 95
| Source File: GraphBuilderSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.util
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, WordSpec}
import org.apache.gearpump.experiments.storm.partitioner.StormPartitioner
import org.apache.gearpump.experiments.storm.topology.GearpumpStormTopology
import org.apache.gearpump.streaming.Processor
import org.apache.gearpump.streaming.task.Task
class GraphBuilderSpec extends WordSpec with Matchers with MockitoSugar {
"GraphBuilder" should {
"build Graph from Storm topology" in {
val topology = mock[GearpumpStormTopology]
val sourceId = "source"
val sourceProcessor = mock[Processor[Task]]
val targetId = "target"
val targetProcessor = mock[Processor[Task]]
when(topology.getProcessors).thenReturn(
Map(sourceId -> sourceProcessor, targetId -> targetProcessor))
when(topology.getTargets(sourceId)).thenReturn(Map(targetId -> targetProcessor))
when(topology.getTargets(targetId)).thenReturn(Map.empty[String, Processor[Task]])
val graph = GraphBuilder.build(topology)
graph.getEdges.size shouldBe 1
val (from, edge, to) = graph.getEdges.head
from shouldBe sourceProcessor
edge shouldBe a[StormPartitioner]
to shouldBe targetProcessor
}
}
}
Example 96
| Source File: StormOutputCollectorSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.util
import java.util.{List => JList, Map => JMap}
import scala.collection.JavaConverters._
import backtype.storm.generated.Grouping
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import org.apache.gearpump.{Message, Time}
import org.apache.gearpump.Time.MilliSeconds
import org.apache.gearpump.experiments.storm.topology.GearpumpTuple
import org.apache.gearpump.streaming.MockUtil
class StormOutputCollectorSpec
extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
private val stormTaskId = 0
private val streamIdGen = Gen.alphaStr
private val valuesGen = Gen.listOf[String](Gen.alphaStr).map(_.asJava.asInstanceOf[JList[AnyRef]])
private val timestampGen = Gen.chooseNum[Long](0L, 1000L)
property("StormOutputCollector emits tuple values into a stream") {
forAll(timestampGen, streamIdGen, valuesGen) {
(timestamp: MilliSeconds, streamId: String, values: JList[AnyRef]) =>
val targets = mock[JMap[String, JMap[String, Grouping]]]
val taskToComponent = mock[JMap[Integer, String]]
val getTargetPartitionsFn = mock[(String, JList[AnyRef]) =>
(Map[String, Array[Int]], JList[Integer])]
val targetPartitions = mock[Map[String, Array[Int]]]
val targetStormTaskIds = mock[JList[Integer]]
when(getTargetPartitionsFn(streamId, values)).thenReturn((targetPartitions,
targetStormTaskIds))
val taskContext = MockUtil.mockTaskContext
val stormOutputCollector = new StormOutputCollector(stormTaskId, taskToComponent,
targets, getTargetPartitionsFn, taskContext, Time.MIN_TIME_MILLIS)
when(targets.containsKey(streamId)).thenReturn(false)
stormOutputCollector.emit(streamId, values) shouldBe StormOutputCollector.EMPTY_LIST
verify(taskContext, times(0)).output(anyObject[Message])
when(targets.containsKey(streamId)).thenReturn(true)
stormOutputCollector.setTimestamp(timestamp)
stormOutputCollector.emit(streamId, values) shouldBe targetStormTaskIds
verify(taskContext, times(1)).output(MockUtil.argMatch[Message]({
message: Message =>
val expected = new GearpumpTuple(values, stormTaskId, streamId, targetPartitions)
message.value == expected && message.timestamp.toEpochMilli == timestamp
}))
}
}
property("StormOutputCollector emit direct to a task") {
val idGen = Gen.chooseNum[Int](0, 1000)
val targetGen = Gen.alphaStr
forAll(idGen, targetGen, timestampGen, streamIdGen, valuesGen) {
(id: Int, target: String, timestamp: Long, streamId: String, values: JList[AnyRef]) =>
val targets = mock[JMap[String, JMap[String, Grouping]]]
val taskToComponent = mock[JMap[Integer, String]]
when(taskToComponent.get(id)).thenReturn(target)
val getTargetPartitionsFn = mock[(String, JList[AnyRef]) =>
(Map[String, Array[Int]], JList[Integer])]
val targetPartitions = mock[Map[String, Array[Int]]]
val targetStormTaskIds = mock[JList[Integer]]
when(getTargetPartitionsFn(streamId, values)).thenReturn((targetPartitions,
targetStormTaskIds))
val taskContext = MockUtil.mockTaskContext
val stormOutputCollector = new StormOutputCollector(stormTaskId, taskToComponent,
targets, getTargetPartitionsFn, taskContext, Time.MIN_TIME_MILLIS)
when(targets.containsKey(streamId)).thenReturn(false)
verify(taskContext, times(0)).output(anyObject[Message])
when(targets.containsKey(streamId)).thenReturn(true)
stormOutputCollector.setTimestamp(timestamp)
stormOutputCollector.emitDirect(id, streamId, values)
val partitions = Array(StormUtil.stormTaskIdToGearpump(id).index)
verify(taskContext, times(1)).output(MockUtil.argMatch[Message]({
message: Message => {
val expected = new GearpumpTuple(values, stormTaskId, streamId,
Map(target -> partitions))
val result = message.value == expected && message.timestamp.toEpochMilli == timestamp
result
}
}))
}
}
}
Example 97
| Source File: GearpumpStormTopologySpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.topology
import java.util.{HashMap => JHashMap, Map => JMap}
import backtype.storm.Config
import org.apache.gearpump.experiments.storm.processor.StormProcessor
import org.apache.gearpump.experiments.storm.producer.StormProducer
import org.apache.gearpump.experiments.storm.util.TopologyUtil
import org.apache.gearpump.streaming.MockUtil
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, WordSpec}
import scala.collection.JavaConverters._
class GearpumpStormTopologySpec extends WordSpec with Matchers with MockitoSugar {
import org.apache.gearpump.experiments.storm.topology.GearpumpStormTopologySpec._
"GearpumpStormTopology" should {
"merge configs with defined priority" in {
val stormTopology = TopologyUtil.getTestTopology
val name = "name"
val sysVal = "sys"
val sysConfig = newJavaConfig(name, sysVal)
val appVal = "app"
val appConfig = newJavaConfig(name, appVal)
implicit val system = MockUtil.system
val topology1 = new GearpumpStormTopology("topology1", stormTopology, newEmptyConfig,
newEmptyConfig)
topology1.getStormConfig.get(Config.TOPOLOGY_NAME) shouldBe "topology1"
topology1.getStormConfig should not contain name
val topology2 = new GearpumpStormTopology("topology2", stormTopology, sysConfig,
newEmptyConfig)
topology2.getStormConfig.get(Config.TOPOLOGY_NAME) shouldBe "topology2"
topology2.getStormConfig.get(name) shouldBe sysVal
val topology3 = new GearpumpStormTopology("topology3", stormTopology, sysConfig, appConfig)
topology3.getStormConfig.get(Config.TOPOLOGY_NAME) shouldBe "topology3"
topology3.getStormConfig.get(name) shouldBe appVal
}
"create Gearpump processors from Storm topology" in {
val stormTopology = TopologyUtil.getTestTopology
implicit val system = MockUtil.system
val gearpumpStormTopology =
GearpumpStormTopology("app", stormTopology, null)
val processors = gearpumpStormTopology.getProcessors
stormTopology.get_spouts().asScala.foreach { case (spoutId, _) =>
val processor = processors(spoutId)
processor.taskClass shouldBe classOf[StormProducer]
processor.description shouldBe spoutId
}
stormTopology.get_bolts().asScala.foreach { case (boltId, _) =>
val processor = processors(boltId)
processor.taskClass shouldBe classOf[StormProcessor]
processor.description shouldBe boltId
}
}
"get target processors from source id" in {
val stormTopology = TopologyUtil.getTestTopology
implicit val system = MockUtil.system
val gearpumpStormTopology =
GearpumpStormTopology("app", stormTopology, null)
val targets0 = gearpumpStormTopology.getTargets("1")
targets0 should contain key "3"
targets0 should contain key "4"
val targets1 = gearpumpStormTopology.getTargets("2")
targets1 should contain key "3"
}
}
}
object GearpumpStormTopologySpec {
def newEmptyConfig: JMap[AnyRef, AnyRef] = {
new JHashMap[AnyRef, AnyRef]
}
def newJavaConfig(key: AnyRef, value: AnyRef): JMap[AnyRef, AnyRef] = {
val config = new JHashMap[AnyRef, AnyRef]
config.put(key, value)
config
}
}
Example 98
| Source File: GearpumpTupleSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.experiments.storm.topology
import java.util.{List => JList}
import backtype.storm.task.GeneralTopologyContext
import backtype.storm.tuple.Fields
import org.apache.gearpump.Time.MilliSeconds
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import scala.collection.JavaConverters._
class GearpumpTupleSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("GearpumpTuple should create Storm Tuple") {
val tupleGen = for {
values <- Gen.listOf[String](Gen.alphaStr).map(_.distinct.asJava.asInstanceOf[JList[AnyRef]])
sourceTaskId <- Gen.chooseNum[Int](0, Int.MaxValue)
sourceStreamId <- Gen.alphaStr
} yield new GearpumpTuple(values, new Integer(sourceTaskId), sourceStreamId, null)
forAll(tupleGen, Gen.alphaStr, Gen.chooseNum[Long](0, Long.MaxValue)) {
(gearpumpTuple: GearpumpTuple, componentId: String, timestamp: MilliSeconds) =>
val topologyContext = mock[GeneralTopologyContext]
val fields = new Fields(gearpumpTuple.values.asScala.map(_.asInstanceOf[String]): _*)
when(topologyContext.getComponentId(gearpumpTuple.sourceTaskId)).thenReturn(componentId)
when(topologyContext.getComponentOutputFields(
componentId, gearpumpTuple.sourceStreamId)).thenReturn(fields)
val tuple = gearpumpTuple.toTuple(topologyContext, timestamp)
tuple shouldBe a[TimedTuple]
val timedTuple = tuple.asInstanceOf[TimedTuple]
timedTuple.getValues shouldBe gearpumpTuple.values
timedTuple.getSourceTask shouldBe gearpumpTuple.sourceTaskId
timedTuple.getSourceComponent shouldBe componentId
timedTuple.getSourceStreamId shouldBe gearpumpTuple.sourceStreamId
timedTuple.getMessageId shouldBe null
timedTuple.getFields shouldBe fields
timedTuple.timestamp shouldBe timestamp
}
}
}
Example 99
| Source File: DefaultWindowRunnerSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.dsl.window.impl
import java.time.{Duration, Instant}
import org.apache.gearpump.Message
import org.apache.gearpump.streaming.dsl.api.functions.ReduceFunction
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.dsl.plan.functions.FoldRunner
import org.apache.gearpump.streaming.dsl.window.api.SessionWindows
import org.apache.gearpump.streaming.source.Watermark
import org.scalatest.{Matchers, PropSpec}
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
class DefaultWindowRunnerSpec extends PropSpec with PropertyChecks
with Matchers with MockitoSugar {
property("DefaultWindowRunner should handle SessionWindow") {
val data = List(
Message(("foo", 1L), Instant.ofEpochMilli(1L)),
Message(("foo", 1L), Instant.ofEpochMilli(15L)),
Message(("foo", 1L), Instant.ofEpochMilli(25L)),
Message(("foo", 1L), Instant.ofEpochMilli(26L))
)
type KV = (String, Long)
implicit val system = MockUtil.system
val reduce = ReduceFunction[KV]((kv1, kv2) => (kv1._1, kv1._2 + kv2._2))
val windows = SessionWindows.apply(Duration.ofMillis(4L))
val windowRunner = new WindowOperator[KV, Option[KV]](windows,
new FoldRunner[KV, Option[KV]](reduce, "reduce"))
data.foreach(m => windowRunner.foreach(TimestampedValue(m.value.asInstanceOf[KV], m.timestamp)))
windowRunner.trigger(Watermark.MAX).outputs.toList shouldBe
List(
TimestampedValue(Some(("foo", 1)), Instant.ofEpochMilli(4)),
TimestampedValue(Some(("foo", 1)), Instant.ofEpochMilli(18)),
TimestampedValue(Some(("foo", 2)), Instant.ofEpochMilli(29))
)
}
}
Example 100
| Source File: StreamAppSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.dsl.scalaapi
import akka.actor.ActorSystem
import org.apache.gearpump.cluster.TestUtil
import org.apache.gearpump.cluster.client.ClientContext
import org.apache.gearpump.streaming.dsl.scalaapi
import org.apache.gearpump.streaming.partitioner.PartitionerDescription
import org.apache.gearpump.streaming.source.DataSourceTask
import org.apache.gearpump.streaming.{ProcessorDescription, StreamApplication}
import org.apache.gearpump.util.Graph
import org.mockito.Mockito.when
import org.scalatest._
import org.scalatest.mock.MockitoSugar
import scala.concurrent.Await
import scala.concurrent.duration.Duration
class StreamAppSpec extends FlatSpec with Matchers with BeforeAndAfterAll with MockitoSugar {
implicit var system: ActorSystem = _
override def beforeAll(): Unit = {
system = ActorSystem("test", TestUtil.DEFAULT_CONFIG)
}
override def afterAll(): Unit = {
system.terminate()
Await.result(system.whenTerminated, Duration.Inf)
}
it should "be able to generate multiple new streams" in {
val context: ClientContext = mock[ClientContext]
when(context.system).thenReturn(system)
val dsl = StreamApp("dsl", context)
dsl.source(List("A"), 2, "A") shouldBe a [scalaapi.Stream[_]]
dsl.source(List("B"), 3, "B") shouldBe a [scalaapi.Stream[_]]
val application = dsl.plan()
application shouldBe a [StreamApplication]
application.name shouldBe "dsl"
val dag = application.userConfig
.getValue[Graph[ProcessorDescription, PartitionerDescription]](StreamApplication.DAG).get
dag.getVertices.size shouldBe 2
dag.getVertices.foreach { processor =>
processor.taskClass shouldBe classOf[DataSourceTask[_, _]].getName
if (processor.description == "A") {
processor.parallelism shouldBe 2
} else if (processor.description == "B") {
processor.parallelism shouldBe 3
} else {
fail(s"undefined source ${processor.description}")
}
}
}
}
Example 101
| Source File: GroupByTaskSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.dsl.task
import java.time.Instant
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.streaming.dsl.plan.functions.DummyRunner
import org.apache.gearpump.streaming.dsl.window.api.GlobalWindows
import org.apache.gearpump.streaming.{Constants, MockUtil}
import org.apache.gearpump.streaming.dsl.window.impl.WindowOperator
import org.apache.gearpump.streaming.source.Watermark
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.{Matchers, PropSpec}
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
class GroupByTaskSpec extends PropSpec with PropertyChecks
with Matchers with MockitoSugar {
property("GroupByTask should trigger on watermark") {
val longGen = Gen.chooseNum[Long](1L, 1000L).map(Instant.ofEpochMilli)
forAll(longGen) { (time: Instant) =>
val groupBy = mock[Any => Int]
val windowRunner = new WindowOperator[Any, Any](GlobalWindows(), new DummyRunner[Any])
val context = MockUtil.mockTaskContext
val config = UserConfig.empty
.withValue(
Constants.GEARPUMP_STREAMING_OPERATOR, windowRunner)(MockUtil.system)
val task = new GroupByTask[Any, Int, Any](groupBy, context, config)
val value = time
val message = Message(value, time)
when(groupBy(time)).thenReturn(0)
task.onNext(message)
task.onWatermarkProgress(Watermark.MAX)
verify(context).output(message)
}
}
}
Example 102
| Source File: TransformTaskSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.dsl.task
import java.time.Instant
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.dsl.window.impl.{TimestampedValue, TriggeredOutputs, StreamingOperator}
import org.mockito.Mockito.{verify, when}
import org.scalacheck.Gen
import org.scalatest.{Matchers, PropSpec}
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
class TransformTaskSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("MergeTask should trigger on watermark") {
val longGen = Gen.chooseNum[Long](1L, 1000L)
val watermarkGen = longGen.map(Instant.ofEpochMilli)
forAll(watermarkGen) { (watermark: Instant) =>
val windowRunner = mock[StreamingOperator[Any, Any]]
val context = MockUtil.mockTaskContext
val config = UserConfig.empty
val task = new TransformTask[Any, Any](windowRunner, context, config)
val time = watermark.minusMillis(1L)
val value: Any = time
val message = Message(value, time)
task.onNext(message)
verify(windowRunner).foreach(TimestampedValue(value, time))
when(windowRunner.trigger(watermark)).thenReturn(
TriggeredOutputs(Some(TimestampedValue(value, time)), watermark))
task.onWatermarkProgress(watermark)
verify(context).output(message)
verify(context).updateWatermark(watermark)
}
}
}
Example 103
| Source File: WindowSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.state.impl
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import org.apache.gearpump.Time.MilliSeconds
class WindowSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
val windowSizeGen = Gen.chooseNum[Long](1L, 1000L)
val windowStepGen = Gen.chooseNum[Long](1L, 1000L)
val timestampGen = Gen.chooseNum[Long](0L, 1000L)
property("Window should only slide when time passes window end") {
forAll(timestampGen, windowSizeGen, windowStepGen) {
(timestamp: MilliSeconds, windowSize: Long, windowStep: Long) =>
val window = new Window(windowSize, windowStep)
window.shouldSlide shouldBe false
window.update(timestamp)
window.shouldSlide shouldBe timestamp >= windowSize
}
}
property("Window should slide by one or to given timestamp") {
forAll(timestampGen, windowSizeGen, windowStepGen) {
(timestamp: MilliSeconds, windowSize: Long, windowStep: Long) =>
val window = new Window(windowSize, windowStep)
window.range shouldBe(0L, windowSize)
window.slideOneStep()
window.range shouldBe(windowStep, windowSize + windowStep)
window.slideTo(timestamp)
val (startTime, endTime) = window.range
if (windowStep > windowSize) {
timestamp should (be >= startTime and be < (startTime + windowStep))
} else {
timestamp should (be >= startTime and be < endTime)
}
}
}
}
Example 104
| Source File: CheckpointManagerSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.state.impl
import org.mockito.Mockito._
import org.mockito.{Matchers => MockitoMatchers}
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}
import org.apache.gearpump.Time.MilliSeconds
import org.apache.gearpump.streaming.transaction.api.CheckpointStore
class CheckpointManagerSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
val timestampGen = Gen.chooseNum[Long](0L, 1000L)
val checkpointIntervalGen = Gen.chooseNum[Long](100L, 10000L)
property("CheckpointManager should recover from CheckpointStore") {
forAll(timestampGen, checkpointIntervalGen) {
(timestamp: MilliSeconds, checkpointInterval: Long) =>
val checkpointStore = mock[CheckpointStore]
val checkpointManager =
new CheckpointManager(checkpointInterval, checkpointStore)
checkpointManager.recover(timestamp)
verify(checkpointStore).recover(timestamp)
}
}
property("CheckpointManager should write checkpoint to CheckpointStore") {
val checkpointGen = Gen.alphaStr.map(_.getBytes("UTF-8"))
forAll(timestampGen, checkpointIntervalGen, checkpointGen) {
(timestamp: MilliSeconds, checkpointInterval: Long, checkpoint: Array[Byte]) =>
val checkpointStore = mock[CheckpointStore]
val checkpointManager =
new CheckpointManager(checkpointInterval, checkpointStore)
checkpointManager.checkpoint(timestamp, checkpoint)
verify(checkpointStore).persist(timestamp, checkpoint)
}
}
property("CheckpointManager should close CheckpointStore") {
forAll(checkpointIntervalGen) {
(checkpointInterval: Long) =>
val checkpointStore = mock[CheckpointStore]
val checkpointManager =
new CheckpointManager(checkpointInterval, checkpointStore)
checkpointManager.close()
verify(checkpointStore).close()
}
}
property("CheckpointManager should update checkpoint time according to max message timestamp") {
forAll(timestampGen, checkpointIntervalGen) {
(timestamp: MilliSeconds, checkpointInterval: Long) =>
val checkpointStore = mock[CheckpointStore]
val checkpointManager =
new CheckpointManager(checkpointInterval, checkpointStore)
checkpointManager.update(timestamp)
checkpointManager.getMaxMessageTime shouldBe timestamp
val checkpointTime = checkpointManager.getCheckpointTime.get
timestamp should (be < checkpointTime and be >= (checkpointTime - checkpointInterval))
checkpointManager.checkpoint(checkpointTime, Array.empty[Byte])
verify(checkpointStore).persist(MockitoMatchers.eq(checkpointTime),
MockitoMatchers.anyObject[Array[Byte]]())
checkpointManager.getCheckpointTime shouldBe empty
}
}
}
Example 105
| Source File: DataSinkTaskSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.sink
import java.time.Instant
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.streaming.MockUtil
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.prop.PropertyChecks
import org.scalatest.{PropSpec, Matchers}
class DataSinkTaskSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("DataSinkTask.onStart should call DataSink.open" ) {
forAll(Gen.chooseNum[Long](0L, 1000L).map(Instant.ofEpochMilli)) { (startTime: Instant) =>
val taskContext = MockUtil.mockTaskContext
val config = UserConfig.empty
val dataSink = mock[DataSink]
val sinkTask = new DataSinkTask(taskContext, config, dataSink)
sinkTask.onStart(startTime)
verify(dataSink).open(taskContext)
}
}
property("DataSinkTask.onNext should call DataSink.write") {
forAll(Gen.alphaStr) { (str: String) =>
val taskContext = MockUtil.mockTaskContext
val config = UserConfig.empty
val dataSink = mock[DataSink]
val sinkTask = new DataSinkTask(taskContext, config, dataSink)
val msg = Message(str)
sinkTask.onNext(msg)
verify(dataSink).write(msg)
}
}
property("DataSinkTask.onStop should call DataSink.close") {
val taskContext = MockUtil.mockTaskContext
val config = UserConfig.empty
val dataSink = mock[DataSink]
val sinkTask = new DataSinkTask(taskContext, config, dataSink)
sinkTask.onStop()
verify(dataSink).close()
}
}
Example 106
| Source File: DataSourceTaskSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.streaming.source
import java.time.Instant
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
import org.apache.gearpump.streaming.MockUtil
import org.apache.gearpump.streaming.dsl.window.impl.{TimestampedValue, TriggeredOutputs, StreamingOperator}
import org.mockito.Mockito._
import org.scalacheck.Gen
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, PropSpec}
import org.scalatest.prop.PropertyChecks
class DataSourceTaskSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
property("DataSourceTask should setup data source") {
forAll(Gen.chooseNum[Long](0L, 1000L).map(Instant.ofEpochMilli)) {
(startTime: Instant) =>
val taskContext = MockUtil.mockTaskContext
implicit val system = MockUtil.system
val dataSource = mock[DataSource]
val config = UserConfig.empty
.withInt(DataSourceConfig.SOURCE_READ_BATCH_SIZE, 1)
val runner = mock[StreamingOperator[Any, Any]]
val sourceTask = new DataSourceTask[Any, Any](dataSource, runner, taskContext, config)
sourceTask.onStart(startTime)
verify(dataSource).open(taskContext, startTime)
}
}
property("DataSourceTask should read from DataSource and transform inputs") {
forAll(Gen.alphaStr, Gen.chooseNum[Long](0L, 1000L).map(Instant.ofEpochMilli)) {
(str: String, timestamp: Instant) =>
val taskContext = MockUtil.mockTaskContext
implicit val system = MockUtil.system
val dataSource = mock[DataSource]
val config = UserConfig.empty
.withInt(DataSourceConfig.SOURCE_READ_BATCH_SIZE, 1)
val processor = mock[StreamingOperator[String, String]]
val sourceTask = new DataSourceTask[String, String](dataSource, processor,
taskContext, config)
val msg = Message(str, timestamp)
when(dataSource.read()).thenReturn(msg)
when(processor.flatMap(new TimestampedValue[String](msg))).thenReturn(
Some(new TimestampedValue[String](msg))
)
when(processor.trigger(Watermark.MAX)).thenReturn(
TriggeredOutputs[String](None, Watermark.MAX))
sourceTask.onNext(Message("next"))
sourceTask.onWatermarkProgress(Watermark.MAX)
verify(taskContext).output(msg)
verify(taskContext).updateWatermark(Watermark.MAX)
}
}
property("DataSourceTask should teardown DataSource") {
val taskContext = MockUtil.mockTaskContext
implicit val system = MockUtil.system
val dataSource = mock[DataSource]
val config = UserConfig.empty
.withInt(DataSourceConfig.SOURCE_READ_BATCH_SIZE, 1)
val runner = mock[StreamingOperator[Any, Any]]
val sourceTask = new DataSourceTask[Any, Any](dataSource, runner, taskContext, config)
sourceTask.onStop()
verify(dataSource).close()
}
}
Example 107
| Source File: NettySpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.transport
import java.util.concurrent.TimeUnit
import scala.concurrent.Await
import scala.concurrent.duration._
import akka.actor.{ActorRef, ActorSystem}
import akka.testkit.TestProbe
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
import org.apache.gearpump.cluster.TestUtil
import org.apache.gearpump.transport.MockTransportSerializer.NettyMessage
import org.apache.gearpump.transport.netty.{TaskMessage, Context}
import org.apache.gearpump.util.Util
class NettySpec extends FlatSpec with Matchers with MockitoSugar {
"Netty Transport" should "send and receive message correctly " in {
val conf = TestUtil.DEFAULT_CONFIG
val system = ActorSystem("transport", conf)
val context = new Context(system, conf)
val serverActor = TestProbe()(system)
val port = Util.findFreePort()
import system.dispatcher
system.scheduler.scheduleOnce(Duration(1, TimeUnit.SECONDS)) {
context.bind("server", new ActorLookupById {
override def lookupLocalActor(id: Long): Option[ActorRef] = Some(serverActor.ref)
}, false, port.get)
}
val client = context.connect(HostPort("127.0.0.1", port.get))
val data = NettyMessage(0)
val msg = new TaskMessage(0, 1, 2, data)
client ! msg
serverActor.expectMsg(15.seconds, data)
context.close()
system.terminate()
Await.result(system.whenTerminated, Duration.Inf)
}
}
Example 108
| Source File: SerializerSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.serializer
import akka.actor.{ActorSystem, ExtendedActorSystem}
import com.esotericsoftware.kryo.io.{Input, Output}
import com.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer}
import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
import org.apache.gearpump.cluster.TestUtil
import org.apache.gearpump.serializer.SerializerSpec._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
import scala.collection.JavaConverters._
import scala.concurrent.Await
import scala.concurrent.duration.Duration
class SerializerSpec extends FlatSpec with Matchers with MockitoSugar {
val config = ConfigFactory.empty.withValue("gearpump.serializers",
ConfigValueFactory.fromAnyRef(Map(classOf[ClassA].getName -> classOf[ClassASerializer].getName,
classOf[ClassB].getName -> classOf[ClassBSerializer].getName).asJava))
"GearpumpSerialization" should "register custom serializers" in {
val serialization = new GearpumpSerialization(config)
val kryo = new Kryo
serialization.customize(kryo)
val forB = kryo.getRegistration(classOf[ClassB])
assert(forB.getSerializer.isInstanceOf[ClassBSerializer])
val forA = kryo.getRegistration(classOf[ClassA])
assert(forA.getSerializer.isInstanceOf[ClassASerializer])
}
"FastKryoSerializer" should "serialize correctly" in {
val myConfig = config.withFallback(TestUtil.DEFAULT_CONFIG.withoutPath("gearpump.serializers"))
val system = ActorSystem("my", myConfig)
val serializer = new FastKryoSerializer(system.asInstanceOf[ExtendedActorSystem])
val bytes = serializer.serialize(new ClassA)
val anotherA = serializer.deserialize(bytes)
assert(anotherA.isInstanceOf[ClassA])
system.terminate()
Await.result(system.whenTerminated, Duration.Inf)
}
}
object SerializerSpec {
class ClassA {}
class ClassASerializer extends KryoSerializer[ClassA] {
override def write(kryo: Kryo, output: Output, `object`: ClassA): Unit = {
output.writeString(classOf[ClassA].getName)
}
override def read(kryo: Kryo, input: Input, `type`: Class[ClassA]): ClassA = {
val className = input.readString()
Class.forName(className).newInstance().asInstanceOf[ClassA]
}
}
class ClassB {}
class ClassBSerializer extends KryoSerializer[ClassA] {
override def write(kryo: Kryo, output: Output, `object`: ClassA): Unit = {}
override def read(kryo: Kryo, input: Input, `type`: Class[ClassA]): ClassA = {
null
}
}
}
Example 109
| Source File: ConfigsSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.util
import java.io.File
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import akka.actor.ActorSystem
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
import org.apache.gearpump.cluster.{ClusterConfig, ClusterConfigSource, UserConfig}
class ConfigsSpec extends FlatSpec with Matchers with MockitoSugar {
"Typesafe Cluster Configs" should "follow the override rules" in {
val conf =
"""
gearpump {
gear = "gearpump"
}
gearpump-master {
conf = "master"
}
gearpump-worker {
conf = "worker"
}
conf = "base"
"""
val file = File.createTempFile("test", ".conf")
FileUtils.write(file, conf)
val raw = ClusterConfig.load(ClusterConfigSource(file.toString))
assert(raw.master.getString("conf") == "master", "master > base")
assert(raw.worker.getString("conf") == "worker", "worker > base")
assert(raw.default.getString("conf") == "base", "application > base")
file.delete()
}
"ClusterConfigSource" should "return empty for non-exist files" in {
val source = ClusterConfigSource("non-exist")
var config = source.getConfig
assert(config.isEmpty)
val nullCheck = ClusterConfigSource(null)
config = nullCheck.getConfig
assert(config.isEmpty)
}
"User Config" should "work" in {
implicit val system = ActorSystem("forSerialization")
val map = Map[String, String]("key1" -> "1", "key2" -> "value2")
val user = new UserConfig(map)
.withLong("key3", 2L)
.withBoolean("key4", value = true)
.withFloat("key5", 3.14F)
.withDouble("key6", 2.718)
assert(user.getInt("key1").get == 1)
assert(user.getString("key1").get == "1")
assert(user.getLong("key3").get == 2L)
assert(user.getBoolean("key4").get == true)
assert(user.getFloat("key5").get == 3.14F)
assert(user.getDouble("key6").get == 2.718)
val data = new ConfigsSpec.Data(3)
assert(data == user.withValue("data", data).getValue[ConfigsSpec.Data]("data").get)
system.terminate()
Await.result(system.whenTerminated, Duration.Inf)
}
}
object ConfigsSpec {
case class Data(value: Int)
}
Example 110
| Source File: UtilSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.util
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
import org.apache.gearpump.transport.HostPort
import org.apache.gearpump.util.Util._
class UtilSpec extends FlatSpec with Matchers with MockitoSugar {
it should "work" in {
assert(findFreePort().isSuccess)
assert(randInt() != randInt())
val hosts = parseHostList("host1:1,host2:2")
assert(hosts(1) == HostPort("host2", 2))
assert(Util.getCurrentClassPath.length > 0)
}
it should "check application name properly" in {
assert(Util.validApplicationName("_application_1"))
assert(Util.validApplicationName("application_1_"))
assert(!Util.validApplicationName("0_application_1"))
assert(!Util.validApplicationName("_applicat&&ion_1"))
}
}
Example 111
| Source File: ActorSystemBooterSpec.scala From incubator-retired-gearpump with Apache License 2.0 | 5 votes |
|
package org.apache.gearpump.util
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import akka.actor.{Actor, ActorSystem, Props}
import akka.testkit.TestProbe
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
import org.apache.gearpump.cluster.TestUtil
import org.apache.gearpump.util.ActorSystemBooter.{ActorCreated, RegisterActorSystem, _}
import org.apache.gearpump.util.ActorSystemBooterSpec._
class ActorSystemBooterSpec extends FlatSpec with Matchers with MockitoSugar {
"ActorSystemBooter" should "report its address back" in {
val boot = bootSystem()
boot.prob.expectMsgType[RegisterActorSystem]
boot.shutdown()
}
"ActorSystemBooter" should "terminate itself when parent actor dies" in {
val boot = bootSystem()
boot.prob.expectMsgType[RegisterActorSystem]
val dummy = boot.host.actorOf(Props(classOf[Dummy]), "dummy")
boot.prob.reply(ActorSystemRegistered(boot.prob.ref))
boot.prob.reply(BindLifeCycle(dummy))
boot.host.stop(dummy)
val terminated = retry(5)(boot.bootedSystem.whenTerminated.isCompleted)
assert(terminated)
boot.shutdown()
}
"ActorSystemBooter" should "create new actor" in {
val boot = bootSystem()
boot.prob.expectMsgType[RegisterActorSystem]
boot.prob.reply(ActorSystemRegistered(boot.prob.ref))
boot.prob.reply(CreateActor(Props(classOf[AcceptThreeArguments], 1, 2, 3), "three"))
boot.prob.expectMsgType[ActorCreated]
boot.prob.reply(CreateActor(Props(classOf[AcceptZeroArguments]), "zero"))
boot.prob.expectMsgType[ActorCreated]
boot.shutdown()
}
private def bootSystem(): Boot = {
val booter = ActorSystemBooter(TestUtil.DEFAULT_CONFIG)
val system = ActorSystem("reportback", TestUtil.DEFAULT_CONFIG)
val receiver = TestProbe()(system)
val address = ActorUtil.getFullPath(system, receiver.ref.path)
val bootSystem = booter.boot("booter", address)
Boot(system, receiver, bootSystem)
}
case class Boot(host: ActorSystem, prob: TestProbe, bootedSystem: ActorSystem) {
def shutdown(): Unit = {
host.terminate()
bootedSystem.terminate()
Await.result(host.whenTerminated, Duration.Inf)
Await.result(bootedSystem.whenTerminated, Duration.Inf)
}
}
def retry(seconds: Int)(fn: => Boolean): Boolean = {
val result = fn
if (result) {
result
} else {
Thread.sleep(1000)
retry(seconds - 1)(fn)
}
}
}
object ActorSystemBooterSpec {
class Dummy extends Actor {
def receive: Receive = {
case _ =>
}
}
class AcceptZeroArguments extends Actor {
def receive: Receive = {
case _ =>
}
}
class AcceptThreeArguments(a: Int, b: Int, c: Int) extends Actor {
def receive: Receive = {
case _ =>
}
}
}
Example 112
| Source File: JeroMQSocketSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.communication.socket
import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer
import org.scalatest.{Matchers, BeforeAndAfter, OneInstancePerTest, FunSpec}
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.zeromq.ZMsg
class JeroMQSocketSpec extends FunSpec with MockitoSugar
with OneInstancePerTest with BeforeAndAfter with Matchers
{
private val runnable = mock[ZeroMQSocketRunnable]
@volatile private var running = true
// Mock the running of the runnable for the tests
doAnswer(new Answer[Unit] {
override def answer(invocation: InvocationOnMock): Unit = while (running) {
Thread.sleep(1)
}
}).when(runnable).run()
// Mock the close of the runnable to shutdown
doAnswer(new Answer[Unit] {
override def answer(invocation: InvocationOnMock): Unit = running = false
}).when(runnable).close()
private val socket: JeroMQSocket = new JeroMQSocket(runnable)
after {
running = false
}
describe("JeroMQSocket") {
describe("#send") {
it("should offer a message to the runnable") {
val message: String = "Some Message"
val expected = ZMsg.newStringMsg(message)
socket.send(message.getBytes)
verify(runnable).offer(expected)
}
it("should thrown and AssertionError when socket is no longer alive") {
socket.close()
intercept[AssertionError] {
socket.send("".getBytes)
}
}
}
describe("#close") {
it("should close the runnable") {
socket.close()
verify(runnable).close()
}
it("should close the socket thread") {
socket.close()
socket.isAlive should be (false)
}
}
describe("#isAlive") {
it("should evaluate to true when the socket thread is alive") {
socket.isAlive should be (true)
}
it("should evaluate to false when the socket thread is dead") {
socket.close()
socket.isAlive should be (false)
}
}
}
}
Example 113
| Source File: OrderedSupportSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.communication.utils
import akka.actor._
import akka.testkit.{ImplicitSender, TestKit}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
case class OrderedType()
case class NotOrderedType()
case class FinishProcessingMessage()
case class ReceiveMessageCount(count: Int)
class TestOrderedSupport extends OrderedSupport {
var receivedCounter = 0
override def orderedTypes(): Seq[Class[_]] = Seq(classOf[OrderedType])
override def receive: Receive = {
case OrderedType() =>
startProcessing()
receivedCounter = receivedCounter + 1
sender ! ReceiveMessageCount(receivedCounter)
case NotOrderedType() =>
receivedCounter = receivedCounter + 1
sender ! ReceiveMessageCount(receivedCounter)
case FinishProcessingMessage() =>
finishedProcessing()
}
}
class OrderedSupportSpec extends TestKit(ActorSystem("OrderedSupportSystem"))
with ImplicitSender with Matchers with FunSpecLike
with MockitoSugar {
describe("OrderedSupport"){
describe("#waiting"){
it("should wait for types defined in orderedTypes"){
val testOrderedSupport = system.actorOf(Props[TestOrderedSupport])
// Send a message having a type in orderedTypes
// Starts processing and is handled with receive()
testOrderedSupport ! new OrderedType
// This message should be handled with waiting()
testOrderedSupport ! new OrderedType
// Verify receive was not called for the second OrderedType
expectMsg(ReceiveMessageCount(1))
}
it("should process types not defined in orderedTypes"){
val testOrderedSupport = system.actorOf(Props[TestOrderedSupport])
// Send a message that starts the processing
testOrderedSupport ! new OrderedType
// Send a message having a type not in orderedTypes
testOrderedSupport ! new NotOrderedType
// Verify receive did get called for NotOrderedType
expectMsg(ReceiveMessageCount(1))
expectMsg(ReceiveMessageCount(2))
}
}
describe("#finishedProcessing"){
it("should switch actor to receive method"){
val testOrderedSupport = system.actorOf(Props[TestOrderedSupport])
// Switch actor to waiting mode
testOrderedSupport ! new OrderedType
// Call finishedProcessing
testOrderedSupport ! new FinishProcessingMessage
// Sending something that would match in receive, and is in orderedTypes
testOrderedSupport ! new OrderedType
expectMsg(ReceiveMessageCount(1))
expectMsg(ReceiveMessageCount(2))
}
}
}
}
Example 114
| Source File: JVMReprSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package integration.interpreter.scala
import java.util
import java.io.ByteArrayOutputStream
import jupyter.{Displayer, Displayers, MIMETypes}
import org.apache.toree.global.StreamState
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.interpreter.Results.Success
import org.apache.toree.kernel.api.{DisplayMethodsLike, KernelLike}
import org.apache.toree.kernel.interpreter.scala.ScalaInterpreter
import org.mockito.Mockito.doReturn
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
import org.scalatest.mock.MockitoSugar
import scala.util.Random
class JVMReprSpec extends FunSpec with Matchers with MockitoSugar with BeforeAndAfter {
private val outputResult = new ByteArrayOutputStream()
private var interpreter: Interpreter = _
before {
val mockKernel = mock[KernelLike]
val mockDisplayMethods = mock[DisplayMethodsLike]
doReturn(mockDisplayMethods).when(mockKernel).display
interpreter = new ScalaInterpreter().init(mockKernel)
StreamState.setStreams(outputStream = outputResult)
}
after {
interpreter.stop()
outputResult.reset()
}
describe("ScalaInterpreter") {
describe("#interpret") {
it("should display Scala int as a text representation") {
val (result, outputOrError) = interpreter.interpret("val a = 12")
result should be(Success)
outputOrError.isLeft should be(true)
outputOrError.left.get should be(Map(MIMETypes.TEXT -> "12"))
}
it("should display Scala Some(str) as a text representation") {
val (result, outputOrError) = interpreter.interpret("""val a = Some("str")""")
result should be(Success)
outputOrError.isLeft should be(true)
outputOrError.left.get should be(Map(MIMETypes.TEXT -> "Some(str)"))
}
ignore("should use the Jupyter REPR API for display representation") {
Displayers.register(classOf[DisplayerTest], new Displayer[DisplayerTest] {
override def display(t: DisplayerTest): util.Map[String, String] = {
val output = new util.HashMap[String, String]()
output.put("text/plain", s"test object: ${t.id}")
output.put("application/json", s"""{"id": ${t.id}""")
output
}
})
val inst = DisplayerTest()
interpreter.bind("inst", classOf[DisplayerTest].getName, inst, List())
val (result, outputOrError) = interpreter.interpret("""inst""")
result should be(Success)
outputOrError.isLeft should be(true)
outputOrError.left.get should be(Map(
MIMETypes.TEXT -> s"test object: ${inst.id}",
"application/json" -> s"""{"id": ${inst.id}"""
))
}
}
}
}
case class DisplayerTest(id: Long = new Random().nextLong())
Example 115
| Source File: ClientCommManagerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.comm
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.client.ActorLoader
import org.apache.toree.kernel.protocol.v5.content.CommContent
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
class ClientCommManagerSpec extends FunSpec with Matchers with BeforeAndAfter
with MockitoSugar
{
private val TestTargetName = "some target"
private var mockActorLoader: ActorLoader = _
private var mockKMBuilder: KMBuilder = _
private var mockCommRegistrar: CommRegistrar = _
private var clientCommManager: ClientCommManager = _
private var generatedCommWriter: CommWriter = _
before {
mockActorLoader = mock[ActorLoader]
mockKMBuilder = mock[KMBuilder]
mockCommRegistrar = mock[CommRegistrar]
clientCommManager = new ClientCommManager(
mockActorLoader,
mockKMBuilder,
mockCommRegistrar
) {
override protected def newCommWriter(commId: UUID): CommWriter = {
val commWriter = super.newCommWriter(commId)
generatedCommWriter = commWriter
val spyCommWriter = spy(commWriter)
doNothing().when(spyCommWriter)
.sendCommKernelMessage(any[KernelMessageContent with CommContent])
spyCommWriter
}
}
}
describe("ClientCommManager") {
describe("#open") {
it("should return a wrapped instance of ClientCommWriter") {
clientCommManager.open(TestTargetName, v5.MsgData.Empty)
// Exposed hackishly for testing
generatedCommWriter shouldBe a [ClientCommWriter]
}
}
}
}
Example 116
| Source File: ShellClientSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.client.socket
import java.util.UUID
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.communication.security.SecurityActorType
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.client.ActorLoader
import org.apache.toree.kernel.protocol.v5.content.ExecuteRequest
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import org.mockito.Mockito._
import org.mockito.Matchers._
import play.api.libs.json.Json
class ShellClientSpec extends TestKit(ActorSystem("ShellActorSpec"))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
private val SignatureEnabled = true
describe("ShellClientActor") {
val socketFactory = mock[SocketFactory]
val mockActorLoader = mock[ActorLoader]
val probe : TestProbe = TestProbe()
when(socketFactory.ShellClient(
any(classOf[ActorSystem]), any(classOf[ActorRef])
)).thenReturn(probe.ref)
val signatureManagerProbe = TestProbe()
doReturn(system.actorSelection(signatureManagerProbe.ref.path.toString))
.when(mockActorLoader).load(SecurityActorType.SignatureManager)
val shellClient = system.actorOf(Props(
classOf[ShellClient], socketFactory, mockActorLoader, SignatureEnabled
))
describe("send execute request") {
it("should send execute request") {
val request = ExecuteRequest(
"foo", false, true, UserExpressions(), true
)
val header = Header(
UUID.randomUUID().toString, "spark",
UUID.randomUUID().toString, MessageType.Incoming.ExecuteRequest.toString,
"5.0"
)
val kernelMessage = KernelMessage(
Seq[Array[Byte]](), "",
header, HeaderBuilder.empty,
Metadata(), Json.toJson(request).toString
)
shellClient ! kernelMessage
// Echo back the kernel message sent to have a signature injected
signatureManagerProbe.expectMsgClass(classOf[KernelMessage])
signatureManagerProbe.reply(kernelMessage)
probe.expectMsgClass(classOf[ZMQMessage])
}
}
}
}
Example 117
| Source File: HeartbeatClientSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.client.socket
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5.client.ActorLoader
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import org.mockito.Matchers._
import org.mockito.Mockito._
class HeartbeatClientSpec extends TestKit(ActorSystem("HeartbeatActorSpec"))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("HeartbeatClientActor") {
val socketFactory = mock[SocketFactory]
val mockActorLoader = mock[ActorLoader]
val probe : TestProbe = TestProbe()
when(socketFactory.HeartbeatClient(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(probe.ref)
val heartbeatClient = system.actorOf(Props(
classOf[HeartbeatClient], socketFactory, mockActorLoader, true
))
describe("send heartbeat") {
it("should send ping ZMQMessage") {
heartbeatClient ! HeartbeatMessage
probe.expectMsgClass(classOf[ZMQMessage])
}
}
}
}
Example 118
| Source File: SparkKernelClientSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.client
import akka.actor.ActorSystem
import akka.testkit.{TestKit, TestProbe}
import org.apache.toree.comm.{CommCallbacks, CommStorage, CommRegistrar}
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.client.execution.ExecuteRequestTuple
import scala.concurrent.duration._
import org.mockito.Mockito._
import org.mockito.Matchers.{eq => mockEq, _}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, FunSpecLike, Matchers}
class SparkKernelClientSpec
extends TestKit(ActorSystem("SparkKernelClientActorSystem"))
with Matchers with MockitoSugar with FunSpecLike with BeforeAndAfter
{
private val TestTargetName = "some target"
private var mockActorLoader: ActorLoader = _
private var mockCommRegistrar: CommRegistrar = _
private var sparkKernelClient: SparkKernelClient = _
private var executeRequestProbe: TestProbe = _
private var shellClientProbe: TestProbe = _
before {
mockActorLoader = mock[ActorLoader]
mockCommRegistrar = mock[CommRegistrar]
executeRequestProbe = TestProbe()
when(mockActorLoader.load(MessageType.Incoming.ExecuteRequest))
.thenReturn(system.actorSelection(executeRequestProbe.ref.path.toString))
shellClientProbe = TestProbe()
when(mockActorLoader.load(SocketType.ShellClient))
.thenReturn(system.actorSelection(shellClientProbe.ref.path.toString))
sparkKernelClient = new SparkKernelClient(
mockActorLoader, system, mockCommRegistrar)
}
describe("SparkKernelClient") {
describe("#execute") {
it("should send an ExecuteRequest message") {
val func = (x: Any) => println(x)
sparkKernelClient.execute("val foo = 2")
executeRequestProbe.expectMsgClass(classOf[ExecuteRequestTuple])
}
}
}
}
Example 119
| Source File: DataFrameConverterSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{DataFrame, Row}
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfterAll, FunSpec, Matchers}
import play.api.libs.json.{JsArray, JsString, Json}
import test.utils.SparkContextProvider
import scala.collection.mutable
class DataFrameConverterSpec extends FunSpec with MockitoSugar with Matchers with BeforeAndAfterAll {
lazy val spark = SparkContextProvider.sparkContext
override protected def afterAll(): Unit = {
spark.stop()
super.afterAll()
}
val dataFrameConverter: DataFrameConverter = new DataFrameConverter
val mockDataFrame = mock[DataFrame]
val mockRdd = spark.parallelize(Seq(Row(new mutable.WrappedArray.ofRef(Array("test1", "test2")), 2, null)))
val mockStruct = mock[StructType]
val columns = Seq("foo", "bar").toArray
doReturn(mockStruct).when(mockDataFrame).schema
doReturn(columns).when(mockStruct).fieldNames
doReturn(mockRdd).when(mockDataFrame).rdd
describe("DataFrameConverter") {
describe("#convert") {
it("should convert to a valid JSON object") {
val someJson = dataFrameConverter.convert(mockDataFrame, "json")
val jsValue = Json.parse(someJson.get)
jsValue \ "columns" should be (JsArray(Seq(JsString("foo"), JsString("bar"))))
jsValue \ "rows" should be (JsArray(Seq(
JsArray(Seq(JsString("[test1, test2]"), JsString("2"), JsString("null")))
)))
}
it("should convert to csv") {
val csv = dataFrameConverter.convert(mockDataFrame, "csv").get
val values = csv.split("\n")
values(0) shouldBe "foo,bar"
values(1) shouldBe "[test1, test2],2,null"
}
it("should convert to html") {
val html = dataFrameConverter.convert(mockDataFrame, "html").get
html.contains("<th>foo</th>") should be(true)
html.contains("<th>bar</th>") should be(true)
html.contains("<td>[test1, test2]</td>") should be(true)
html.contains("<td>2</td>") should be(true)
html.contains("<td>null</td>") should be(true)
}
it("should convert limit the selection") {
val someLimited = dataFrameConverter.convert(mockDataFrame, "csv", 1)
val limitedLines = someLimited.get.split("\n")
limitedLines.length should be(2)
}
it("should return a Failure for invalid types") {
val result = dataFrameConverter.convert(mockDataFrame, "Invalid Type")
result.isFailure should be(true)
}
}
}
}
Example 120
| Source File: KernelCommManagerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.comm
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content.CommContent
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
class KernelCommManagerSpec extends FunSpec with Matchers with BeforeAndAfter
with MockitoSugar
{
private val TestTargetName = "some target"
private var mockActorLoader: ActorLoader = _
private var mockKMBuilder: KMBuilder = _
private var mockCommRegistrar: CommRegistrar = _
private var kernelCommManager: KernelCommManager = _
private var generatedCommWriter: CommWriter = _
before {
mockActorLoader = mock[ActorLoader]
mockKMBuilder = mock[KMBuilder]
mockCommRegistrar = mock[CommRegistrar]
kernelCommManager = new KernelCommManager(
mockActorLoader,
mockKMBuilder,
mockCommRegistrar
) {
override protected def newCommWriter(commId: UUID): CommWriter = {
val commWriter = super.newCommWriter(commId)
generatedCommWriter = commWriter
val spyCommWriter = spy(commWriter)
doNothing().when(spyCommWriter)
.sendCommKernelMessage(any[KernelMessageContent with CommContent])
spyCommWriter
}
}
}
describe("KernelCommManager") {
describe("#open") {
it("should return a wrapped instance of KernelCommWriter") {
kernelCommManager.open(TestTargetName, v5.MsgData.Empty)
// Exposed hackishly for testing
generatedCommWriter shouldBe a [KernelCommWriter]
}
}
}
}
Example 121
| Source File: LSMagicSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import java.io.OutputStream
import java.net.URL
import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.dependencies.{IncludeOutputStream, IncludeInterpreter}
import org.apache.toree.magic.{CellMagic, LineMagic}
import org.apache.spark.SparkContext
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
class TestLSMagic(sc: SparkContext, intp: Interpreter, os: OutputStream)
extends LSMagic
with IncludeInterpreter
with IncludeOutputStream
{
override val interpreter: Interpreter = intp
override val outputStream: OutputStream = os
}
class LSMagicSpec extends FunSpec with Matchers with MockitoSugar {
describe("LSMagic") {
describe("#execute") {
it("should call println with a magics message") {
val lsm = spy(new TestLSMagic(
mock[SparkContext], mock[Interpreter], mock[OutputStream])
)
val classList = new BuiltinLoader().loadClasses()
lsm.execute("")
verify(lsm).magicNames("%", classOf[LineMagic], classList)
verify(lsm).magicNames("%%", classOf[CellMagic], classList)
}
}
describe("#magicNames") {
it("should filter classnames by interface") {
val prefix = "%"
val interface = classOf[LineMagic]
val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
val lsm = new TestLSMagic(
mock[SparkContext], mock[Interpreter], mock[OutputStream])
lsm.magicNames(prefix, interface, classes).length should be(1)
}
it("should prepend prefix to each name"){
val prefix = "%"
val className = classOf[LSMagic].getSimpleName
val interface = classOf[LineMagic]
val expected = s"${prefix}${className}"
val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
val lsm = new TestLSMagic(
mock[SparkContext], mock[Interpreter], mock[OutputStream])
lsm.magicNames(prefix, interface, classes) should be(List(expected))
}
}
}
}
Example 122
| Source File: BuiltinLoaderSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpec}
class BuiltinLoaderSpec extends FunSpec with Matchers with MockitoSugar {
describe("BuiltinLoader") {
describe("#getClasses") {
it("should return classes in a package") {
val pkg = this.getClass.getPackage.getName
val classes = new BuiltinLoader().getClasses(pkg)
classes.size shouldNot be(0)
}
}
describe("#loadClasses") {
it("should return class objects for classes in a package") {
val pkg = this.getClass.getPackage.getName
val classes = new BuiltinLoader().loadClasses(pkg).toList
classes.contains(this.getClass) should be (true)
}
}
}
}
Example 123
| Source File: HtmlSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import org.apache.toree.kernel.protocol.v5.MIMEType
import org.apache.toree.magic.CellMagicOutput
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers}
class HtmlSpec extends FunSpec with Matchers with MockitoSugar {
describe("Html"){
describe("#execute") {
it("should return the entire cell's contents with the MIME type of " +
"text/html") {
val htmlMagic = new Html
val code = "some code on a line\nanother line"
val expected = CellMagicOutput(MIMEType.TextHtml -> code)
htmlMagic.execute(code) should be (expected)
}
}
}
}
Example 124
| Source File: JavaScriptSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic.builtin
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers}
import org.apache.toree.magic.CellMagicOutput
import org.apache.toree.kernel.protocol.v5.MIMEType
class JavaScriptSpec extends FunSpec with Matchers with MockitoSugar {
describe("JavaScript"){
describe("#execute") {
it("should return the entire cell's contents with the MIME type of text/javascript") {
val javaScriptMagic = new JavaScript
val code = "some code on a line\nmore code on another line"
val expected = CellMagicOutput(MIMEType.ApplicationJavaScript -> code)
javaScriptMagic.execute(code) should be (expected)
}
}
}
}
Example 125
| Source File: CodeCompleteHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.handler
import akka.actor._
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import org.apache.toree.Main
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content.CompleteRequest
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5Test._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, BeforeAndAfter, Matchers}
import org.mockito.Mockito._
import test.utils.MaxAkkaTestTimeout
class CodeCompleteHandlerSpec extends TestKit(
ActorSystem("CodeCompleteHandlerSpec", None, Some(Main.getClass.getClassLoader))
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar
with BeforeAndAfter {
var actorLoader: ActorLoader = _
var handlerActor: ActorRef = _
var kernelMessageRelayProbe: TestProbe = _
var interpreterProbe: TestProbe = _
var statusDispatchProbe: TestProbe = _
before {
actorLoader = mock[ActorLoader]
handlerActor = system.actorOf(Props(classOf[CodeCompleteHandler], actorLoader))
kernelMessageRelayProbe = TestProbe()
when(actorLoader.load(SystemActorType.KernelMessageRelay))
.thenReturn(system.actorSelection(kernelMessageRelayProbe.ref.path.toString))
interpreterProbe = new TestProbe(system)
when(actorLoader.load(SystemActorType.Interpreter))
.thenReturn(system.actorSelection(interpreterProbe.ref.path.toString))
statusDispatchProbe = new TestProbe(system)
when(actorLoader.load(SystemActorType.StatusDispatch))
.thenReturn(system.actorSelection(statusDispatchProbe.ref.path.toString))
}
def replyToHandlerWithOkAndResult() = {
val expectedClass = classOf[CompleteRequest]
interpreterProbe.expectMsgClass(expectedClass)
interpreterProbe.reply((0, List[String]()))
}
def replyToHandlerWithOkAndBadResult() = {
val expectedClass = classOf[CompleteRequest]
interpreterProbe.expectMsgClass(expectedClass)
interpreterProbe.reply("hello")
}
describe("CodeCompleteHandler (ActorLoader)") {
it("should send a CompleteRequest") {
handlerActor ! MockCompleteRequestKernelMessage
replyToHandlerWithOkAndResult()
kernelMessageRelayProbe.fishForMessage(MaxAkkaTestTimeout) {
case KernelMessage(_, _, header, _, _, _) =>
header.msg_type == MessageType.Outgoing.CompleteReply.toString
}
}
it("should throw an error for bad JSON") {
handlerActor ! MockKernelMessageWithBadJSON
var result = false
try {
replyToHandlerWithOkAndResult()
}
catch {
case t: Throwable => result = true
}
result should be (true)
}
it("should throw an error for bad code completion") {
handlerActor ! MockCompleteRequestKernelMessage
try {
replyToHandlerWithOkAndBadResult()
}
catch {
case error: Exception => error.getMessage should be ("Parse error in CodeCompleteHandler")
}
}
it("should send an idle message") {
handlerActor ! MockCompleteRequestKernelMessage
replyToHandlerWithOkAndResult()
statusDispatchProbe.fishForMessage(MaxAkkaTestTimeout) {
case Tuple2(status, _) =>
status == KernelStatusType.Idle
}
}
}
}
Example 126
| Source File: GenericSocketMessageHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.handler
import akka.actor.{ActorSystem, Props, ActorRef, ActorSelection}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5Test._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import test.utils.MaxAkkaTestTimeout
class GenericSocketMessageHandlerSpec extends TestKit(
ActorSystem(
"GenericSocketMessageHandlerSystem",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("GenericSocketMessageHandler( ActorLoader, SocketType )") {
// Create a mock ActorLoader for the Relay we are going to test
val actorLoader: ActorLoader = mock[ActorLoader]
// Create a probe for the ActorSelection that the ActorLoader will return
val selectionProbe: TestProbe = TestProbe()
val selection: ActorSelection = system.actorSelection(selectionProbe.ref.path.toString)
when(actorLoader.load(SocketType.Control)).thenReturn(selection)
// The Relay we are going to be testing against
val genericHandler: ActorRef = system.actorOf(
Props(classOf[GenericSocketMessageHandler], actorLoader, SocketType.Control)
)
describe("#receive( KernelMessage )") {
genericHandler ! MockKernelMessage
it("should send the message to the selected actor"){
selectionProbe.expectMsg(MaxAkkaTestTimeout, MockKernelMessage)
}
}
}
}
Example 127
| Source File: KernelInfoRequestHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.handler
import akka.actor.{ActorSelection, ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.Main
import org.apache.toree.kernel.protocol.v5.content.KernelInfoReply
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5._
import org.mockito.AdditionalMatchers.{not => mockNot}
import org.mockito.Matchers.{eq => mockEq}
import com.typesafe.config.ConfigFactory
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import play.api.libs.json.Json
import test.utils.MaxAkkaTestTimeout
object KernelInfoRequestHandlerSpec {
val config = """
akka {
loglevel = "WARNING"
}"""
}
class KernelInfoRequestHandlerSpec extends TestKit(
ActorSystem("KernelInfoRequestHandlerSpec",
ConfigFactory.parseString(KernelInfoRequestHandlerSpec.config),
Main.getClass.getClassLoader)
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val actorLoader: ActorLoader = mock[ActorLoader]
val actor = system.actorOf(Props(classOf[KernelInfoRequestHandler], actorLoader, LanguageInfo("test", "1.0.0", Some(".test"))))
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection =
system.actorSelection(relayProbe.ref.path)
when(actorLoader.load(SystemActorType.KernelMessageRelay))
.thenReturn(relaySelection)
when(actorLoader.load(mockNot(mockEq(SystemActorType.KernelMessageRelay))))
.thenReturn(system.actorSelection(""))
val header = Header("","","","","")
val kernelMessage = new KernelMessage(
Seq[Array[Byte]](), "test message", header, header, Metadata(), "{}"
)
describe("Kernel Info Request Handler") {
it("should return a KernelMessage containing kernel info response") {
actor ! kernelMessage
val reply = relayProbe.receiveOne(MaxAkkaTestTimeout).asInstanceOf[KernelMessage]
val kernelInfo = Json.parse(reply.contentString).as[KernelInfoReply]
kernelInfo.implementation should be ("spark")
}
}
}
Example 128
| Source File: ShellSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import java.nio.charset.Charset
import akka.actor.{ActorSelection, ActorRef, ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import akka.util.ByteString
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.kernel.{ActorLoader, Utilities}
import org.apache.toree.kernel.protocol.v5Test._
import Utilities._
import com.typesafe.config.ConfigFactory
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.MaxAkkaTestTimeout
object ShellSpec {
val config ="""
akka {
loglevel = "WARNING"
}"""
}
class ShellSpec extends TestKit(
ActorSystem(
"ShellActorSpec",
ConfigFactory.parseString(ShellSpec.config),
org.apache.toree.Main.getClass.getClassLoader
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("Shell") {
val socketFactory = mock[SocketFactory]
val actorLoader = mock[ActorLoader]
val socketProbe : TestProbe = TestProbe()
when(socketFactory.Shell(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(socketProbe.ref)
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection = system.actorSelection(relayProbe.ref.path)
when(actorLoader.load(SystemActorType.KernelMessageRelay)).thenReturn(relaySelection)
val shell = system.actorOf(Props(classOf[Shell], socketFactory, actorLoader))
describe("#receive") {
it("( KernelMessage ) should reply with a ZMQMessage via the socket") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
shell ! MockKernelMessage
socketProbe.expectMsg(MockZMQMessage)
}
it("( ZMQMessage ) should forward ZMQ Strings and KernelMessage to Relay") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
shell ! MockZMQMessage
// Should get the last four (assuming no buffer) strings in UTF-8
val zmqStrings = MockZMQMessage.frames.map((byteString: ByteString) =>
new String(byteString.toArray, Charset.forName("UTF-8"))
).takeRight(4)
val kernelMessage: KernelMessage = MockZMQMessage
relayProbe.expectMsg(MaxAkkaTestTimeout, (zmqStrings, kernelMessage))
}
}
}
}
Example 129
| Source File: IOPubSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import akka.actor.{ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5.kernel.Utilities
import org.apache.toree.kernel.protocol.v5Test._
import Utilities._
import com.typesafe.config.ConfigFactory
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.MaxAkkaTestTimeout
object IOPubSpec {
val config ="""
akka {
loglevel = "WARNING"
}"""
}
class IOPubSpec extends TestKit(
ActorSystem("IOPubActorSpec",
ConfigFactory.parseString(IOPubSpec.config),
org.apache.toree.Main.getClass.getClassLoader
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("IOPubActor") {
val socketFactory = mock[SocketFactory]
val probe : TestProbe = TestProbe()
when(socketFactory.IOPub(any(classOf[ActorSystem]))).thenReturn(probe.ref)
val socket = system.actorOf(Props(classOf[IOPub], socketFactory))
// TODO test that the response type changed
describe("#receive") {
it("should reply with a ZMQMessage") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
socket ! MockKernelMessage
probe.expectMsg(MaxAkkaTestTimeout, MockZMQMessage)
}
}
}
}
Example 130
| Source File: HeartbeatSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import akka.util.ByteString
import org.apache.toree.communication.ZMQMessage
import com.typesafe.config.ConfigFactory
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.MaxAkkaTestTimeout
object HeartbeatSpec {
val config = """
akka {
loglevel = "WARNING"
}"""
}
class HeartbeatSpec extends TestKit(
ActorSystem(
"HeartbeatActorSpec",
ConfigFactory.parseString(HeartbeatSpec.config),
org.apache.toree.Main.getClass.getClassLoader
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
val SomeMessage: String = "some message"
val SomeZMQMessage: ZMQMessage = ZMQMessage(ByteString(SomeMessage.getBytes))
describe("HeartbeatActor") {
val socketFactory = mock[SocketFactory]
val probe : TestProbe = TestProbe()
when(socketFactory.Heartbeat(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(probe.ref)
val heartbeat = system.actorOf(Props(classOf[Heartbeat], socketFactory))
describe("send heartbeat") {
it("should receive and send same ZMQMessage") {
heartbeat ! SomeZMQMessage
probe.expectMsg(MaxAkkaTestTimeout, SomeZMQMessage)
}
}
}
}
Example 131
| Source File: StdinSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel.socket
import java.nio.charset.Charset
import akka.actor.{Props, ActorSelection, ActorRef, ActorSystem}
import akka.testkit.{TestProbe, ImplicitSender, TestKit}
import akka.util.ByteString
import org.apache.toree.communication.ZMQMessage
import org.apache.toree.kernel.protocol.v5.kernel.Utilities._
import org.apache.toree.kernel.protocol.v5Test._
import org.apache.toree.kernel.protocol.v5.{KernelMessage, SystemActorType}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import com.typesafe.config.ConfigFactory
import org.scalatest.mock.MockitoSugar
import org.scalatest.{Matchers, FunSpecLike}
import org.mockito.Mockito._
import org.mockito.Matchers._
import test.utils.MaxAkkaTestTimeout
object StdinSpec {
val config ="""
akka {
loglevel = "WARNING"
}"""
}
class StdinSpec extends TestKit(ActorSystem(
"StdinActorSpec",
ConfigFactory.parseString(StdinSpec.config),
org.apache.toree.Main.getClass.getClassLoader
)) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("Stdin") {
val socketFactory = mock[SocketFactory]
val actorLoader = mock[ActorLoader]
val socketProbe : TestProbe = TestProbe()
when(socketFactory.Stdin(any(classOf[ActorSystem]), any(classOf[ActorRef]))).thenReturn(socketProbe.ref)
val relayProbe : TestProbe = TestProbe()
val relaySelection : ActorSelection = system.actorSelection(relayProbe.ref.path)
when(actorLoader.load(SystemActorType.KernelMessageRelay)).thenReturn(relaySelection)
val stdin = system.actorOf(Props(classOf[Stdin], socketFactory, actorLoader))
describe("#receive") {
it("( KernelMessage ) should reply with a ZMQMessage via the socket") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
stdin ! MockKernelMessage
socketProbe.expectMsg(MockZMQMessage)
}
it("( ZMQMessage ) should forward ZMQ Strings and KernelMessage to Relay") {
// Use the implicit to convert the KernelMessage to ZMQMessage
val MockZMQMessage : ZMQMessage = MockKernelMessage
stdin ! MockZMQMessage
// Should get the last four (assuming no buffer) strings in UTF-8
val zmqStrings = MockZMQMessage.frames.map((byteString: ByteString) =>
new String(byteString.toArray, Charset.forName("UTF-8"))
).takeRight(4)
val kernelMessage: KernelMessage = MockZMQMessage
relayProbe.expectMsg(MaxAkkaTestTimeout, (zmqStrings, kernelMessage))
}
}
}
}
Example 132
| Source File: ActorLoaderSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.kernel
import akka.actor.{ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5.{MessageType, SocketType}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, Matchers}
import test.utils.TestProbeProxyActor
import test.utils.MaxAkkaTestTimeout
class ActorLoaderSpec extends TestKit(
ActorSystem(
"ActorLoaderSpecSystem",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
))
with ImplicitSender with FunSpecLike with Matchers with MockitoSugar {
describe("ActorLoader"){
describe("#load( MessageType )"){
it("should load an ActorSelection that has been loaded into the system"){
val testProbe: TestProbe = TestProbe()
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe),
MessageType.Outgoing.ClearOutput.toString)
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(MessageType.Outgoing.ClearOutput) ! "<Test Message>"
testProbe.expectMsg("<Test Message>")
}
it("should expect no message when there is no actor"){
val testProbe: TestProbe = TestProbe()
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(MessageType.Outgoing.CompleteReply) ! "<Test Message>"
testProbe.expectNoMessage(MaxAkkaTestTimeout)
// This is to test to see if there the messages go to the actor inbox or the dead mail inbox
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe),
MessageType.Outgoing.CompleteReply.toString)
testProbe.expectNoMessage(MaxAkkaTestTimeout)
}
}
describe("#load( SocketType )"){
it("should load an ActorSelection that has been loaded into the system"){
val testProbe: TestProbe = TestProbe()
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe), SocketType.Shell.toString)
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(SocketType.Shell) ! "<Test Message>"
testProbe.expectMsg("<Test Message>")
}
it("should expect no message when there is no actor"){
val testProbe: TestProbe = TestProbe()
val actorLoader: ActorLoader = SimpleActorLoader(system)
actorLoader.load(SocketType.IOPub) ! "<Test Message>"
testProbe.expectNoMessage(MaxAkkaTestTimeout)
// This is to test to see if there the messages go to the actor inbox or the dead mail inbox
system.actorOf(Props(classOf[TestProbeProxyActor], testProbe), SocketType.IOPub.toString)
testProbe.expectNoMessage(MaxAkkaTestTimeout)
}
}
}
}
Example 133
| Source File: StatusDispatchSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.protocol.v5.dispatch
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.{TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content.KernelStatus
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, FunSpecLike, Matchers}
import play.api.libs.json.Json
import test.utils.MaxAkkaTestTimeout
class StatusDispatchSpec extends TestKit(
ActorSystem(
"StatusDispatchSystem",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
)
)
with FunSpecLike with Matchers with MockitoSugar with BeforeAndAfter{
var statusDispatchRef: ActorRef = _
var relayProbe: TestProbe = _
before {
// Mock the relay with a probe
relayProbe = TestProbe()
// Mock the ActorLoader
val mockActorLoader: ActorLoader = mock[ActorLoader]
when(mockActorLoader.load(SystemActorType.KernelMessageRelay))
.thenReturn(system.actorSelection(relayProbe.ref.path.toString))
statusDispatchRef = system.actorOf(Props(classOf[StatusDispatch],mockActorLoader))
}
describe("StatusDispatch") {
describe("#receive( KernelStatusType )") {
it("should send a status message to the relay") {
statusDispatchRef ! KernelStatusType.Busy
// Check the kernel message is the correct type
val statusMessage: KernelMessage = relayProbe.receiveOne(MaxAkkaTestTimeout).asInstanceOf[KernelMessage]
statusMessage.header.msg_type should be (MessageType.Outgoing.Status.toString)
// Check the status is what we sent
val status: KernelStatus = Json.parse(statusMessage.contentString).as[KernelStatus]
status.execution_state should be (KernelStatusType.Busy.toString)
}
}
describe("#receive( KernelStatusType, Header )") {
it("should send a status message to the relay") {
val tuple = Tuple2(KernelStatusType.Busy, mock[Header])
statusDispatchRef ! tuple
// Check the kernel message is the correct type
val statusMessage: KernelMessage = relayProbe.receiveOne(MaxAkkaTestTimeout).asInstanceOf[KernelMessage]
statusMessage.header.msg_type should be (MessageType.Outgoing.Status.toString)
// Check the status is what we sent
val status: KernelStatus = Json.parse(statusMessage.contentString).as[KernelStatus]
status.execution_state should be (KernelStatusType.Busy.toString)
}
}
}
}
Example 134
| Source File: StreamMethodsSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.kernel.api
import akka.actor.ActorSystem
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5.KernelMessage
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpecLike, BeforeAndAfter, Matchers}
import play.api.libs.json.Json
import test.utils.MaxAkkaTestTimeout
import org.mockito.Mockito._
class StreamMethodsSpec extends TestKit(
ActorSystem(
"StreamMethodsSpec",
None,
Some(org.apache.toree.Main.getClass.getClassLoader)
)
) with ImplicitSender with FunSpecLike with Matchers with MockitoSugar
with BeforeAndAfter
{
private var kernelMessageRelayProbe: TestProbe = _
private var mockParentHeader: v5.ParentHeader = _
private var mockActorLoader: v5.kernel.ActorLoader = _
private var mockKernelMessage: v5.KernelMessage = _
private var streamMethods: StreamMethods = _
before {
kernelMessageRelayProbe = TestProbe()
mockParentHeader = mock[v5.ParentHeader]
mockActorLoader = mock[v5.kernel.ActorLoader]
doReturn(system.actorSelection(kernelMessageRelayProbe.ref.path))
.when(mockActorLoader).load(v5.SystemActorType.KernelMessageRelay)
mockKernelMessage = mock[v5.KernelMessage]
doReturn(mockParentHeader).when(mockKernelMessage).header
streamMethods = new StreamMethods(mockActorLoader, mockKernelMessage)
}
describe("StreamMethods") {
describe("#()") {
it("should put the header of the given message as the parent header") {
val expected = mockKernelMessage.header
val actual = streamMethods.kmBuilder.build.parentHeader
actual should be (expected)
}
}
describe("#sendAll") {
it("should send a message containing all of the given text") {
val expected = "some text"
streamMethods.sendAll(expected)
val outgoingMessage = kernelMessageRelayProbe.receiveOne(MaxAkkaTestTimeout)
val kernelMessage = outgoingMessage.asInstanceOf[KernelMessage]
val actual = Json.parse(kernelMessage.contentString)
.as[v5.content.StreamContent].text
actual should be (expected)
}
}
}
}
Example 135
| Source File: MultiOutputStreamSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import java.io.OutputStream
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
import org.mockito.Matchers._
import org.mockito.Mockito._
class MultiOutputStreamSpec
extends FunSpec with Matchers with MockitoSugar with BeforeAndAfter {
describe("MultiOutputStream") {
val listOfMockOutputStreams = List(mock[OutputStream], mock[OutputStream])
val multiOutputStream = MultiOutputStream(listOfMockOutputStreams)
describe("#close") {
it("should call #close on all internal output streams") {
multiOutputStream.close()
listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).close())
}
}
describe("#flush") {
it("should call #flush on all internal output streams") {
multiOutputStream.flush()
listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).flush())
}
}
describe("#write(int)") {
it("should call #write(int) on all internal output streams") {
multiOutputStream.write(anyInt())
listOfMockOutputStreams.foreach(
mockOutputStream => verify(mockOutputStream).write(anyInt()))
}
}
describe("#write(byte[])") {
it("should call #write(byte[]) on all internal output streams") {
multiOutputStream.write(any[Array[Byte]])
listOfMockOutputStreams.foreach(
mockOutputStream => verify(mockOutputStream).write(any[Array[Byte]]))
}
}
describe("#write(byte[], int, int)") {
it("should call #write(byte[], int, int) on all internal output streams") {
multiOutputStream.write(any[Array[Byte]], anyInt(), anyInt())
listOfMockOutputStreams.foreach(
mockOutputStream =>
verify(mockOutputStream).write(any[Array[Byte]], anyInt(), anyInt()))
}
}
}
}
Example 136
| Source File: ArgumentParsingSupportSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
import joptsimple.{OptionSet, OptionSpec, OptionParser}
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import collection.JavaConverters._
class ArgumentParsingSupportSpec extends FunSpec with Matchers
with BeforeAndAfter with MockitoSugar
{
private var mockOptions: OptionSet = _
private var mockParser: OptionParser = _
private var argumentParsingInstance: ArgumentParsingSupport = _
before {
mockOptions = mock[OptionSet]
mockParser = mock[OptionParser]
doReturn(mockOptions).when(mockParser).parse(anyVararg[String]())
argumentParsingInstance = new Object() with ArgumentParsingSupport {
override protected lazy val parser: OptionParser = mockParser
}
}
describe("ArgumentParsingSupport") {
describe("#parseArgs") {
it("should invoke the underlying parser's parse method") {
doReturn(Nil.asJava).when(mockOptions).nonOptionArguments()
argumentParsingInstance.parseArgs("")
verify(mockParser).parse(anyString())
}
it("should return an empty list if there are no non-option arguments") {
val expected = Nil
doReturn(expected.asJava).when(mockOptions).nonOptionArguments()
val actual = argumentParsingInstance.parseArgs((
"--transitive" :: expected
).mkString(" "))
actual should be (expected)
}
it("should return a list containing non-option arguments") {
val expected = "non-option" :: Nil
doReturn(expected.asJava).when(mockOptions).nonOptionArguments()
val actual = argumentParsingInstance.parseArgs((
"--transitive" :: expected
).mkString(" "))
actual should be (expected)
}
}
}
}
Example 137
| Source File: ConditionalOutputStreamSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.utils
import java.io.OutputStream
import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{Matchers, FunSpec}
class ConditionalOutputStreamSpec extends FunSpec with Matchers with MockitoSugar {
describe("ConditionalOutputStream") {
describe("#()") {
it("should throw an exception if the output stream is null") {
intercept[IllegalArgumentException] {
new ConditionalOutputStream(null, true)
}
}
}
describe("#write") {
it("should call the underlying write if the condition is true") {
val mockOutputStream = mock[OutputStream]
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, true)
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream).write(expected)
}
it("should call the underlying write if the condition becomes true") {
val mockOutputStream = mock[OutputStream]
var condition = false
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, condition)
condition = true
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream).write(expected)
}
it("should not call the underlying write if the condition is false") {
val mockOutputStream = mock[OutputStream]
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, false)
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream, never()).write(any[Byte])
}
it("should not call the underlying write if the condition becomes false") {
val mockOutputStream = mock[OutputStream]
var condition = true
val conditionalOutputStream =
new ConditionalOutputStream(mockOutputStream, condition)
condition = false
val expected = 101
conditionalOutputStream.write(expected)
verify(mockOutputStream, never()).write(any[Byte])
}
}
}
}
Example 138
| Source File: InternalClassLoaderSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.magic
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar
class InternalClassLoaderSpec extends FunSpec with Matchers with MockitoSugar {
abstract class MockClassLoader extends ClassLoader(null) {
override def loadClass(name: String): Class[_] = null
}
describe("InternalClassLoader") {
describe("#loadClass") {
it("should invoke super loadClass with loader's package prepended") {
val expected = classOf[Class[_]]
val packageName = "org.apache.toree.magic"
val className = "SomeClass"
var parentLoadClassCorrectlyInvoked = false
val internalClassLoader = new InternalClassLoader(null) {
override private[magic] def parentLoadClass(name: String, resolve: Boolean): Class[_] = {
parentLoadClassCorrectlyInvoked =
name == s"$packageName.$className" && resolve
expected
}
}
internalClassLoader.loadClass(className, true) should be (expected)
parentLoadClassCorrectlyInvoked should be (true)
}
it("should use loader's package instead of provided package first") {
val expected = classOf[Class[_]]
val forcedPackageName = "org.apache.toree.magic"
val packageName = "some.other.package"
val className = "SomeClass"
var parentLoadClassCorrectlyInvoked = false
val internalClassLoader = new InternalClassLoader(null) {
override private[magic] def parentLoadClass(name: String, resolve: Boolean): Class[_] = {
parentLoadClassCorrectlyInvoked =
name == s"$forcedPackageName.$className" && resolve
expected
}
}
internalClassLoader.loadClass(s"$packageName.$className", true) should be (expected)
parentLoadClassCorrectlyInvoked should be (true)
}
it("should invoke super loadClass with given package if internal missing") {
val expected = classOf[Class[_]]
val packageName = "some.other.package"
val className = "SomeClass"
var parentLoadClassCorrectlyInvoked = false
var methodCalled = false
val internalClassLoader = new InternalClassLoader(null) {
override private[magic] def parentLoadClass(name: String, resolve: Boolean): Class[_] = {
if (!methodCalled) {
methodCalled = true
throw new ClassNotFoundException()
}
parentLoadClassCorrectlyInvoked =
name == s"$packageName.$className" && resolve
expected
}
}
internalClassLoader.loadClass(s"$packageName.$className", true) should
be (expected)
parentLoadClassCorrectlyInvoked should be (true)
}
}
}
}
Example 139
| Source File: BrokerProcessHandlerSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.interpreter.broker
import org.apache.commons.exec.ExecuteException
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers, OneInstancePerTest}
import org.mockito.Mockito._
import org.mockito.Matchers._
class BrokerProcessHandlerSpec extends FunSpec with Matchers
with OneInstancePerTest with MockitoSugar
{
private val mockBrokerBridge = mock[BrokerBridge]
private val brokerProcessHandler = new BrokerProcessHandler(
mockBrokerBridge,
restartOnFailure = true,
restartOnCompletion = true
)
describe("BrokerProcessHandler") {
describe("#onProcessFailed") {
it("should invoke the reset method") {
val mockResetMethod = mock[String => Unit]
brokerProcessHandler.setResetMethod(mockResetMethod)
brokerProcessHandler.onProcessFailed(mock[ExecuteException])
verify(mockResetMethod).apply(anyString())
}
it("should invoke the restart method if the proper flag is set to true") {
val mockRestartMethod = mock[() => Unit]
brokerProcessHandler.setRestartMethod(mockRestartMethod)
brokerProcessHandler.onProcessFailed(mock[ExecuteException])
verify(mockRestartMethod).apply()
}
}
describe("#onProcessComplete") {
it("should invoke the reset method") {
val mockResetMethod = mock[String => Unit]
brokerProcessHandler.setResetMethod(mockResetMethod)
brokerProcessHandler.onProcessComplete(0)
verify(mockResetMethod).apply(anyString())
}
it("should invoke the restart method if the proper flag is set to true") {
val mockRestartMethod = mock[() => Unit]
brokerProcessHandler.setRestartMethod(mockRestartMethod)
brokerProcessHandler.onProcessComplete(0)
verify(mockRestartMethod).apply()
}
}
}
}
Example 140
| Source File: BrokerBridgeSpec.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.toree.interpreter.broker
import org.apache.toree.kernel.api.KernelLike
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FunSpec, Matchers, OneInstancePerTest}
class BrokerBridgeSpec extends FunSpec with Matchers with OneInstancePerTest
with MockitoSugar
{
private val mockBrokerState = mock[BrokerState]
private val mockKernel = mock[KernelLike]
private val brokerBridge = new BrokerBridge(
mockBrokerState,
mockKernel
)
describe("BrokerBridge") {
describe("#state") {
it("should return the broker state from the constructor") {
brokerBridge.state should be (mockBrokerState)
}
}
describe("#kernel") {
it("should return the kernel from the constructor") {
brokerBridge.kernel should be (mockKernel)
}
}
}
}
Example 141
| Source File: PartiallyUnrolledIteratorSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.storage
import org.mockito.Matchers
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.apache.spark.SparkFunSuite
import org.apache.spark.memory.MemoryMode.ON_HEAP
import org.apache.spark.storage.memory.{MemoryStore, PartiallyUnrolledIterator}
class PartiallyUnrolledIteratorSuite extends SparkFunSuite with MockitoSugar {
test("join two iterators") {
val unrollSize = 1000
val unroll = (0 until unrollSize).iterator
val restSize = 500
val rest = (unrollSize until restSize + unrollSize).iterator
val memoryStore = mock[MemoryStore]
val joinIterator = new PartiallyUnrolledIterator(memoryStore, ON_HEAP, unrollSize, unroll, rest)
// Firstly iterate over unrolling memory iterator
(0 until unrollSize).foreach { value =>
assert(joinIterator.hasNext)
assert(joinIterator.hasNext)
assert(joinIterator.next() == value)
}
joinIterator.hasNext
joinIterator.hasNext
verify(memoryStore, times(1))
.releaseUnrollMemoryForThisTask(Matchers.eq(ON_HEAP), Matchers.eq(unrollSize.toLong))
// Secondly, iterate over rest iterator
(unrollSize until unrollSize + restSize).foreach { value =>
assert(joinIterator.hasNext)
assert(joinIterator.hasNext)
assert(joinIterator.next() == value)
}
joinIterator.close()
// MemoryMode.releaseUnrollMemoryForThisTask is called only once
verifyNoMoreInteractions(memoryStore)
}
}
Example 142
| Source File: NettyBlockTransferSecuritySuite.scala From SparkCore with Apache License 2.0 | 5 votes |
|
package org.apache.spark.network.netty
import java.nio._
import java.util.concurrent.TimeUnit
import scala.concurrent.duration._
import scala.concurrent.{Await, Promise}
import scala.util.{Failure, Success, Try}
import org.apache.commons.io.IOUtils
import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.shuffle.BlockFetchingListener
import org.apache.spark.network.{BlockDataManager, BlockTransferService}
import org.apache.spark.storage.{BlockId, ShuffleBlockId}
import org.apache.spark.{SecurityManager, SparkConf}
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, FunSuite, ShouldMatchers}
class NettyBlockTransferSecuritySuite extends FunSuite with MockitoSugar with ShouldMatchers {
test("security default off") {
val conf = new SparkConf()
.set("spark.app.id", "app-id")
testConnection(conf, conf) match {
case Success(_) => // expected
case Failure(t) => fail(t)
}
}
test("security on same password") {
val conf = new SparkConf()
.set("spark.authenticate", "true")
.set("spark.authenticate.secret", "good")
.set("spark.app.id", "app-id")
testConnection(conf, conf) match {
case Success(_) => // expected
case Failure(t) => fail(t)
}
}
test("security on mismatch password") {
val conf0 = new SparkConf()
.set("spark.authenticate", "true")
.set("spark.authenticate.secret", "good")
.set("spark.app.id", "app-id")
val conf1 = conf0.clone.set("spark.authenticate.secret", "bad")
testConnection(conf0, conf1) match {
case Success(_) => fail("Should have failed")
case Failure(t) => t.getMessage should include ("Mismatched response")
}
}
test("security mismatch auth off on server") {
val conf0 = new SparkConf()
.set("spark.authenticate", "true")
.set("spark.authenticate.secret", "good")
.set("spark.app.id", "app-id")
val conf1 = conf0.clone.set("spark.authenticate", "false")
testConnection(conf0, conf1) match {
case Success(_) => fail("Should have failed")
case Failure(t) => // any funny error may occur, sever will interpret SASL token as RPC
}
}
test("security mismatch auth off on client") {
val conf0 = new SparkConf()
.set("spark.authenticate", "false")
.set("spark.authenticate.secret", "good")
.set("spark.app.id", "app-id")
val conf1 = conf0.clone.set("spark.authenticate", "true")
testConnection(conf0, conf1) match {
case Success(_) => fail("Should have failed")
case Failure(t) => t.getMessage should include ("Expected SaslMessage")
}
}
private def fetchBlock(
self: BlockTransferService,
from: BlockTransferService,
execId: String,
blockId: BlockId): Try[ManagedBuffer] = {
val promise = Promise[ManagedBuffer]()
self.fetchBlocks(from.hostName, from.port, execId, Array(blockId.toString),
new BlockFetchingListener {
override def onBlockFetchFailure(blockId: String, exception: Throwable): Unit = {
promise.failure(exception)
}
override def onBlockFetchSuccess(blockId: String, data: ManagedBuffer): Unit = {
promise.success(data.retain())
}
})
Await.ready(promise.future, FiniteDuration(1000, TimeUnit.MILLISECONDS))
promise.future.value.get
}
}
Example 143
| Source File: CacheManagerSuite.scala From SparkCore with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import org.mockito.Mockito._
import org.scalatest.{BeforeAndAfter, FunSuite}
import org.scalatest.mock.MockitoSugar
import org.apache.spark.executor.DataReadMethod
import org.apache.spark.rdd.RDD
import org.apache.spark.storage._
// TODO: Test the CacheManager's thread-safety aspects
class CacheManagerSuite extends FunSuite with LocalSparkContext with BeforeAndAfter
with MockitoSugar {
var blockManager: BlockManager = _
var cacheManager: CacheManager = _
var split: Partition = _
var rdd: RDD[Int] = _
var rdd2: RDD[Int] = _
var rdd3: RDD[Int] = _
before {
sc = new SparkContext("local", "test")
blockManager = mock[BlockManager]
cacheManager = new CacheManager(blockManager)
split = new Partition { override def index: Int = 0 }
rdd = new RDD[Int](sc, Nil) {
override def getPartitions: Array[Partition] = Array(split)
override val getDependencies = List[Dependency[_]]()
override def compute(split: Partition, context: TaskContext) = Array(1, 2, 3, 4).iterator
}
rdd2 = new RDD[Int](sc, List(new OneToOneDependency(rdd))) {
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext) =
firstParent[Int].iterator(split, context)
}.cache()
rdd3 = new RDD[Int](sc, List(new OneToOneDependency(rdd2))) {
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext) =
firstParent[Int].iterator(split, context)
}.cache()
}
test("get uncached rdd") {
// Do not mock this test, because attempting to match Array[Any], which is not covariant,
// in blockManager.put is a losing battle. You have been warned.
blockManager = sc.env.blockManager
cacheManager = sc.env.cacheManager
val context = new TaskContextImpl(0, 0, 0, 0)
val computeValue = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
val getValue = blockManager.get(RDDBlockId(rdd.id, split.index))
assert(computeValue.toList === List(1, 2, 3, 4))
assert(getValue.isDefined, "Block cached from getOrCompute is not found!")
assert(getValue.get.data.toList === List(1, 2, 3, 4))
}
test("get cached rdd") {
val result = new BlockResult(Array(5, 6, 7).iterator, DataReadMethod.Memory, 12)
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(Some(result))
val context = new TaskContextImpl(0, 0, 0, 0)
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(5, 6, 7))
}
test("get uncached local rdd") {
// Local computation should not persist the resulting value, so don't expect a put().
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(None)
val context = new TaskContextImpl(0, 0, 0, 0, true)
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(1, 2, 3, 4))
}
test("verify task metrics updated correctly") {
cacheManager = sc.env.cacheManager
val context = new TaskContextImpl(0, 0, 0, 0)
cacheManager.getOrCompute(rdd3, split, context, StorageLevel.MEMORY_ONLY)
assert(context.taskMetrics.updatedBlocks.getOrElse(Seq()).size === 2)
}
}
Example 144
| Source File: HistoryServerSuite.scala From SparkCore with Apache License 2.0 | 5 votes |
|
package org.apache.spark.deploy.history
import javax.servlet.http.HttpServletRequest
import scala.collection.mutable
import org.apache.hadoop.fs.Path
import org.mockito.Mockito.{when}
import org.scalatest.FunSuite
import org.scalatest.Matchers
import org.scalatest.mock.MockitoSugar
import org.apache.spark.ui.SparkUI
class HistoryServerSuite extends FunSuite with Matchers with MockitoSugar {
test("generate history page with relative links") {
val historyServer = mock[HistoryServer]
val request = mock[HttpServletRequest]
val ui = mock[SparkUI]
val link = "/history/app1"
val info = new ApplicationHistoryInfo("app1", "app1", 0, 2, 1, "xxx", true)
when(historyServer.getApplicationList()).thenReturn(Seq(info))
when(ui.basePath).thenReturn(link)
when(historyServer.getProviderConfig()).thenReturn(Map[String, String]())
val page = new HistoryPage(historyServer)
//when
val response = page.render(request)
//then
val links = response \\ "a"
val justHrefs = for {
l <- links
attrs <- l.attribute("href")
} yield (attrs.toString)
justHrefs should contain(link)
}
}
Example 145
| Source File: AbstractSpec.scala From kafka-connect-cassandra with Apache License 2.0 | 5 votes |
|
package com.tuplejump.kafka.connect.cassandra
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfterAll, WordSpec, FlatSpec, Matchers}
trait ConfigFixture extends {
import InternalConfig._,Types._,TaskConfig._
final val EmptyProperties = Map.empty[String, String]
protected val multipleTopics: String = "test1,test2"
protected lazy val topicList = multipleTopics.split(TaskConfig.TopicSeparator).toList
protected lazy val sourceSchemas: List[Schema] =
topicList.zipWithIndex.map { case (t,i) =>
val route = Route(TaskConfig.SourceRoute + t, s"SELECT * FROM ks$i.table$i").get
Schema(route, Nil, Nil, Nil, List("a", "b"), "")
}
protected lazy val sinkSchemas: List[Schema] =
topicList.zipWithIndex.map { case (t,i) =>
val route = Route(TaskConfig.SinkRoute + t, s"ks$i.table$i").get
Schema(route, Nil, Nil, Nil, List("a", "b"), "")
}
protected lazy val sinkTopicMap = sinkSchemas.map(s => s.route.topic -> s.namespace).toMap
protected lazy val commonConfig = Map(
CassandraCluster.ConnectionHosts -> CassandraCluster.DefaultHosts,
CassandraCluster.ConnectionPort -> CassandraCluster.DefaultPort.toString,
CassandraCluster.ConnectionConsistency -> CassandraCluster.DefaultConsistency.name
)
protected def sinkProperties(config:Map[String,String] = sinkTopicMap): Map[String, String] =
config.map { case (topic,ns) => TaskConfig.SinkRoute + topic -> ns} ++ commonConfig
protected def sourceProperties(query: String, topic: String): Map[String,String] =
commonConfig ++ Map(TaskConfig.SourceRoute + topic -> query)
protected def sinkConfig(topic: TopicName,
keyspace: KeyspaceName,
table: TableName,
columnNames: List[ColumnName] = Nil): SinkConfig = {
import com.tuplejump.kafka.connect.cassandra.Syntax.PreparedQuery
val route = Route(TaskConfig.SinkRoute + topic, s"$keyspace.$table").get
val schema = Schema(route, Nil, Nil, Nil, columnNames, "")
SinkConfig(schema, PreparedQuery(schema), WriteOptions(DefaultSinkConsistency))
}
}
trait AbstractSpec extends WordSpec with Matchers with BeforeAndAfterAll with ConfigFixture
trait AbstractFlatSpec extends FlatSpec with Matchers with BeforeAndAfterAll
with ConfigFixture with MockitoSugar
Example 146
| Source File: SessionManagerSpec.scala From pizza-auth-3 with MIT License | 5 votes |
|
package moe.pizza.auth.webapp
import moe.pizza.auth.interfaces.UserDatabase
import org.http4s._
import org.http4s.dsl._
import org.http4s.util.CaseInsensitiveString
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, MustMatchers}
import moe.pizza.auth.webapp.Utils._
class SessionManagerSpec extends FlatSpec with MustMatchers with MockitoSugar {
val emptyservice = HttpService {
case req @ GET -> Root =>
Ok(req.getSession.toString)
case req @ GET -> Root / "flash" =>
val newsession = req.flash(Alerts.info, "this is an alert")
Ok(req.getSession.toString).attachSessionifDefined(newsession)
case req @ GET -> Root / "logout" =>
Ok(req.getSession.toString).clearSession()
}
val ud = mock[UserDatabase]
val svc = new SessionManager("keygoeshere", ud).apply(emptyservice)
"when wrapping a service it" should "add a session cookie" in {
val r = svc.apply(new Request(uri = Uri.uri("/"))).run
r.status must equal(Ok)
val session = r.headers.get(CaseInsensitiveString("set-cookie")).get
session.value.startsWith("authsession=") must equal(true)
val bodytxt = EntityDecoder.decodeString(r)(Charset.`UTF-8`).run
bodytxt must equal("Some(HydratedSession(List(),None,None,None))")
}
"when wrapping a service it" should "add a session cookie and use it to store state between calls" in {
val r = svc.apply(new Request(uri = Uri.uri("/flash"))).run
r.status must equal(Ok)
val session = r.headers.get(CaseInsensitiveString("set-cookie")).get
session.value.startsWith("authsession=") must equal(true)
val cookie = session.value
val r2 = svc
.apply(
new Request(uri = Uri.uri("/"),
headers = Headers(Header("Cookie", cookie))))
.run
r.status must equal(Ok)
val bodytxt = EntityDecoder.decodeString(r2)(Charset.`UTF-8`).run
bodytxt must equal(
"Some(HydratedSession(List(Alert(info,this is an alert)),None,None,None))")
}
"when wrapping a service it" should "be able to remove sessions" in {
val r = svc.apply(new Request(uri = Uri.uri("/logout"))).run
r.status must equal(Ok)
val removal = r.headers.get(CaseInsensitiveString("set-cookie")).get
assert(removal.value.startsWith("authsession=;"))
}
}
Example 147
| Source File: GraderChainSpec.scala From pizza-auth-3 with MIT License | 5 votes |
|
package moe.pizza.auth.adapters
import moe.pizza.auth.interfaces.PilotGrader
import moe.pizza.auth.models.Pilot
import moe.pizza.auth.models.Pilot.Status
import org.scalatest.{FlatSpec, MustMatchers}
import org.scalatest.mock.MockitoSugar
class GraderChainSpec extends FlatSpec with MustMatchers with MockitoSugar {
"GraderChain" should "chain graders and return the first result one gives" in {
val grader1 = new PilotGrader {
override def grade(p: Pilot): Status.Value = Status.banned
}
val grader2 = new PilotGrader {
override def grade(p: Pilot): Status.Value = Status.ineligible
}
val chain = List(grader1, grader2)
val graderchain = new GraderChain(chain)
val p = Pilot("bob",
Pilot.Status.internal,
"myalliance",
"mycorp",
"Bob",
"[email protected]",
Pilot.OM.readTree("{\"meta\": \"%s\"}".format("metafield")),
List("group1", "group3"),
List("123:bobkey"),
List.empty)
graderchain.grade(p) must equal(Status.banned)
}
"GraderChain" should "fall through all unclassified results" in {
val grader1 = new PilotGrader {
override def grade(p: Pilot): Status.Value = Status.unclassified
}
val grader2 = new PilotGrader {
override def grade(p: Pilot): Status.Value = Status.internal
}
val chain = List(grader1, grader2)
val graderchain = new GraderChain(chain)
val p = Pilot("bob",
Pilot.Status.internal,
"myalliance",
"mycorp",
"Bob",
"[email protected]",
Pilot.OM.readTree("{\"meta\": \"%s\"}".format("metafield")),
List("group1", "group3"),
List("123:bobkey"),
List.empty)
graderchain.grade(p) must equal(Status.internal)
}
}
Example 148
| Source File: EveMapDbSpec.scala From pizza-auth-3 with MIT License | 5 votes |
|
package moe.pizza.auth.graphdb
import org.scalatest.{MustMatchers, WordSpec}
import org.scalatest.mock.MockitoSugar
class EveMapDbSpec extends WordSpec with MustMatchers with MockitoSugar {
"EveMapDb" when {
"being used" should {
"do all of the normal expected things" in {
val e = new EveMapDb("map-tests1")
// initialise the database
e.provisionIfRequired()
e.withGraph { g =>
g.getEdgeType("gate") must not equal (null)
}
// fail gracefully on bad system names
e.getDistanceBetweenSystemsByName("amor", "jota") must equal(None)
// fail gracefully on bad system numbers
e.getDistanceBetweenSystemsById(1, -42) must equal(None)
// correctly find the distance between named systems
e.getDistanceBetweenSystemsByName("Amarr", "Jita") must equal(Some(10))
// correctly find the distance between system ids
e.getDistanceBetweenSystemsById(30000142, 30004711) must equal(
Some(40))
// describe the distance between the same system and itself as 0
e.getDistanceBetweenSystemsById(30000142, 30000142) must equal(Some(0))
e.getDistanceBetweenSystemsByName("Amarr", "Amarr") must equal(Some(0))
e.cleanUp()
}
}
}
}
Example 149
| Source File: InternalWhitelistPilotGraderSpec.scala From pizza-auth-3 with MIT License | 5 votes |
|
package moe.pizza.auth.plugins.pilotgraders
import moe.pizza.auth.models.Pilot
import moe.pizza.auth.plugins.pilotgraders.MembershipPilotGraders.{AlliancePilotGrader, CorporationPilotGrader, PublicAccessPilotGrader}
import moe.pizza.crestapi.CrestApi
import moe.pizza.crestapi.CrestApi.{CallbackResponse, VerifyResponse}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{MustMatchers, WordSpec}
import org.mockito.Mockito._
import org.http4s.client.blaze.PooledHttp1Client
import scalaz.concurrent.Task
class InternalWhitelistPilotGraderSpec
extends WordSpec
with MustMatchers
with MockitoSugar {
implicit val client = PooledHttp1Client() //TODO: no mocking?
"InternalWhitelistPilotGrader" when {
"grading" should {
"grade pilots who are in the list as internal" in {
val crest = mock[CrestApi]
when(crest.refresh("REF")).thenReturn(Task {
new CallbackResponse("access", "type", 1000, Some("refresh"))
})
when(crest.verify("access")).thenReturn(Task {
new VerifyResponse(1, "Bob", "", "", "", "", "")
})
val iwpg = new InternalWhitelistPilotGrader(crest, List(1L, 2L, 3L))
val p = new Pilot("bob",
Pilot.Status.unclassified,
"boballiance",
"bobcorp",
"Bob",
"none@none",
Pilot.OM.createObjectNode(),
List.empty[String],
List("1:REF"),
List.empty[String])
iwpg.grade(p) must equal(Pilot.Status.internal)
verify(crest).refresh("REF")
verify(crest).verify("access")
}
"grade pilots who are not in the list as unclassified" in {
val crest = mock[CrestApi]
when(crest.refresh("REF")).thenReturn(Task {
new CallbackResponse("access", "type", 1000, Some("refresh"))
})
when(crest.verify("access")).thenReturn(Task {
new VerifyResponse(1, "Bob", "", "", "", "", "")
})
val iwpg = new InternalWhitelistPilotGrader(crest, List(2L, 3L))
val p = new Pilot("bob",
Pilot.Status.unclassified,
"boballiance",
"bobcorp",
"Bob",
"none@none",
Pilot.OM.createObjectNode(),
List.empty[String],
List("1:REF"),
List.empty[String])
iwpg.grade(p) must equal(Pilot.Status.unclassified)
verify(crest).refresh("REF")
verify(crest).verify("access")
}
}
}
}
Example 150
| Source File: CaldariCrestKeyGraderSpec.scala From pizza-auth-3 with MIT License | 5 votes |
|
package moe.pizza.auth.plugins.pilotgraders
import moe.pizza.auth.models.Pilot
import moe.pizza.crestapi.CrestApi
import moe.pizza.crestapi.CrestApi.{CallbackResponse, VerifyResponse}
import moe.pizza.eveapi.{EVEAPI, XMLApiResponse}
import moe.pizza.eveapi.endpoints.Character
import org.http4s.client.blaze.PooledHttp1Client
import org.joda.time.DateTime
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{MustMatchers, WordSpec}
import scalaz.concurrent.Task
class CaldariCrestKeyGraderSpec
extends WordSpec
with MustMatchers
with MockitoSugar {
implicit val client = PooledHttp1Client() //TODO: no mocking?
"CaldariCrestKeyGrader" when {
"grading" should {
"grade pilots who are caldari as THE ENEMY" in {
val crest = mock[CrestApi]
val eveapi = mock[EVEAPI]
val char = mock[Character]
when(eveapi.char).thenReturn(char)
when(crest.refresh("REF")).thenReturn(Task {
new CallbackResponse("access", "type", 1000, Some("refresh"))
})
when(crest.verify("access")).thenReturn(Task {
new VerifyResponse(1, "Bob", "", "", "", "", "")
})
val pilotInfo =
mock[moe.pizza.eveapi.generated.char.CharacterInfo.Result]
when(pilotInfo.race).thenReturn("Caldari")
when(char.CharacterInfo(1)).thenReturn(Task {
new XMLApiResponse(DateTime.now(), DateTime.now(), pilotInfo)
})
val iwpg = new CaldariCrestKeyGrader(crest, eve = Some(eveapi))
val p = new Pilot("bob",
Pilot.Status.unclassified,
"boballiance",
"bobcorp",
"Bob",
"none@none",
Pilot.OM.createObjectNode(),
List.empty[String],
List("1:REF"),
List.empty[String])
iwpg.grade(p) must equal(Pilot.Status.banned)
verify(crest).refresh("REF")
verify(crest).verify("access")
verify(char).CharacterInfo(1)
verify(pilotInfo).race
}
"grade pilots who are not caldari as not THE ENEMY" in {
val crest = mock[CrestApi]
val eveapi = mock[EVEAPI]
val char = mock[Character]
when(eveapi.char).thenReturn(char)
when(crest.refresh("REF")).thenReturn(Task {
new CallbackResponse("access", "type", 1000, Some("refresh"))
})
when(crest.verify("access")).thenReturn(Task {
new VerifyResponse(1, "Bob", "", "", "", "", "")
})
val pilotInfo =
mock[moe.pizza.eveapi.generated.char.CharacterInfo.Result]
when(pilotInfo.race).thenReturn("Gallente")
when(char.CharacterInfo(1)).thenReturn(Task {
new XMLApiResponse(DateTime.now(), DateTime.now(), pilotInfo)
})
val iwpg = new CaldariCrestKeyGrader(crest, eve = Some(eveapi))
val p = new Pilot("bob",
Pilot.Status.unclassified,
"boballiance",
"bobcorp",
"Bob",
"none@none",
Pilot.OM.createObjectNode(),
List.empty[String],
List("1:REF"),
List.empty[String])
iwpg.grade(p) must equal(Pilot.Status.unclassified)
verify(crest).refresh("REF")
verify(crest).verify("access")
verify(char).CharacterInfo(1)
verify(pilotInfo).race
}
}
}
}
Example 151
| Source File: KafkaMetricFactoryTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.metrics
import com.aol.one.dwh.bandarlog.connectors.KafkaConnector
import com.aol.one.dwh.bandarlog.metrics.BaseMetrics._
import com.aol.one.dwh.bandarlog.metrics.KafkaMetricFactoryTest._
import com.aol.one.dwh.bandarlog.providers.{KafkaInMessagesProvider, KafkaLagProvider, KafkaOutMessagesProvider}
import com.aol.one.dwh.infra.config.{Tag, Topic}
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
object KafkaMetricFactoryTest {
private val metricPrefix = "kafka_prefix"
private val topic = Topic("some_topic_id", Set("topic-1", "topic-2"), "some_group_id")
private val expectedTags = List(Tag("topic", topic.id), Tag("group-id", topic.groupId))
}
class KafkaMetricFactoryTest extends FunSuite with MockitoSugar {
private val kafkaConnector = mock[KafkaConnector]
private val kafkaMetricFactory = new KafkaMetricFactory(kafkaConnector)
test("create kafka Metric & Provider for IN metric id") {
val result = kafkaMetricFactory.create(IN, metricPrefix, topic)
assertMetric(result.metric, "in_messages")
assert(result.provider.isInstanceOf[KafkaInMessagesProvider])
}
test("create kafka Metric & Provider for OUT metric id") {
val result = kafkaMetricFactory.create(OUT, metricPrefix, topic)
assertMetric(result.metric, "out_messages")
assert(result.provider.isInstanceOf[KafkaOutMessagesProvider])
}
test("create kafka Metric & Provider for LAG metric id") {
val result = kafkaMetricFactory.create(LAG, metricPrefix, topic)
assertMetric(result.metric, "lag")
assert(result.provider.isInstanceOf[KafkaLagProvider])
}
test("throw exception in unknown metric case") {
intercept[IllegalArgumentException] {
kafkaMetricFactory.create("UNKNOWN_METRIC", metricPrefix, topic)
}
}
private def assertMetric[V](metric: Metric[V], expectedName: String) = {
assert(metric.prefix == metricPrefix)
assert(metric.name == expectedName)
assert(metric.tags == expectedTags)
assert(metric.value == AtomicValue(None))
}
}
Example 152
| Source File: SqlTimestampProviderTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.providers
import com.aol.one.dwh.bandarlog.connectors.JdbcConnector
import com.aol.one.dwh.infra.config.Table
import com.aol.one.dwh.infra.sql.{Query, VerticaMaxValuesQuery}
import org.mockito.Matchers.any
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class SqlTimestampProviderTest extends FunSuite with MockitoSugar {
private val query = VerticaMaxValuesQuery(Table("table", List("column"), None))
private val jdbcConnector = mock[JdbcConnector]
private val sqlTimestampProvider = new SqlTimestampProvider(jdbcConnector, query)
test("check timestamp value by connector and query") {
val resultTimestamp = Some(1234567890L)
when(jdbcConnector.runQuery(any(classOf[Query]), any())).thenReturn(resultTimestamp)
val result = sqlTimestampProvider.provide()
assert(result.getValue == resultTimestamp)
}
test("return none if can't get timestamp value") {
when(jdbcConnector.runQuery(any(classOf[Query]), any())).thenReturn(None)
val result = sqlTimestampProvider.provide()
assert(result.getValue.isEmpty)
}
}
Example 153
| Source File: GlueTimestampProviderTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.providers
import com.aol.one.dwh.bandarlog.connectors.GlueConnector
import com.aol.one.dwh.infra.config.Table
import org.mockito.Matchers.any
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class GlueTimestampProviderTest extends FunSuite with MockitoSugar{
private val table = mock[Table]
private val glueConnector = mock[GlueConnector]
private val glueTimestampProvider = new GlueTimestampProvider(glueConnector, table)
test("check timestamp value by glue connector and table") {
val glueTimestamp = 1533709910004L
when(glueConnector.getMaxPartitionValue(any())).thenReturn(glueTimestamp)
val result = glueTimestampProvider.provide()
assert(result.getValue == Some(glueTimestamp))
}
test("return zero if partition column does not have values") {
when(glueConnector.getMaxPartitionValue(any())).thenReturn(0)
val result = glueTimestampProvider.provide()
assert(result.getValue == Some(0))
}
}
Example 154
| Source File: SqlLagProviderTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.providers
import com.aol.one.dwh.bandarlog.metrics.AtomicValue
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class SqlLagProviderTest extends FunSuite with MockitoSugar {
private val fromProvider = mock[SqlTimestampProvider]
private val toProvider = mock[SqlTimestampProvider]
private val toGlueProvider = mock[GlueTimestampProvider]
private val lagProvider1 = new SqlLagProvider(fromProvider, toProvider)
private val lagProvider2 = new SqlLagProvider(fromProvider, toGlueProvider)
test("check lag between from and to providers") {
val fromValue = AtomicValue(Some(7L))
val toValue = AtomicValue(Some(4L))
val toGlueValue = AtomicValue(Some(6L))
when(fromProvider.provide()).thenReturn(fromValue)
when(toProvider.provide()).thenReturn(toValue)
when(toGlueProvider.provide()).thenReturn(toGlueValue)
val lag1 = lagProvider1.provide()
val lag2 = lagProvider2.provide()
assert(lag1.getValue.nonEmpty)
assert(lag1.getValue.get == 3)
assert(lag2.getValue.nonEmpty)
assert(lag2.getValue.get == 1)
}
test("return none if 'from provider' value is none") {
val toValue = AtomicValue(Some(4L))
when(fromProvider.provide()).thenReturn(AtomicValue[Long](None))
when(toProvider.provide()).thenReturn(toValue)
val lag = lagProvider1.provide()
assert(lag.getValue.isEmpty)
}
test("return none if 'to provider' value is none") {
val fromValue = AtomicValue(Some(7L))
when(fromProvider.provide()).thenReturn(fromValue)
when(toProvider.provide()).thenReturn(AtomicValue[Long](None))
when(toGlueProvider.provide()).thenReturn(AtomicValue[Long](None))
val lag1 = lagProvider1.provide()
val lag2 = lagProvider2.provide()
assert(lag1.getValue.isEmpty)
assert(lag2.getValue.isEmpty)
}
test("return none if both providers values is none") {
when(fromProvider.provide()).thenReturn(AtomicValue[Long](None))
when(toProvider.provide()).thenReturn(AtomicValue[Long](None))
when(toGlueProvider.provide()).thenReturn(AtomicValue[Long](None))
val lag1 = lagProvider1.provide()
val lag2 = lagProvider2.provide()
assert(lag1.getValue.isEmpty)
assert(lag2.getValue.isEmpty)
}
}
Example 155
| Source File: KafkaOutMessagesProviderTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.providers
import com.aol.one.dwh.bandarlog.connectors.KafkaConnector
import com.aol.one.dwh.infra.config.Topic
import kafka.common.TopicAndPartition
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class KafkaOutMessagesProviderTest extends FunSuite with MockitoSugar {
private val kafkaConnector = mock[KafkaConnector]
private val topic = Topic("topic_id", Set("topic_1", "topic_2"), "group_id")
test("check count of out messages/offsets over all topic partitions") {
val offsets = Option(Map(
TopicAndPartition("topic_1", 1) -> 1L,
TopicAndPartition("topic_2", 2) -> 2L,
TopicAndPartition("topic_3", 3) -> 3L
))
when(kafkaConnector.getOffsets(topic)).thenReturn(offsets)
val result = new KafkaOutMessagesProvider(kafkaConnector, topic).provide()
assert(result.getValue.nonEmpty)
assert(result.getValue.get == 6) // 1 + 2 + 3
}
test("check count of out messages/offsets for empty offsets result") {
when(kafkaConnector.getOffsets(topic)).thenReturn(Some(Map[TopicAndPartition, Long]()))
val result = new KafkaOutMessagesProvider(kafkaConnector, topic).provide()
assert(result.getValue.nonEmpty)
assert(result.getValue.get == 0)
}
test("return none if can't retrieve offsets") {
when(kafkaConnector.getOffsets(topic)).thenReturn(None)
val result = new KafkaOutMessagesProvider(kafkaConnector, topic).provide()
assert(result.getValue.isEmpty)
}
}
Example 156
| Source File: KafkaLagProviderTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.providers
import com.aol.one.dwh.infra.config.Topic
import com.aol.one.dwh.bandarlog.connectors.KafkaConnector
import kafka.common.TopicAndPartition
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class KafkaLagProviderTest extends FunSuite with MockitoSugar {
private val kafkaConnector = mock[KafkaConnector]
private val topic = Topic("topic_id", Set("topic_1", "topic_2", "topic_3"), "group_id")
test("check lag per topic") {
val heads = Map(
TopicAndPartition("topic_1", 1) -> 4L,
TopicAndPartition("topic_2", 2) -> 5L,
TopicAndPartition("topic_3", 3) -> 6L
)
val offsets = Map(
TopicAndPartition("topic_1", 1) -> 1L,
TopicAndPartition("topic_2", 2) -> 2L,
TopicAndPartition("topic_3", 3) -> 3L
)
val kafkaState = Option((heads, offsets))
when(kafkaConnector.getKafkaState(topic)).thenReturn(kafkaState)
val result = new KafkaLagProvider(kafkaConnector, topic).provide()
// topic partition heads offsets lag
// topic_1 1 4 1 4-1=3
// topic_2 2 5 2 5-2=3
// topic_3 3 6 3 6-3=3
assert(result.getValue.nonEmpty)
assert(result.getValue.get == 9) // lag sum 3 + 3 + 3
}
test("check 0 lag case per topic") {
val heads = Map(
TopicAndPartition("topic_1", 1) -> 1L,
TopicAndPartition("topic_2", 2) -> 2L,
TopicAndPartition("topic_3", 3) -> 3L
)
val offsets = Map(
TopicAndPartition("topic_1", 1) -> 4L,
TopicAndPartition("topic_2", 2) -> 5L,
TopicAndPartition("topic_3", 3) -> 6L
)
val kafkaState = Option((heads, offsets))
when(kafkaConnector.getKafkaState(topic)).thenReturn(kafkaState)
val result = new KafkaLagProvider(kafkaConnector, topic).provide()
// topic partition heads offsets lag
// topic_1 1 1 4 1-4= -3
// topic_2 2 2 5 2-5= -3
// topic_3 3 3 6 3-6= -3
assert(result.getValue.nonEmpty)
assert(result.getValue.get == 0) // lag.max(0) = 0
}
test("check lag for empty heads and offsets") {
val kafkaState = Option((Map[TopicAndPartition, Long](), Map[TopicAndPartition, Long]()))
when(kafkaConnector.getKafkaState(topic)).thenReturn(kafkaState)
val result = new KafkaLagProvider(kafkaConnector, topic).provide()
assert(result.getValue.nonEmpty)
assert(result.getValue.get == 0)
}
test("return none if can't retrieve kafka state") {
when(kafkaConnector.getKafkaState(topic)).thenReturn(None)
val result = new KafkaLagProvider(kafkaConnector, topic).provide()
assert(result.getValue.isEmpty)
}
}
Example 157
| Source File: KafkaInMessagesProviderTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.providers
import com.aol.one.dwh.bandarlog.connectors.KafkaConnector
import com.aol.one.dwh.infra.config.Topic
import kafka.common.TopicAndPartition
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class KafkaInMessagesProviderTest extends FunSuite with MockitoSugar {
private val kafkaConnector = mock[KafkaConnector]
private val topic = Topic("topic_id", Set("topic_1", "topic_2"), "group_id")
test("check count of in messages/heads over all topic partitions") {
val heads = Some(Map(
TopicAndPartition("topic_1", 1) -> 1L,
TopicAndPartition("topic_2", 2) -> 2L,
TopicAndPartition("topic_3", 3) -> 3L
))
when(kafkaConnector.getHeads(topic)).thenReturn(heads)
val result = new KafkaInMessagesProvider(kafkaConnector, topic).provide()
assert(result.getValue.nonEmpty)
assert(result.getValue.get == 6) // 1 + 2 + 3
}
test("check count of in messages/heads for empty heads result") {
when(kafkaConnector.getHeads(topic)).thenReturn(Some(Map[TopicAndPartition, Long]()))
val result = new KafkaInMessagesProvider(kafkaConnector, topic).provide()
assert(result.getValue.nonEmpty)
assert(result.getValue.get == 0)
}
test("return none if can't retrieve heads") {
when(kafkaConnector.getHeads(topic)).thenReturn(None)
val result = new KafkaInMessagesProvider(kafkaConnector, topic).provide()
assert(result.getValue.isEmpty)
}
}
Example 158
| Source File: GlueConnectorTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.connectors
import com.aol.one.dwh.infra.config._
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
import scala.concurrent.duration._
class GlueConnectorTest extends FunSuite with MockitoSugar {
private val config = GlueConfig("eu-central-1", "default", "accessKey", "secretKey", 5, 2, 10.seconds)
private val glueConnector = mock[GlueConnector]
test("Check max batchId from glue metadata tables") {
val resultValue = 100L
val numericTable = Table("table", List("column"), None)
when(glueConnector.getMaxPartitionValue(numericTable)).thenReturn(resultValue)
val result = glueConnector.getMaxPartitionValue(numericTable)
assert(result == resultValue)
}
test("Check max date partitions' value from glue metadata table") {
val resultValue = 15681377656L
val datetimeTable = Table("table", List("year", "month", "day"), Some(List("yyyy", "MM", "dd")))
when(glueConnector.getMaxPartitionValue(datetimeTable)).thenReturn(resultValue)
val result = glueConnector.getMaxPartitionValue(datetimeTable)
assert(result == resultValue)
}
}
Example 159
| Source File: JdbcConnectorTest.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.bandarlog.connectors
import java.sql.{Connection, DatabaseMetaData, ResultSet, Statement}
import com.aol.one.dwh.infra.config._
import com.aol.one.dwh.infra.sql.pool.HikariConnectionPool
import com.aol.one.dwh.infra.sql.{ListStringResultHandler, Setting, VerticaMaxValuesQuery}
import org.apache.commons.dbutils.ResultSetHandler
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar
class JdbcConnectorTest extends FunSuite with MockitoSugar {
private val statement = mock[Statement]
private val resultSet = mock[ResultSet]
private val connectionPool = mock[HikariConnectionPool]
private val connection = mock[Connection]
private val databaseMetaData = mock[DatabaseMetaData]
private val resultSetHandler = mock[ResultSetHandler[Long]]
private val listStringResultHandler = mock[ListStringResultHandler]
test("check run query result for numeric batch_id column") {
val resultValue = 100L
val table = Table("table", List("column"), None)
val query = VerticaMaxValuesQuery(table)
when(connectionPool.getConnection).thenReturn(connection)
when(connectionPool.getName).thenReturn("connection_pool_name")
when(connection.createStatement()).thenReturn(statement)
when(statement.executeQuery("SELECT MAX(column) AS column FROM table")).thenReturn(resultSet)
when(connection.getMetaData).thenReturn(databaseMetaData)
when(databaseMetaData.getURL).thenReturn("connection_url")
when(resultSetHandler.handle(resultSet)).thenReturn(resultValue)
val result = new DefaultJdbcConnector(connectionPool).runQuery(query, resultSetHandler)
assert(result == resultValue)
}
test("check run query result for date/time partitions") {
val resultValue = Some(20190924L)
val table = Table("table", List("year", "month", "day"), Some(List("yyyy", "MM", "dd")))
val query = VerticaMaxValuesQuery(table)
when(connectionPool.getConnection).thenReturn(connection)
when(connectionPool.getName).thenReturn("connection_pool_name")
when(connection.createStatement()).thenReturn(statement)
when(statement.executeQuery("SELECT DISTINCT year, month, day FROM table")).thenReturn(resultSet)
when(connection.getMetaData).thenReturn(databaseMetaData)
when(databaseMetaData.getURL).thenReturn("connection_url")
when(listStringResultHandler.handle(resultSet)).thenReturn(resultValue)
val result = new DefaultJdbcConnector(connectionPool).runQuery(query, listStringResultHandler)
assert(result == resultValue)
}
}
class DefaultJdbcConnector(connectionPool: HikariConnectionPool) extends JdbcConnector(connectionPool) {
override def applySetting(connection: Connection, statement: Statement, setting: Setting): Unit = {}
}
Example 160
| Source File: BulkIndexerActorTest.scala From elasticsearch-client with Apache License 2.0 | 5 votes |
|
package com.sumologic.elasticsearch.akkahelpers
import java.util.concurrent.TimeUnit
import akka.actor.{ActorSystem, Terminated}
import akka.testkit.{ImplicitSender, TestActorRef, TestKit}
import com.sumologic.elasticsearch.akkahelpers.BulkIndexerActor.{BulkSession, CreateRequest, DocumentIndexed, ForceFlush}
import com.sumologic.elasticsearch.restlastic.{RestlasticSearchClient, RestlasticSearchClient6}
import com.sumologic.elasticsearch.restlastic.RestlasticSearchClient.ReturnTypes.BulkItem
import com.sumologic.elasticsearch.restlastic.dsl.Dsl._
import org.junit.runner.RunWith
import org.mockito.ArgumentMatchers._
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest._
import org.scalatest.concurrent.Eventually
import org.scalatest.mock.MockitoSugar
import org.scalatestplus.junit.JUnitRunner
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.concurrent.duration.Duration
@RunWith(classOf[JUnitRunner])
class BulkIndexerActorTest extends TestKit(ActorSystem("TestSystem")) with WordSpecLike with Matchers
with BeforeAndAfterAll with BeforeAndAfterEach with MockitoSugar with ImplicitSender with Eventually {
val executionContext = scala.concurrent.ExecutionContext.Implicits.global
var indexerActor: TestActorRef[BulkIndexerActor] = _
var mockEs = mock[RestlasticSearchClient]
var flushTimeoutMs = 100000L
var maxMessages = 100000
override def beforeEach(): Unit = {
mockEs = mock[RestlasticSearchClient]
when(mockEs.indexExecutionCtx).thenReturn(executionContext)
def timeout() = Duration(flushTimeoutMs, TimeUnit.MILLISECONDS)
def max() = maxMessages
val config = BulkConfig(timeout, max)
indexerActor = TestActorRef[BulkIndexerActor](BulkIndexerActor.props(mockEs, config))
}
override def afterAll(): Unit = {
val terminationFuture: Future[Terminated] = system.terminate()
Await.result(terminationFuture, 5.seconds)
}
"BulkIndexerActor" should {
"flush every message when set to 1" in {
maxMessages = 1
when(mockEs.bulkIndex(any())).thenReturn(Future.successful(Seq(BulkItem("index","type", "_id", 201, None))))
val sess = BulkSession.create()
indexerActor ! CreateRequest(sess, Index("i"), Type("tpe"), Document("id", Map("k" -> "v")))
eventually {
mockEs.bulkIndex(any())
}
val msg = expectMsgType[DocumentIndexed]
msg.sessionId should be(sess)
}
"not flush when set to 2" in {
maxMessages = 2
indexerActor ! CreateRequest(BulkSession.create(), Index("i"), Type("tpe"), Document("id", Map("k" -> "v")))
verify(mockEs, times(0)).bulkIndex(any())
}
"not flush when there are no messages" in {
indexerActor ! ForceFlush
verify(mockEs, times(0)).bulkIndex(any())
}
}
}
Example 161
| Source File: PartiallyUnrolledIteratorSuite.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.storage
import org.mockito.Matchers
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.apache.spark.SparkFunSuite
import org.apache.spark.memory.MemoryMode.ON_HEAP
import org.apache.spark.storage.memory.{MemoryStore, PartiallyUnrolledIterator}
class PartiallyUnrolledIteratorSuite extends SparkFunSuite with MockitoSugar {
test("join two iterators") {
val unrollSize = 1000
val unroll = (0 until unrollSize).iterator
val restSize = 500
val rest = (unrollSize until restSize + unrollSize).iterator
val memoryStore = mock[MemoryStore]
val joinIterator = new PartiallyUnrolledIterator(memoryStore, ON_HEAP, unrollSize, unroll, rest)
// Firstly iterate over unrolling memory iterator
(0 until unrollSize).foreach { value =>
assert(joinIterator.hasNext)
assert(joinIterator.hasNext)
assert(joinIterator.next() == value)
}
joinIterator.hasNext
joinIterator.hasNext
verify(memoryStore, times(1))
.releaseUnrollMemoryForThisTask(Matchers.eq(ON_HEAP), Matchers.eq(unrollSize.toLong))
// Secondly, iterate over rest iterator
(unrollSize until unrollSize + restSize).foreach { value =>
assert(joinIterator.hasNext)
assert(joinIterator.hasNext)
assert(joinIterator.next() == value)
}
joinIterator.close()
// MemoryMode.releaseUnrollMemoryForThisTask is called only once
verifyNoMoreInteractions(memoryStore)
}
}
Example 162
| Source File: CacheManagerSuite.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import org.mockito.Mockito._
import org.scalatest.BeforeAndAfter
import org.scalatest.mock.MockitoSugar
import org.apache.spark.executor.DataReadMethod
import org.apache.spark.rdd.RDD
import org.apache.spark.storage._
// TODO: Test the CacheManager's thread-safety aspects
class CacheManagerSuite extends SparkFunSuite with LocalSparkContext with BeforeAndAfter
with MockitoSugar {
var blockManager: BlockManager = _
var cacheManager: CacheManager = _
var split: Partition = _
var rdd: RDD[Int] = _
var rdd2: RDD[Int] = _
var rdd3: RDD[Int] = _
before {
sc = new SparkContext("local", "test")
blockManager = mock[BlockManager]
cacheManager = new CacheManager(blockManager)
split = new Partition { override def index: Int = 0 }
rdd = new RDD[Int](sc, Nil) {
override def getPartitions: Array[Partition] = Array(split)
override val getDependencies = List[Dependency[_]]()
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
Array(1, 2, 3, 4).iterator
}
rdd2 = new RDD[Int](sc, List(new OneToOneDependency(rdd))) {
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
firstParent[Int].iterator(split, context)
}.cache()
rdd3 = new RDD[Int](sc, List(new OneToOneDependency(rdd2))) {
override def getPartitions: Array[Partition] = firstParent[Int].partitions
override def compute(split: Partition, context: TaskContext): Iterator[Int] =
firstParent[Int].iterator(split, context)
}.cache()
}
test("get uncached rdd") {
// Do not mock this test, because attempting to match Array[Any], which is not covariant,
// in blockManager.put is a losing battle. You have been warned.
blockManager = sc.env.blockManager
cacheManager = sc.env.cacheManager
val context = new TaskContextImpl(0, 0, 0, 0, null)
val computeValue = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
val getValue = blockManager.get(RDDBlockId(rdd.id, split.index))
assert(computeValue.toList === List(1, 2, 3, 4))
assert(getValue.isDefined, "Block cached from getOrCompute is not found!")
assert(getValue.get.data.toList === List(1, 2, 3, 4))
}
test("get cached rdd") {
val result = new BlockResult(Array(5, 6, 7).iterator, DataReadMethod.Memory, 12)
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(Some(result))
val context = new TaskContextImpl(0, 0, 0, 0, null)
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(5, 6, 7))
}
test("get uncached local rdd") {
// Local computation should not persist the resulting value, so don't expect a put().
when(blockManager.get(RDDBlockId(0, 0))).thenReturn(None)
val context = new TaskContextImpl(0, 0, 0, 0, null, true)
val value = cacheManager.getOrCompute(rdd, split, context, StorageLevel.MEMORY_ONLY)
assert(value.toList === List(1, 2, 3, 4))
}
test("verify task metrics updated correctly") {
cacheManager = sc.env.cacheManager
val context = new TaskContextImpl(0, 0, 0, 0, null)
cacheManager.getOrCompute(rdd3, split, context, StorageLevel.MEMORY_ONLY)
assert(context.taskMetrics.updatedBlocks.getOrElse(Seq()).size === 2)
}
}
Example 163
| Source File: MesosClusterSchedulerSuite.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.scheduler.mesos
import java.util.Date
import org.scalatest.mock.MockitoSugar
import org.apache.spark.deploy.Command
import org.apache.spark.deploy.mesos.MesosDriverDescription
import org.apache.spark.scheduler.cluster.mesos._
import org.apache.spark.{LocalSparkContext, SparkConf, SparkFunSuite}
class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext with MockitoSugar {
private val command = new Command("mainClass", Seq("arg"), null, null, null, null)
test("can queue drivers") {
val conf = new SparkConf()
conf.setMaster("mesos://localhost:5050")
conf.setAppName("spark mesos")
val scheduler = new MesosClusterScheduler(
new BlackHoleMesosClusterPersistenceEngineFactory, conf) {
override def start(): Unit = { ready = true }
}
scheduler.start()
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", 1000, 1, true,
command, Map[String, String](), "s1", new Date()))
assert(response.success)
val response2 =
scheduler.submitDriver(new MesosDriverDescription(
"d1", "jar", 1000, 1, true, command, Map[String, String](), "s2", new Date()))
assert(response2.success)
val state = scheduler.getSchedulerState()
val queuedDrivers = state.queuedDrivers.toList
assert(queuedDrivers(0).submissionId == response.submissionId)
assert(queuedDrivers(1).submissionId == response2.submissionId)
}
test("can kill queued drivers") {
val conf = new SparkConf()
conf.setMaster("mesos://localhost:5050")
conf.setAppName("spark mesos")
val scheduler = new MesosClusterScheduler(
new BlackHoleMesosClusterPersistenceEngineFactory, conf) {
override def start(): Unit = { ready = true }
}
scheduler.start()
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", 1000, 1, true,
command, Map[String, String](), "s1", new Date()))
assert(response.success)
val killResponse = scheduler.killDriver(response.submissionId)
assert(killResponse.success)
val state = scheduler.getSchedulerState()
assert(state.queuedDrivers.isEmpty)
}
}
Example 164
| Source File: MemoryUtilsSuite.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.scheduler.cluster.mesos
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
class MemoryUtilsSuite extends SparkFunSuite with MockitoSugar {
test("MesosMemoryUtils should always override memoryOverhead when it's set") {
val sparkConf = new SparkConf
val sc = mock[SparkContext]
when(sc.conf).thenReturn(sparkConf)
// 384 > sc.executorMemory * 0.1 => 512 + 384 = 896
when(sc.executorMemory).thenReturn(512)
assert(MemoryUtils.calculateTotalMemory(sc) === 896)
// 384 < sc.executorMemory * 0.1 => 4096 + (4096 * 0.1) = 4505.6
when(sc.executorMemory).thenReturn(4096)
assert(MemoryUtils.calculateTotalMemory(sc) === 4505)
// set memoryOverhead
sparkConf.set("spark.mesos.executor.memoryOverhead", "100")
assert(MemoryUtils.calculateTotalMemory(sc) === 4196)
sparkConf.set("spark.mesos.executor.memoryOverhead", "400")
assert(MemoryUtils.calculateTotalMemory(sc) === 4496)
}
}
Example 165
| Source File: LoadBalanceServiceSuite.scala From kyuubi with Apache License 2.0 | 5 votes |
|
package yaooqinn.kyuubi.ha
import java.io.IOException
import org.apache.spark.{KyuubiSparkUtil, SparkFunSuite}
import org.scalatest.{BeforeAndAfterEach, Matchers}
import org.scalatest.mock.MockitoSugar
import yaooqinn.kyuubi.SecuredFunSuite
import yaooqinn.kyuubi.server.KyuubiServer
import yaooqinn.kyuubi.service.ServiceException
import yaooqinn.kyuubi.service.State._
class LoadBalanceServiceSuite extends SparkFunSuite
with ZookeeperFunSuite
with Matchers
with SecuredFunSuite
with MockitoSugar
with BeforeAndAfterEach {
private var server: KyuubiServer = _
private var haService: HighAvailableService = _
override def beforeEach(): Unit = {
server = new KyuubiServer()
haService = new LoadBalanceService(server)
super.beforeEach()
}
override def afterEach(): Unit = {
if (server != null) {
server.stop()
}
if (haService != null) {
haService.stop()
}
super.afterEach()
}
test("Init") {
haService.getConf should be(null)
haService.getStartTime should be(0)
haService.getName should be(classOf[LoadBalanceService].getSimpleName)
haService.getServiceState should be(NOT_INITED)
haService.init(conf)
haService.getConf should be(conf)
haService.getStartTime should be(0)
haService.getServiceState should be(INITED)
tryWithSecurityEnabled {
val e = intercept[IOException](haService.init(conf))
e.getMessage should startWith(KyuubiSparkUtil.KEYTAB)
}
}
test("Start") {
val e1 = intercept[ServiceException](haService.start())
e1.getCause.getMessage should startWith("Unable to address the server")
server.init(conf)
val e2 = intercept[ServiceException](haService.start())
e2.getCause.getMessage should be("client cannot be null")
haService.init(conf)
haService.start()
haService.getServiceState should be(STARTED)
haService.getStartTime should not be 0
}
test("Stop before init") {
haService.stop()
}
test("Stop after init") {
haService.init(conf)
haService.stop()
}
test("Stop after start") {
server.init(conf)
haService.init(conf)
haService.start()
haService.stop()
}
}
Example 166
| Source File: KyuubiDistributedCacheManagerSuite.scala From kyuubi with Apache License 2.0 | 5 votes |
|
package org.apache.spark.deploy.yarn
import java.net.URI
import scala.collection.mutable.{HashMap, Map}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.hadoop.yarn.api.records.{LocalResource, LocalResourceType, LocalResourceVisibility}
import org.apache.hadoop.yarn.util.ConverterUtils
import org.apache.spark.{KyuubiSparkUtil, SparkFunSuite}
import org.mockito.Mockito.when
import org.scalatest.mock.MockitoSugar
import yaooqinn.kyuubi.utils.ReflectUtils
class KyuubiDistributedCacheManagerSuite extends SparkFunSuite with MockitoSugar {
class MockClientDistributedCacheManager extends ClientDistributedCacheManager {
override def getVisibility(conf: Configuration, uri: URI, statCache: Map[URI, FileStatus]):
LocalResourceVisibility = {
LocalResourceVisibility.PRIVATE
}
}
test("add resource") {
val fs = mock[FileSystem]
val conf = new Configuration()
val destPath = new Path("file:///foo.bar.com:8080/tmp/testing")
val localResources = HashMap[String, LocalResource]()
val statCache = HashMap[URI, FileStatus]()
val status = new FileStatus()
when(fs.getFileStatus(destPath)).thenReturn(status)
val fileLink = "link"
ReflectUtils.setFieldValue(
KyuubiDistributedCacheManager, "cacheManager", new MockClientDistributedCacheManager)
KyuubiDistributedCacheManager.addResource(
fs, conf, destPath, localResources, LocalResourceType.FILE, fileLink, statCache)
val res = localResources(fileLink)
assert(res.getVisibility === LocalResourceVisibility.PRIVATE)
assert(ConverterUtils.getPathFromYarnURL(res.getResource) === destPath)
assert(res.getSize === 0)
assert(res.getTimestamp === 0)
assert(res.getType === LocalResourceType.FILE)
val status2 = new FileStatus(
10, false, 1, 1024, 10,
10, null, KyuubiSparkUtil.getCurrentUserName, null, new Path("/tmp/testing2"))
val destPath2 = new Path("file:///foo.bar.com:8080/tmp/testing2")
when(fs.getFileStatus(destPath2)).thenReturn(status2)
val fileLink2 = "link2"
KyuubiDistributedCacheManager.addResource(
fs, conf, destPath2, localResources, LocalResourceType.FILE, fileLink2, statCache)
val res2 = localResources(fileLink2)
assert(res2.getVisibility === LocalResourceVisibility.PRIVATE)
assert(ConverterUtils.getPathFromYarnURL(res2.getResource) === destPath2)
assert(res2.getSize === 10)
assert(res2.getTimestamp === 10)
assert(res2.getType === LocalResourceType.FILE)
}
test("add resource when link null") {
val distMgr = new MockClientDistributedCacheManager()
val fs = mock[FileSystem]
val conf = new Configuration()
val destPath = new Path("file:///foo.bar.com:8080/tmp/testing")
ReflectUtils.setFieldValue(KyuubiDistributedCacheManager, "cacheManager", distMgr)
val localResources = HashMap[String, LocalResource]()
val statCache = HashMap[URI, FileStatus]()
when(fs.getFileStatus(destPath)).thenReturn(new FileStatus())
intercept[Exception] {
KyuubiDistributedCacheManager.addResource(
fs, conf, destPath, localResources, LocalResourceType.FILE, null, statCache)
}
assert(localResources.get("link") === None)
assert(localResources.size === 0)
}
test("test addResource archive") {
val distMgr = new MockClientDistributedCacheManager()
ReflectUtils.setFieldValue(KyuubiDistributedCacheManager, "cacheManager", distMgr)
val fs = mock[FileSystem]
val conf = new Configuration()
val destPath = new Path("file:///foo.bar.com:8080/tmp/testing")
val localResources = HashMap[String, LocalResource]()
val statCache = HashMap[URI, FileStatus]()
val realFileStatus = new FileStatus(10, false, 1, 1024, 10, 10, null, "testOwner",
null, new Path("/tmp/testing"))
when(fs.getFileStatus(destPath)).thenReturn(realFileStatus)
KyuubiDistributedCacheManager.addResource(
fs, conf, destPath, localResources, LocalResourceType.ARCHIVE, "link", statCache)
val resource = localResources("link")
assert(resource.getVisibility === LocalResourceVisibility.PRIVATE)
assert(ConverterUtils.getPathFromYarnURL(resource.getResource) === destPath)
assert(resource.getTimestamp === 10)
assert(resource.getSize === 10)
assert(resource.getType === LocalResourceType.ARCHIVE)
}
}
Example 167
| Source File: BigQueryClientSpecs.scala From spark-bigquery with Apache License 2.0 | 4 votes |
|
package com.samelamin.spark.bigquery
import java.io.File
import com.google.api.services.bigquery.Bigquery
import com.google.api.services.bigquery.model._
import com.google.cloud.hadoop.io.bigquery._
import com.holdenkarau.spark.testing.DataFrameSuiteBase
import com.samelamin.spark.bigquery.converters.{BigQueryAdapter, SchemaConverters}
import org.apache.commons.io.FileUtils
import org.apache.spark.sql._
import org.mockito.Matchers.{any, eq => mockitoEq}
import org.mockito.Mockito._
import org.scalatest.FeatureSpec
import org.scalatest.mock.MockitoSugar
class BigQueryClientSpecs extends FeatureSpec with DataFrameSuiteBase with MockitoSugar {
val BQProjectId = "google.com:foo-project"
def setupBigQueryClient(sqlCtx: SQLContext, bigQueryMock: Bigquery): BigQueryClient = {
val fakeJobReference = new JobReference()
fakeJobReference.setProjectId(BQProjectId)
fakeJobReference.setJobId("bigquery-job-1234")
val dataProjectId = "publicdata"
// Create the job result.
val jobStatus = new JobStatus()
jobStatus.setState("DONE")
jobStatus.setErrorResult(null)
val jobHandle = new Job()
jobHandle.setStatus(jobStatus)
jobHandle.setJobReference(fakeJobReference)
// Create table reference.
val tableRef = new TableReference()
tableRef.setProjectId(dataProjectId)
tableRef.setDatasetId("test_dataset")
tableRef.setTableId("test_table")
// Mock getting Bigquery jobs
when(bigQueryMock.jobs().get(any[String], any[String]).execute())
.thenReturn(jobHandle)
when(bigQueryMock.jobs().insert(any[String], any[Job]).execute())
.thenReturn(jobHandle)
val bigQueryClient = new BigQueryClient(sqlCtx, bigQueryMock)
bigQueryClient
}
scenario("When writing to BQ") {
val sqlCtx = sqlContext
import sqlCtx.implicits._
val gcsPath = "/tmp/testfile2.json"
FileUtils.deleteQuietly(new File(gcsPath))
val adaptedDf = BigQueryAdapter(sc.parallelize(List(1, 2, 3)).toDF)
val bigQueryMock = mock[Bigquery](RETURNS_DEEP_STUBS)
val fullyQualifiedOutputTableId = "testProjectID:test_dataset.test"
val targetTable = BigQueryStrings.parseTableReference(fullyQualifiedOutputTableId)
val bigQueryClient = setupBigQueryClient(sqlCtx, bigQueryMock)
val bigQuerySchema = SchemaConverters.SqlToBQSchema(adaptedDf)
bigQueryClient.load(targetTable,bigQuerySchema,gcsPath)
verify(bigQueryMock.jobs().insert(mockitoEq(BQProjectId),any[Job]), times(1)).execute()
}
scenario("When reading from BQ") {
val sqlCtx = sqlContext
val fullyQualifiedOutputTableId = "testProjectID:test_dataset.test"
val sqlQuery = s"select * from $fullyQualifiedOutputTableId"
val bqQueryContext = new BigQuerySQLContext(sqlCtx)
bqQueryContext.setBigQueryProjectId(BQProjectId)
val bigQueryMock = mock[Bigquery](RETURNS_DEEP_STUBS)
val bigQueryClient = setupBigQueryClient(sqlCtx, bigQueryMock)
bigQueryClient.selectQuery(sqlQuery)
verify(bigQueryMock.jobs().insert(mockitoEq(BQProjectId),any[Job]), times(1)).execute()
}
scenario("When running a DML Queries") {
val sqlCtx = sqlContext
val fullyQualifiedOutputTableId = "testProjectID:test_dataset.test"
val dmlQuery = s"UPDATE $fullyQualifiedOutputTableId SET test_col = new_value WHERE test_col = old_value"
val bqQueryContext = new BigQuerySQLContext(sqlCtx)
bqQueryContext.setBigQueryProjectId(BQProjectId)
val bigQueryMock = mock[Bigquery](RETURNS_DEEP_STUBS)
val bigQueryClient = setupBigQueryClient(sqlCtx, bigQueryMock)
bigQueryClient.runDMLQuery(dmlQuery)
verify(bigQueryMock.jobs().insert(mockitoEq(BQProjectId),any[Job]), times(1)).execute()
}
}
