org.I0Itec.zkclient.exception.ZkTimeoutException Java Examples
The following examples show how to use
org.I0Itec.zkclient.exception.ZkTimeoutException.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: ZkClient.java From DDMQ with Apache License 2.0 | 6 votes |
|
/**
* Connect to ZooKeeper.
*
* @param maxMsToWaitUntilConnected
* @param watcher
* @throws ZkInterruptedException
* if the connection timed out due to thread interruption
* @throws ZkTimeoutException
* if the connection timed out
* @throws IllegalStateException
* if the connection timed out due to thread interruption
*/
public void connect(final long maxMsToWaitUntilConnected, Watcher watcher) throws ZkInterruptedException, ZkTimeoutException, IllegalStateException {
boolean started = false;
acquireEventLock();
try {
setShutdownTrigger(false);
_eventThread = new ZkEventThread(_connection.getServers());
_eventThread.start();
_connection.connect(watcher);
LOG.debug("Awaiting connection to Zookeeper server");
boolean waitSuccessful = waitUntilConnected(maxMsToWaitUntilConnected, TimeUnit.MILLISECONDS);
if (!waitSuccessful) {
throw new ZkTimeoutException("Unable to connect to zookeeper server '" + _connection.getServers() + "' with timeout of " + maxMsToWaitUntilConnected + " ms");
}
started = true;
} finally {
getEventLock().unlock();
// we should close the zookeeper instance, otherwise it would keep
// on trying to connect
if (!started) {
close();
}
}
}
Example #2
| Source File: SamzaExecutor.java From samza with Apache License 2.0 | 6 votes |
|
@Override
public List<String> listTables(ExecutionContext context) throws ExecutorException {
String address = environmentVariableHandler.getEnvironmentVariable(SAMZA_SQL_SYSTEM_KAFKA_ADDRESS);
if (address == null || address.isEmpty()) {
address = DEFAULT_SERVER_ADDRESS;
}
try {
ZkUtils zkUtils = new ZkUtils(new ZkClient(address, DEFAULT_ZOOKEEPER_CLIENT_TIMEOUT),
new ZkConnection(address), false);
return JavaConversions.seqAsJavaList(zkUtils.getAllTopics())
.stream()
.map(x -> SAMZA_SYSTEM_KAFKA + "." + x)
.collect(Collectors.toList());
} catch (ZkTimeoutException ex) {
throw new ExecutorException(ex);
}
}
Example #3
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 6 votes |
|
@Test(expectedExceptions = SiddhiAppValidationException.class,
dependsOnMethods = "testKafkaMultipleTopicWithThreadingPerPartitionSource")
public void testKafkaWithoutThreadingOptionSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for without any threading option defined.");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"no_threading_option_topic"};
KafkaTestUtil.createTopic(topics, 2);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='no_threading_option_topic', partition.no.list='0,1,2', "
+ "group.id='test_no_threading_option_topic'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.start();
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #4
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 6 votes |
|
@Test(expectedExceptions = SiddhiAppValidationException.class,
dependsOnMethods = "testKafkaMultipleTopic_MultiplePartition_AllPartitionSubscribe_Source")
public void testKafkaWithoutBootstrapServerSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for without any bootstrap servers defined.");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"no_bootstrap_server_topic"};
KafkaTestUtil.createTopic(topics, 2);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='no_bootstrap_server_topic', partition.no.list='0,1,2', "
+ "group.id='test', threading.option='single.thread'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.start();
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #5
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 6 votes |
|
@Test(expectedExceptions = SiddhiAppValidationException.class,
dependsOnMethods = "testTransportCreationDisabledProperty")
public void testKafkaWithoutTopicSource() {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for without topic");
log.info("-------------------------------------------------------------------------------------------");
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', group.id='test', threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.start();
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #6
| Source File: ZkClient.java From DDMQ with Apache License 2.0 | 6 votes |
|
/**
* Connect to ZooKeeper.
*
* @param maxMsToWaitUntilConnected
* @param watcher
* @throws ZkInterruptedException
* if the connection timed out due to thread interruption
* @throws ZkTimeoutException
* if the connection timed out
* @throws IllegalStateException
* if the connection timed out due to thread interruption
*/
public void connect(final long maxMsToWaitUntilConnected, Watcher watcher) throws ZkInterruptedException, ZkTimeoutException, IllegalStateException {
boolean started = false;
acquireEventLock();
try {
setShutdownTrigger(false);
_eventThread = new ZkEventThread(_connection.getServers());
_eventThread.start();
_connection.connect(watcher);
LOG.debug("Awaiting connection to Zookeeper server");
boolean waitSuccessful = waitUntilConnected(maxMsToWaitUntilConnected, TimeUnit.MILLISECONDS);
if (!waitSuccessful) {
throw new ZkTimeoutException("Unable to connect to zookeeper server '" + _connection.getServers() + "' with timeout of " + maxMsToWaitUntilConnected + " ms");
}
started = true;
} finally {
getEventLock().unlock();
// we should close the zookeeper instance, otherwise it would keep
// on trying to connect
if (!started) {
close();
}
}
}
Example #7
| Source File: KAFKAMessageListener.java From micro-integrator with Apache License 2.0 | 5 votes |
|
/**
* Create the connection with the zookeeper to consume the messages
*/
public boolean createKafkaConsumerConnector() throws Exception {
log.debug("Create the connection and start to consume the streams");
boolean isCreated;
try {
if (consumerConnector == null) {
log.info("Creating Kafka Consumer Connector...");
//set default consumer timeout to 3000ms if it is not set by the user
if (!kafkaProperties.containsKey(KAFKAConstants.CONSUMER_TIMEOUT)) {
kafkaProperties.put(KAFKAConstants.CONSUMER_TIMEOUT, "3000");
}
consumerConnector = Consumer.createJavaConsumerConnector(new ConsumerConfig(kafkaProperties));
log.info("Kafka Consumer Connector is created");
start();
}
isCreated = true;
} catch (ZkTimeoutException toe) {
log.error(" Error in Creating Kafka Consumer Connector | ZkTimeout" + toe.getMessage());
throw new SynapseException(" Error in Creating Kafka Consumer Connector| ZkTimeout");
} catch (Exception e) {
log.error(" Error in Creating Kafka Consumer Connector." + e.getMessage(), e);
throw new SynapseException(" Error in Creating Kafka Consumer Connector ", e);
}
return isCreated;
}
Example #8
| Source File: EmbeddedKafkaServer.java From twill with Apache License 2.0 | 5 votes |
|
@Override
protected void startUp() throws Exception {
int tries = 0;
do {
KafkaConfig kafkaConfig = createKafkaConfig(properties);
KafkaServer kafkaServer = createKafkaServer(kafkaConfig);
try {
kafkaServer.startup();
server = kafkaServer;
} catch (Exception e) {
kafkaServer.shutdown();
kafkaServer.awaitShutdown();
Throwable rootCause = Throwables.getRootCause(e);
if (rootCause instanceof ZkTimeoutException) {
// Potentially caused by race condition bug described in TWILL-139.
LOG.warn("Timeout when connecting to ZooKeeper from KafkaServer. Attempt number {}.", tries, rootCause);
} else if (rootCause instanceof BindException) {
LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause);
} else {
throw e;
}
// Do a random sleep of < 200ms
TimeUnit.MILLISECONDS.sleep(new Random().nextInt(200) + 1L);
}
} while (server == null && ++tries < startTimeoutRetries);
if (server == null) {
throw new IllegalStateException("Failed to start Kafka server after " + tries + " attempts.");
}
}
Example #9
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 5 votes |
|
@Test(expectedExceptions = SiddhiAppValidationException.class,
dependsOnMethods = "testKafkaWithoutThreadingOptionSource")
public void testKafkaSingleTopicWithoutGroupIdSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for single topic without group ID");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"single_topic_without_groupid"};
KafkaTestUtil.createTopic(topics, 1);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='single_topic_without_groupid', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.start();
KafkaTestUtil.deleteTopic(topics);
Thread.sleep(4000);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #10
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 5 votes |
|
@Test(dependsOnMethods = "testKafkaSingleTopicWithSpecificSubscribeSource")
public void testKafkaSpecificSubscribeForUnavailablePartitionSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for single topic with partitions which subscribes for an unavailable partition.");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"topic_without_some_partition"};
KafkaTestUtil.createTopic(topics, 2);
SiddhiManager siddhiManager = new SiddhiManager();
Logger logger = Logger.getLogger(Source.class);
UnitTestAppender appender = new UnitTestAppender();
logger.addAppender(appender);
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='topic_without_some_partition', "
+ "partition.no.list='0,1,2', "
+ "group.id='test_topic_without_some_partition', threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.start();
Thread.sleep(5000);
if (appender.getMessages() != null) {
AssertJUnit.assertTrue(appender.getMessages().contains("Error on 'TestExecutionPlan'. Partition " +
"number(s) 2 aren't available for the topic: topic_without_some_partition Error while " +
"connecting at Source 'kafka' at 'FooStream'."));
}
logger.removeAppender(appender);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #11
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaSpecificSubscribeForUnavailablePartitionSource")
public void testKafkaMultipleTopic_MultiplePartition_OnePartitionSubscribe_Source() throws
InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test to configure Kafka source with multiple topics having multiple partitions "
+ "subscribing for single partition id");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"multiple_topic1_two_par_one_sub", "multiple_topic2_two_par_one_sub"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 2);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='multiple_topic1_two_par_one_sub,multiple_topic2_two_par_one_sub', "
+ "partition.no.list='1', "
+ "group.id='test_multiple_topic1_two_par_one_sub', "
+ "threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("multiple_topic1_two_par_one_sub");
expectedNames.add("multiple_topic2_two_par_one_sub");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(1L);
expectedValues.add(1L);
AssertJUnit.assertEquals(2, count);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #12
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaMultipleTopicPartitionTopicWiseSubscription")
public void testKafkaTopic_MultiplePartition_AllSubscribe_Source() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test to configure Kafka source with a single topic having multiple partitions "
+ "subscribing for all partition ids");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"kafka_topic_888"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 4);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='kafka_topic_888', "
+ "partition.no.list='0,1,2,3', "
+ "group.id='test_single_topic1_multiple_par_all_sub', "
+ "threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 4, 4, true, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("kafka_topic_888");
expectedNames.add("kafka_topic_888");
expectedNames.add("kafka_topic_888");
expectedNames.add("kafka_topic_888");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
expectedValues.add(1L);
expectedValues.add(2L);
expectedValues.add(3L);
AssertJUnit.assertEquals(4, count);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames, receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #13
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
public void testKafkaSingleTopic_MultiplePartition_AllPartitionSubscribe_Source_106() throws
InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test to configure Kafka source with multiple topics having multiple partitions "
+ "subscribing for all partition ids");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"single_topic1_two_par_two_servers"};
KafkaTestUtil.setupKafkaBroker2();
receivedEventNameList = new ArrayList<>(2);
receivedValueList = new ArrayList<>(2);
Thread.sleep(10000);
KafkaTestUtil.createTopic(topics, 2);
Thread.sleep(5000);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='single_topic1_two_par_two_servers', "
+ "partition.no.list='0,1', "
+ "group.id='test', "
+ "threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092,localhost:9093'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
Thread.sleep(2000);
KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, "localhost:9093,localhost:9092", true);
Thread.sleep(1000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("single_topic1_two_par_two_servers");
expectedNames.add("single_topic1_two_par_two_servers");
expectedNames.add("single_topic1_two_par_two_servers");
expectedNames.add("single_topic1_two_par_two_servers");
List<Long> expectedValues = new ArrayList<>(2);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals(2, count);
KafkaTestUtil.deleteTopic(topics);
Thread.sleep(1000);
KafkaTestUtil.stopKafkaBroker2();
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #14
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaMultipleTopicPartitionPartitionWiseSubscription")
public void testKafkaMultipleTopicPartitionTopicWiseSubscription() throws InterruptedException {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for multiple topics and partitions and thread topic wise");
log.info("-------------------------------------------------------------------------------------------");
try {
String topics[] = new String[]{"kafka_topic_with_2par", "kafka_topic2_with_2par"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 2);
Thread.sleep(100);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='kafka_topic_with_2par,kafka_topic2_with_2par', "
+ "group.id='test_kafka_topic_with_2par_kafka_topic2_with_2par', "
+ "threading.option='topic.wise', bootstrap.servers='localhost:9092', "
+ "partition.no.list='0,1', " +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
Thread.sleep(100);
AssertJUnit.assertEquals(4, count);
AssertJUnit.assertTrue(eventArrived);
List<String> expectedNames = new ArrayList<>(4);
expectedNames.add("kafka_topic_with_2par");
expectedNames.add("kafka_topic_with_2par");
expectedNames.add("kafka_topic2_with_2par");
expectedNames.add("kafka_topic2_with_2par");
List<Long> expectedValues = new ArrayList<>(4);
expectedValues.add(0L);
expectedValues.add(1L);
expectedValues.add(0L);
expectedValues.add(1L);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #15
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaSingleTopicSameGroupIdsSource")
public void testKafkaNonExistingTopicSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for non-existing topic. This will create a topic with default partition");
log.info("-------------------------------------------------------------------------------------------");
receivedEventNameList = new ArrayList<>(2);
receivedValueList = new ArrayList<>(2);
String topics[] = new String[]{"non_existing_topic1"};
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='non_existing_topic1', enable.auto.commit='false', "
+ "group.id='test_non_existing_topic1', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 1, 1, false, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("non_existing_topic1");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
KafkaTestUtil.deleteTopic(topics);
AssertJUnit.assertEquals(1, count);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #16
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaMultipleTopicWithThreadingPerTopicSource")
public void testKafkaMultipleTopicWithThreadingPerPartitionSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for multiple topic with thread per partition");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"multiple_topic1_2par", "multiple_topic2_2par"};
receivedEventNameList = new ArrayList<>(2);
receivedValueList = new ArrayList<>(2);
KafkaTestUtil.createTopic(topics, 2);
Thread.sleep(100);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='multiple_topic1_2par,multiple_topic2_2par', "
+ "group.id='test_multiple_topic1_2par_multiple_topic2_2par', "
+ "threading.option='partition.wise', "
+ "bootstrap.servers='localhost:9092', "
+ "partition.no.list='0,1'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("multiple_topic1_2par");
expectedNames.add("multiple_topic1_2par");
expectedNames.add("multiple_topic2_2par");
expectedNames.add("multiple_topic2_2par");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
expectedValues.add(1L);
expectedValues.add(0L);
expectedValues.add(1L);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
AssertJUnit.assertEquals(4, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #17
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaWithoutBootstrapServerSource")
public void testKafkaMultipleTopicWithThreadingPerTopicSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for multiple topic with thread per topic");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"multiple_topic1", "multiple_topic2"};
receivedEventNameList = new ArrayList<>(2);
receivedValueList = new ArrayList<>(2);
KafkaTestUtil.createTopic(topics, 1);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='multiple_topic1,multiple_topic2', "
+ "group.id='test_multiple_topic1_multiple_topic2', "
+ "threading.option='topic.wise', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
Thread.sleep(10000);
KafkaTestUtil.kafkaPublisher(topics, 1, 2, false, null, true);
Thread.sleep(300);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("multiple_topic1");
expectedNames.add("multiple_topic1");
expectedNames.add("multiple_topic2");
expectedNames.add("multiple_topic2");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
expectedValues.add(1L);
expectedValues.add(0L);
expectedValues.add(1L);
SiddhiTestHelper.waitForEvents(5000, 4, count, 15000);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
AssertJUnit.assertEquals(4, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #18
| Source File: KafkaSinkwithBinaryMapperTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test
public void testPublisherUsingBinaryMapper() throws InterruptedException {
LOG.info("Creating test for publishing events using binary mapper.");
String topics[] = new String[]{"single_topic"};
KafkaTestUtil.createTopic(topics, 1);
receivedEventNameList = new ArrayList<>(3);
receivedValueList = new ArrayList<>(3);
try {
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan1') " +
"define stream BarStream2 (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='single_topic', group.id='single_topic_test', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092', " +
"is.binary.message='true'," +
"@map(type='binary'))" +
"Define stream FooStream2 (symbol string, price float, volume long);" +
"from FooStream2 select symbol, price, volume insert into BarStream2;");
siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
@Override
public void receive(Event[] events) {
EventPrinter.print(events);
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntimeSource.start();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='single_topic', bootstrap.servers='localhost:9092', " +
"is.binary.message = 'true'," +
"@map(type='binary'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
siddhiAppRuntime.start();
fooStream.send(new Object[]{"single_topic", 55.6f, 100L});
fooStream.send(new Object[]{"single_topic2", 75.6f, 102L});
fooStream.send(new Object[]{"single_topic3", 57.6f, 103L});
Thread.sleep(2000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("single_topic");
expectedNames.add("single_topic2");
expectedNames.add("single_topic3");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(100L);
expectedValues.add(102L);
expectedValues.add(103L);
AssertJUnit.assertEquals("Kafka Sink published the expected events", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Sink published the expected events", expectedValues, receivedValueList);
AssertJUnit.assertEquals(3, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
siddhiAppRuntimeSource.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
Example #19
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaMultipleTopic_MultiplePartition_OnePartitionSubscribe_Source")
public void testKafkaMultipleTopic_MultiplePartition_AllPartitionSubscribe_Source() throws
InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test to configure Kafka source with multiple topics having multiple partitions "
+ "subscribing for all partition ids");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"multiple_topic1_two_par_all_sub", "multiple_topic2_two_par_all_sub"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 2);
Thread.sleep(1000);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='multiple_topic1_two_par_all_sub,multiple_topic2_two_par_all_sub', "
+ "partition.no.list='0,1', "
+ "group.id='test_multiple_topic1_two_par_all_sub', "
+ "threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("multiple_topic1_two_par_all_sub");
expectedNames.add("multiple_topic1_two_par_all_sub");
expectedNames.add("multiple_topic2_two_par_all_sub");
expectedNames.add("multiple_topic2_two_par_all_sub");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
expectedValues.add(1L);
expectedValues.add(0L);
expectedValues.add(1L);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
AssertJUnit.assertEquals(4, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #20
| Source File: KafkaSinkTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testPublisherWithTopicWithoutPartitionKafkaTransport")
public void testPublisherWithTopicWithPartitionKafkaTransport() throws InterruptedException {
LOG.info("Creating test for publishing events for static topic with a partition");
String topics[] = new String[]{"topic_with_two_partitions_sub0"};
KafkaTestUtil.createTopic(topics, 2);
receivedEventNameList = new ArrayList<>(3);
receivedValueList = new ArrayList<>(3);
try {
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan2') " +
"define stream BarStream2 (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='topic_with_two_partitions_sub0', partition.no.list='0',"
+ "group.id='topic_with_two_partitions_sub0_test', threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream2 (symbol string, price float, volume long);" +
"from FooStream2 select symbol, price, volume insert into BarStream2;");
siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntimeSource.start();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan3') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='topic_with_two_partitions_sub0', partition.no='0', "
+ "bootstrap.servers='localhost:9092', " +
"@map(type='xml'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
siddhiAppRuntime.start();
fooStream.send(new Object[]{"single_topic", 55.6f, 100L});
fooStream.send(new Object[]{"single_topic2", 75.6f, 102L});
fooStream.send(new Object[]{"single_topic3", 57.6f, 103L});
Thread.sleep(2000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("single_topic");
expectedNames.add("single_topic2");
expectedNames.add("single_topic3");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(100L);
expectedValues.add(102L);
expectedValues.add(103L);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
AssertJUnit.assertEquals(3, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
siddhiAppRuntimeSource.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
Example #21
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaMultipleTopicSource")
public void testKafkaSingleTopicWithSpecificSubscribeSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for single topic with one partition which subscribes for the partition "
+ "specifically");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"topic_with_one_partition"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 1);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='topic_with_one_partition', "
+ "partition.no.list='0', "
+ "group.id='test_topic_with_one_partition', "
+ "threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 1, 1, true, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("topic_with_one_partition");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
AssertJUnit.assertEquals(1, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #22
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testKafkaWithoutTopicSource")
public void testKafkaMultipleTopicSource() throws InterruptedException {
try {
log.info("-------------------------------------------------------------------------------------------");
log.info("Creating test for multiple topic");
log.info("-------------------------------------------------------------------------------------------");
String topics[] = new String[]{"multiple_topic1", "multiple_topic2"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 1);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='multiple_topic1,multiple_topic2', "
+ "group.id='test_multiple_topic1_multiple_topic2', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 1, 2, false, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("multiple_topic1");
expectedNames.add("multiple_topic1");
expectedNames.add("multiple_topic2");
expectedNames.add("multiple_topic2");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
expectedValues.add(1L);
expectedValues.add(0L);
expectedValues.add(1L);
AssertJUnit.assertEquals(4, count);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #23
| Source File: KafkaSinkTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test
public void testPublisherWithTopicWithoutPartitionKafkaTransport() throws InterruptedException {
LOG.info("Creating test for publishing events for static topic without a partition");
String topics[] = new String[]{"single_topic"};
KafkaTestUtil.createTopic(topics, 1);
receivedEventNameList = new ArrayList<>(3);
receivedValueList = new ArrayList<>(3);
try {
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan1') " +
"define stream BarStream2 (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='single_topic', group.id='single_topic_test', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream2 (symbol string, price float, volume long);" +
"from FooStream2 select symbol, price, volume insert into BarStream2;");
siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntimeSource.start();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='single_topic', bootstrap.servers='localhost:9092', " +
"@map(type='xml'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
siddhiAppRuntime.start();
fooStream.send(new Object[]{"single_topic", 55.6f, 100L});
fooStream.send(new Object[]{"single_topic2", 75.6f, 102L});
fooStream.send(new Object[]{"single_topic3", 57.6f, 103L});
Thread.sleep(3000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("single_topic");
expectedNames.add("single_topic2");
expectedNames.add("single_topic3");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(100L);
expectedValues.add(102L);
expectedValues.add(103L);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
AssertJUnit.assertEquals(3, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
siddhiAppRuntimeSource.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
Example #24
| Source File: KafkaSourceTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test
public void testKafkaSingleTopicSource() throws InterruptedException {
try {
log.info("Creating test for single topic");
String topics[] = new String[]{"single_topic"};
receivedEventNameList = new ArrayList<>(2);
receivedValueList = new ArrayList<>(2);
KafkaTestUtil.createTopic(topics, 1);
Thread.sleep(1000);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') @App:transportChannelCreationEnabled('false')" +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='single_topic', group.id='test_single_topic', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
KafkaTestUtil.kafkaPublisher(topics, 1, 2, false, null, true);
Thread.sleep(100);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("single_topic");
expectedNames.add("single_topic");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
expectedValues.add(1L);
SiddhiTestHelper.waitForEvents(2000, 2, count, 20000);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
AssertJUnit.assertEquals(2, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #25
| Source File: KafkaSourceHATestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test
public void testAKafkaPauseAndResume() throws InterruptedException {
try {
log.info("Test to verify the pause and resume functionality of Kafka source");
String topics[] = new String[]{"kafka_topic3"};
KafkaTestUtil.createTopic(topics, 2);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='kafka_topic3', group.id='test1', threading" +
".option='partition.wise', " +
"bootstrap.servers='localhost:9092', partition.no.list='0,1', " +
"@map(type='text'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
log.info(event);
eventArrived = true;
count++;
}
}
});
siddhiAppRuntime.start();
Future eventSender = executorService.submit(new Runnable() {
@Override
public void run() {
KafkaTestUtil.kafkaPublisher(topics, 2, 4, false, null, false);
}
});
while (!eventSender.isDone()) {
Thread.sleep(1000);
}
Thread.sleep(2000);
assertEquals(4, count);
assertTrue(eventArrived);
Collection<List<Source>> sources = siddhiAppRuntime.getSources();
// pause the transports
sources.forEach(e -> e.forEach(Source::pause));
init2();
eventSender = executorService.submit(new Runnable() {
@Override
public void run() {
KafkaTestUtil.kafkaPublisher(topics, 2, 4, false, null, false);
}
});
while (!eventSender.isDone()) {
Thread.sleep(1000);
}
Thread.sleep(5000);
assertFalse(eventArrived);
// resume the transports
sources.forEach(e -> e.forEach(Source::resume));
Thread.sleep(2000);
assertEquals(4, count);
assertTrue(eventArrived);
siddhiAppRuntime.shutdown();
} catch (ZkTimeoutException ex) {
log.warn("No zookeeper may not be available.", ex);
}
}
Example #26
| Source File: KafkaSinkTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
public void testPublisherWithKafkaTransportWithDynamicTopicAndPartition() throws InterruptedException {
LOG.info("Creating test for publishing events for dynamic topic with dynamic partition");
try {
String topics[] = new String[]{"multiple_topic1_two_par_all_sub1", "multiple_topic2_two_par_all_sub1"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 2);
Thread.sleep(10000);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime executionPlanRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan14') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='{{symbol}}', partition.no='{{volume}}', "
+ "bootstrap.servers='localhost:9092', " +
"@map(type='xml'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream; ");
InputHandler fooStream = executionPlanRuntime.getInputHandler("FooStream");
executionPlanRuntime.start();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan15') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', "
+ "topic.list='multiple_topic1_two_par_all_sub1,multiple_topic2_two_par_all_sub1', "
+ "group.id='multiple_topic1_two_par_all_sub1_test', threading.option='single.thread', "
+ "bootstrap.servers='localhost:9092', "
+ "partition.no.list='0,1'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
Thread.sleep(2000);
fooStream.send(new Object[]{"multiple_topic1_two_par_all_sub1", 55.6f, 0L});
Thread.sleep(1000);
fooStream.send(new Object[]{"multiple_topic1_two_par_all_sub1", 55.6f, 1L});
Thread.sleep(1000);
fooStream.send(new Object[]{"multiple_topic2_two_par_all_sub1", 75.6f, 0L});
Thread.sleep(1000);
fooStream.send(new Object[]{"multiple_topic2_two_par_all_sub1", 75.6f, 1L});
Thread.sleep(5000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("multiple_topic1_two_par_all_sub1");
expectedNames.add("multiple_topic1_two_par_all_sub1");
expectedNames.add("multiple_topic2_two_par_all_sub1");
expectedNames.add("multiple_topic2_two_par_all_sub1");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(0L);
expectedValues.add(1L);
expectedValues.add(0L);
expectedValues.add(1L);
AssertJUnit.assertEquals(4, count);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
KafkaTestUtil.deleteTopic(topics);
executionPlanRuntime.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
Example #27
| Source File: KafkaSinkTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods =
"testPublisherWithInvalidTopicWithPartitionOtherThan0KafkaTransportReceiveMessage")
public void testPublisherWithKafkaTransportWithDynamicTopic() throws InterruptedException {
LOG.info("Creating test for publishing events for dynamic topic without partition");
try {
String topics[] = new String[]{"multiple_topic1", "multiple_topic2"};
receivedEventNameList = new ArrayList<>(4);
receivedValueList = new ArrayList<>(4);
KafkaTestUtil.createTopic(topics, 1);
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime executionPlanRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan12') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='{{symbol}}', bootstrap.servers='localhost:9092', " +
"@map(type='xml'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream; ");
InputHandler fooStream = executionPlanRuntime.getInputHandler("FooStream");
executionPlanRuntime.start();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan13') " +
"define stream BarStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='multiple_topic1,multiple_topic2', "
+ "group.id='multiple_topic1_multiple_topic2_test', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092'," +
"@map(type='xml'))" +
"Define stream FooStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntime.start();
fooStream.send(new Object[]{"multiple_topic1", 55.6f, 100L});
fooStream.send(new Object[]{"multiple_topic2", 75.6f, 102L});
Thread.sleep(2000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("multiple_topic1");
expectedNames.add("multiple_topic2");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(100L);
expectedValues.add(102L);
AssertJUnit.assertEquals(2, count);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
KafkaTestUtil.deleteTopic(topics);
executionPlanRuntime.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
Example #28
| Source File: KafkaSinkTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
public void testPublisherWithInvalidTopicWithPartitionOtherThan0KafkaTransport() throws InterruptedException {
LOG.info("Creating test for publishing events for invalid topic with a partition other than 0 but the source "
+ "will be not be getting events from the default partition since the key is not defined");
String topics[] = new String[]{"invalid_topic_with_partition_2"};
receivedEventNameList = new ArrayList<>(3);
receivedValueList = new ArrayList<>(3);
try {
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan10') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='invalid_topic_with_partition_2', "
+ "bootstrap.servers='localhost:9092', partition.no='2', " +
"@map(type='xml'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
siddhiAppRuntime.start();
Thread.sleep(2000);
//this event will be published to the cluster and create the topic
fooStream.send(new Object[]{"invalid_topic_with_partition_2", 55.6f, 100L});
//this thread sleep is to slowdown the below execution plan because, if there are no topics kafka source
// will create the topic.
Thread.sleep(2000);
SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan11') " +
"define stream BarStream2 (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='invalid_topic_with_partition_2', "
+ "group.id='invalid_topic_with_partition_2_test', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092', " +
"@map(type='xml'))" +
"Define stream FooStream2 (symbol string, price float, volume long);" +
"from FooStream2 select symbol, price, volume insert into BarStream2;");
siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntimeSource.start();
Thread.sleep(2000);
fooStream.send(new Object[]{"invalid_topic_with_partition_22", 75.6f, 102L});
fooStream.send(new Object[]{"invalid_topic_with_partition_23", 57.6f, 103L});
Thread.sleep(2000);
AssertJUnit.assertEquals(0, count);
KafkaTestUtil.deleteTopic(topics);
Thread.sleep(4000);
siddhiAppRuntime.shutdown();
siddhiAppRuntimeSource.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
Example #29
| Source File: KafkaSinkTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testPublisherWithInvalidTopicWithPartitionKafkaTransport")
public void testPublisherWithInvalidTopicWithPartitionOtherThan0KafkaTransportReceiveMessage() throws
InterruptedException {
LOG.info("Creating test for publishing events for invalid topic with a partition other than 0 but the source "
+ "will be getting events from the default partition when given a key");
String topics[] = new String[]{"invalid_topic_with_partition_3"};
receivedEventNameList = new ArrayList<>(3);
receivedValueList = new ArrayList<>(3);
try {
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan8') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='invalid_topic_with_partition_3', "
+ "bootstrap.servers='localhost:9092', key='{{volume}}', " +
"@map(type='xml'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
siddhiAppRuntime.start();
//this event will be published to the cluster and create the topic
fooStream.send(new Object[]{"invalid_topic_with_partition_2", 55.6f, 100L});
//this thread sleep is to slowdown the below execution plan because, if there are no topics kafka source
// will create the topic.
Thread.sleep(500);
SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan9') " +
"define stream BarStream2 (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='invalid_topic_with_partition_3', "
+ "group.id='invalid_topic_with_partition_3_test', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092', " +
"@map(type='xml'))" +
"Define stream FooStream2 (symbol string, price float, volume long);" +
"from FooStream2 select symbol, price, volume insert into BarStream2;");
siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntimeSource.start();
fooStream.send(new Object[]{"invalid_topic_with_partition_22", 75.6f, 102L});
fooStream.send(new Object[]{"invalid_topic_with_partition_23", 57.6f, 103L});
Thread.sleep(2000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("invalid_topic_with_partition_2");
expectedNames.add("invalid_topic_with_partition_22");
expectedNames.add("invalid_topic_with_partition_23");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(100L);
expectedValues.add(102L);
expectedValues.add(103L);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
AssertJUnit.assertEquals(3, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
siddhiAppRuntimeSource.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
Example #30
| Source File: KafkaSinkTestCase.java From siddhi-io-kafka with Apache License 2.0 | 4 votes |
|
@Test(dependsOnMethods = "testPublisherWithInvalidTopicKafkaTransport")
public void testPublisherWithInvalidTopicWithPartitionKafkaTransport() throws InterruptedException {
LOG.info("Creating test for publishing events for invalid topic with a partition");
String topics[] = new String[]{"invalid_topic_with_partition"};
receivedEventNameList = new ArrayList<>(3);
receivedValueList = new ArrayList<>(3);
try {
SiddhiManager siddhiManager = new SiddhiManager();
SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan6') " +
"define stream FooStream (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@sink(type='kafka', topic='invalid_topic_with_partition', "
+ "bootstrap.servers='localhost:9092', partition.no='0', " +
"@map(type='xml'))" +
"Define stream BarStream (symbol string, price float, volume long);" +
"from FooStream select symbol, price, volume insert into BarStream;");
InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
siddhiAppRuntime.start();
//this event will be published to the cluster and create the topic
fooStream.send(new Object[]{"invalid_topic_with_partition", 55.6f, 100L});
//this thread sleep is to slowdown the below execution plan because, if there are no topics kafka source
// will create the topic.
Thread.sleep(500);
SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
"@App:name('TestExecutionPlan7') " +
"define stream BarStream2 (symbol string, price float, volume long); " +
"@info(name = 'query1') " +
"@source(type='kafka', topic.list='invalid_topic_with_partition', "
+ "group.id='invalid_topic_with_partition_test', " +
"threading.option='single.thread', bootstrap.servers='localhost:9092', "
+ "partition.no.list='0', " +
"@map(type='xml'))" +
"Define stream FooStream2 (symbol string, price float, volume long);" +
"from FooStream2 select symbol, price, volume insert into BarStream2;");
siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
@Override
public void receive(Event[] events) {
for (Event event : events) {
LOG.info(event);
eventArrived = true;
count++;
receivedEventNameList.add(event.getData(0).toString());
receivedValueList.add((long) event.getData(2));
}
}
});
siddhiAppRuntimeSource.start();
fooStream.send(new Object[]{"invalid_topic_with_partition2", 75.6f, 102L});
fooStream.send(new Object[]{"invalid_topic_with_partition3", 57.6f, 103L});
Thread.sleep(2000);
List<String> expectedNames = new ArrayList<>(2);
expectedNames.add("invalid_topic_with_partition");
expectedNames.add("invalid_topic_with_partition2");
expectedNames.add("invalid_topic_with_partition3");
List<Long> expectedValues = new ArrayList<>(2);
expectedValues.add(100L);
expectedValues.add(102L);
expectedValues.add(103L);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
receivedEventNameList);
AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
AssertJUnit.assertEquals(3, count);
KafkaTestUtil.deleteTopic(topics);
siddhiAppRuntime.shutdown();
siddhiAppRuntimeSource.shutdown();
} catch (ZkTimeoutException ex) {
LOG.warn("No zookeeper may not be available.", ex);
}
}
