Java Code Examples for org.apache.nifi.util.TestRunner#getProcessSessionFactory()

The following examples show how to use org.apache.nifi.util.TestRunner#getProcessSessionFactory() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestUpdateAttribute.java From nifi with Apache License 2.0

5 votes

@Test
public void testStateFailures() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
    final UpdateAttribute processor = (UpdateAttribute) runner.getProcessor();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    MockStateManager mockStateManager = runner.getStateManager();

    runner.setProperty(UpdateAttribute.STORE_STATE, STORE_STATE_LOCALLY);
    runner.setProperty("count", "${getStateValue('count'):plus(1)}");
    runner.setProperty("sum", "${getStateValue('sum'):plus(${pencils})}");
    runner.setProperty(UpdateAttribute.STATEFUL_VARIABLES_INIT_VALUE, "0");

    processor.onScheduled(runner.getProcessContext());

    final Map<String, String> attributes2 = new HashMap<>();
    attributes2.put("pencils", "2");

    mockStateManager.setFailOnStateGet(Scope.LOCAL, true);

    runner.enqueue(new byte[0],attributes2);
    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueNotEmpty();

    mockStateManager.setFailOnStateGet(Scope.LOCAL, false);
    mockStateManager.setFailOnStateSet(Scope.LOCAL, true);

    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueEmpty();

    runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_FAILED_SET_STATE, 1);
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("count", "1");
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("sum", "2");
}

Example 2

Source File: TestListenSyslog.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testInvalid() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    final int numMessages = 10;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // write some TCP messages to the port in the background
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, numMessages, 100, INVALID_MESSAGE));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(50);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_INVALID).size();
        }

        // all messages should be transferred to invalid
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 3

Source File: AttributeRollingWindowIT.java From nifi with Apache License 2.0

5 votes

@Test
public void testStateFailures() throws InterruptedException, IOException {
    final TestRunner runner = TestRunners.newTestRunner(AttributeRollingWindow.class);
    MockStateManager mockStateManager = runner.getStateManager();
    final AttributeRollingWindow processor = (AttributeRollingWindow) runner.getProcessor();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    runner.setProperty(AttributeRollingWindow.VALUE_TO_TRACK, "${value}");
    runner.setProperty(AttributeRollingWindow.TIME_WINDOW, "3 sec");

    processor.onScheduled(runner.getProcessContext());

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("value", "1");

    mockStateManager.setFailOnStateGet(Scope.LOCAL, true);

    runner.enqueue(new byte[0],attributes);
    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueNotEmpty();

    mockStateManager.setFailOnStateGet(Scope.LOCAL, false);
    mockStateManager.setFailOnStateSet(Scope.LOCAL, true);

    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueEmpty();

    runner.assertAllFlowFilesTransferred(AttributeRollingWindow.REL_FAILED_SET_STATE, 1);
    MockFlowFile mockFlowFile = runner.getFlowFilesForRelationship(REL_FAILED_SET_STATE).get(0);
    mockFlowFile.assertAttributeNotExists(ROLLING_WINDOW_VALUE_KEY);
    mockFlowFile.assertAttributeNotExists(ROLLING_WINDOW_COUNT_KEY);
    mockFlowFile.assertAttributeNotExists(ROLLING_WINDOW_MEAN_KEY);
}

Example 4

Source File: TestUpdateAttribute.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testStateFailures() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
    final UpdateAttribute processor = (UpdateAttribute) runner.getProcessor();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    MockStateManager mockStateManager = runner.getStateManager();

    runner.setProperty(UpdateAttribute.STORE_STATE, STORE_STATE_LOCALLY);
    runner.setProperty("count", "${getStateValue('count'):plus(1)}");
    runner.setProperty("sum", "${getStateValue('sum'):plus(${pencils})}");
    runner.setProperty(UpdateAttribute.STATEFUL_VARIABLES_INIT_VALUE, "0");

    processor.onScheduled(runner.getProcessContext());

    final Map<String, String> attributes2 = new HashMap<>();
    attributes2.put("pencils", "2");

    mockStateManager.setFailOnStateGet(Scope.LOCAL, true);

    runner.enqueue(new byte[0],attributes2);
    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueNotEmpty();

    mockStateManager.setFailOnStateGet(Scope.LOCAL, false);
    mockStateManager.setFailOnStateSet(Scope.LOCAL, true);

    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueEmpty();

    runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_FAILED_SET_STATE, 1);
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("count", "1");
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("sum", "2");
}

Example 5

Source File: ITListenGRPC.java From nifi with Apache License 2.0

5 votes

@Test
public void testOutOfSpaceRoundTrip() throws UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException {
    final int randPort = TestGRPCClient.randomPort();
    final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort);
    final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel);

    final ListenGRPC listenGRPC = new ListenGRPC();
    final TestRunner runner = TestRunners.newTestRunner(listenGRPC);
    runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort));

    final ProcessContext processContext = spy(runner.getProcessContext());
    // force the context to return that space isn't available, prompting an error message to be returned.
    when(processContext.getAvailableRelationships()).thenReturn(Sets.newHashSet());
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    try {
        // start the server. The order of the following statements shouldn't matter, because the
        // startServer() method waits for a processSessionFactory to be available to it.
        listenGRPC.startServer(processContext);
        listenGRPC.onTrigger(processContext, processSessionFactory);

        final FlowFileRequest ingestFile = FlowFileRequest.newBuilder()
                .putAttributes("FOO", "BAR")
                .setContent(ByteString.copyFrom("content".getBytes()))
                .build();
        final FlowFileReply reply = stub.send(ingestFile);
        assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.ERROR));
        assertThat(reply.getBody(), containsString("but no space available; Indicating Service Unavailable"));

        runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 0);
    } finally {
        // stop the server
        listenGRPC.stopServer(processContext);
        channel.shutdown();
    }
}

Example 6

Source File: TestUpdateAttribute.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testStateFailuresWithRulesUsingClone() throws Exception {
    final Criteria criteria = getCriteria();
    criteria.setFlowFilePolicy(FlowFilePolicy.USE_CLONE);
    addRule(criteria, "rule", Collections.singletonList(
            // conditions
            "${getStateValue('maxValue'):lt(${value})}"), getMap(
            // actions
            "maxValue", "${value}"));
    addRule(criteria, "rule2", Collections.singletonList(
            // conditions
            "${getStateValue('maxValue2'):lt(${value})}"), getMap(
            // actions
            "maxValue2", "${value}"));

    TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
    final UpdateAttribute processor = (UpdateAttribute) runner.getProcessor();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    MockStateManager mockStateManager = runner.getStateManager();

    runner.setProperty(UpdateAttribute.STORE_STATE, STORE_STATE_LOCALLY);
    runner.setProperty(UpdateAttribute.STATEFUL_VARIABLES_INIT_VALUE, "0");
    runner.setAnnotationData(serialize(criteria));


    processor.onScheduled(runner.getProcessContext());

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("value", "1");
    runner.enqueue(new byte[0], attributes);

    mockStateManager.setFailOnStateGet(Scope.LOCAL, true);
    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueNotEmpty();
    mockStateManager.setFailOnStateGet(Scope.LOCAL, false);
    mockStateManager.setFailOnStateSet(Scope.LOCAL, true);

    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueEmpty();

    runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_FAILED_SET_STATE, 1);
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("maxValue", "1");
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeNotExists("maxValue2");
}

Example 7

Source File: TestListenSyslog.java From nifi with Apache License 2.0

4 votes

@Test
public void testBatching() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.UDP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");
    runner.setProperty(ListenSyslog.MAX_BATCH_SIZE, "25");
    runner.setProperty(ListenSyslog.MESSAGE_DELIMITER, "|");
    runner.setProperty(ListenSyslog.PARSE_MESSAGES, "false");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    // the processor has internal blocking queue with capacity 10 so we have to send
    // less than that since we are sending all messages before the processors ever runs
    final int numMessages = 5;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // write some UDP messages to the port in the background
    final Thread sender = new Thread(new DatagramSender(port, numMessages, 10, VALID_MESSAGE));
    sender.setDaemon(true);
    sender.start();
    sender.join();

    try {
        proc.onTrigger(context, processSessionFactory);
        runner.assertAllFlowFilesTransferred(ListenSyslog.REL_SUCCESS, 1);

        final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        Assert.assertEquals("0", flowFile.getAttribute(SyslogAttributes.SYSLOG_PORT.key()));
        Assert.assertEquals(ListenSyslog.UDP_VALUE.getValue(), flowFile.getAttribute(SyslogAttributes.SYSLOG_PROTOCOL.key()));
        Assert.assertTrue(!StringUtils.isBlank(flowFile.getAttribute(SyslogAttributes.SYSLOG_SENDER.key())));

        final String content = new String(flowFile.toByteArray(), StandardCharsets.UTF_8);
        final String[] splits = content.split("\\|");
        Assert.assertEquals(numMessages, splits.length);

        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(1, events.size());

        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("udp"));
    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 8

Source File: ITListenGRPC.java From nifi with Apache License 2.0

4 votes

@Test
public void testSecureTwoWaySSLPassAuthorizedDNCheck() throws UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException {
    final int randPort = TestGRPCClient.randomPort();
    final Map<String, String> sslProperties = getKeystoreProperties();
    sslProperties.putAll(getTruststoreProperties());
    final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort, sslProperties);
    final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel);

    final ListenGRPC listenGRPC = new ListenGRPC();
    final TestRunner runner = TestRunners.newTestRunner(listenGRPC);
    runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort));
    runner.setProperty(ListenGRPC.PROP_USE_SECURE, "true");
    runner.setProperty(ListenGRPC.PROP_AUTHORIZED_DN_PATTERN, "CN=localhost.*");
    useSSLContextService(runner, sslProperties);

    final ProcessContext processContext = runner.getProcessContext();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    try {
        // start the server. The order of the following statements shouldn't matter, because the
        // startServer() method waits for a processSessionFactory to be available to it.
        listenGRPC.startServer(processContext);
        listenGRPC.onTrigger(processContext, processSessionFactory);


        final FlowFileRequest ingestFile = FlowFileRequest.newBuilder()
                .putAttributes("FOO", "BAR")
                .setContent(ByteString.copyFrom("content".getBytes()))
                .build();
        final FlowFileReply reply = stub.send(ingestFile);
        assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS));
        assertThat(reply.getBody(), equalTo("FlowFile successfully received."));

        runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1);
        final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS);
        assertThat(successFiles.size(), equalTo(1));
        final MockFlowFile mockFlowFile = successFiles.get(0);
        assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_HOST), equalTo("127.0.0.1"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_USER_DN), equalTo(CERT_DN));

    } finally {
        // stop the server
        listenGRPC.stopServer(processContext);
        channel.shutdown();
    }
}

Example 9

Source File: ITListenGRPC.java From nifi with Apache License 2.0

4 votes

@Test(expected = io.grpc.StatusRuntimeException.class)
public void testSecureTwoWaySSLFailAuthorizedDNCheck() throws UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException {
    final int randPort = TestGRPCClient.randomPort();
    final Map<String, String> sslProperties = getKeystoreProperties();
    sslProperties.putAll(getTruststoreProperties());
    final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort, sslProperties);
    final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel);

    final ListenGRPC listenGRPC = new ListenGRPC();
    final TestRunner runner = TestRunners.newTestRunner(listenGRPC);
    runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort));
    runner.setProperty(ListenGRPC.PROP_USE_SECURE, "true");
    runner.setProperty(ListenGRPC.PROP_AUTHORIZED_DN_PATTERN, "CN=FAKE.*");
    useSSLContextService(runner, sslProperties);

    final ProcessContext processContext = runner.getProcessContext();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    try {
        // start the server. The order of the following statements shouldn't matter, because the
        // startServer() method waits for a processSessionFactory to be available to it.
        listenGRPC.startServer(processContext);
        listenGRPC.onTrigger(processContext, processSessionFactory);


        final FlowFileRequest ingestFile = FlowFileRequest.newBuilder()
                .putAttributes("FOO", "BAR")
                .setContent(ByteString.copyFrom("content".getBytes()))
                .build();
        final FlowFileReply reply = stub.send(ingestFile);
        assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS));
        assertThat(reply.getBody(), equalTo("FlowFile successfully received."));

        runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1);
        final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS);
        assertThat(successFiles.size(), equalTo(1));
        final MockFlowFile mockFlowFile = successFiles.get(0);
        assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_HOST), equalTo("127.0.0.1"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_USER_DN), equalTo(CERT_DN));

    } finally {
        // stop the server
        listenGRPC.stopServer(processContext);
        channel.shutdown();
    }
}

Example 10

Source File: ITListenGRPC.java From nifi with Apache License 2.0

4 votes

@Test
public void testSecureOneWaySSL() throws UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException, InterruptedException {
    final int randPort = TestGRPCClient.randomPort();
    final Map<String, String> sslProperties = getTruststoreProperties();
    final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort, sslProperties);
    final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel);

    final ListenGRPC listenGRPC = new ListenGRPC();
    final TestRunner runner = TestRunners.newTestRunner(listenGRPC);
    runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort));
    runner.setProperty(ListenGRPC.PROP_USE_SECURE, "true");
    useSSLContextService(runner, getKeystoreProperties());

    final ProcessContext processContext = runner.getProcessContext();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    try {
        // start the server. The order of the following statements shouldn't matter, because the
        // startServer() method waits for a processSessionFactory to be available to it.
        listenGRPC.startServer(processContext);
        listenGRPC.onTrigger(processContext, processSessionFactory);


        final FlowFileRequest ingestFile = FlowFileRequest.newBuilder()
                .putAttributes("FOO", "BAR")
                .setContent(ByteString.copyFrom("content".getBytes()))
                .build();
        final FlowFileReply reply = stub.send(ingestFile);
        assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS));
        assertThat(reply.getBody(), equalTo("FlowFile successfully received."));

        // known race condition spot: grpc reply vs flowfile transfer
        Thread.sleep(10);
        runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1);
        final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS);
        assertThat(successFiles.size(), equalTo(1));
        final MockFlowFile mockFlowFile = successFiles.get(0);
        assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_HOST), equalTo("127.0.0.1"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_USER_DN), equalTo(FlowFileIngestServiceInterceptor.DEFAULT_FOUND_SUBJECT));

    } finally {
        // stop the server
        listenGRPC.stopServer(processContext);
        channel.shutdown();
    }
}

Example 11

Source File: ITListenGRPC.java From nifi with Apache License 2.0

4 votes

@Test
public void testSecureTwoWaySSL() throws UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException {
    final int randPort = TestGRPCClient.randomPort();
    final Map<String, String> sslProperties = getKeystoreProperties();
    sslProperties.putAll(getTruststoreProperties());
    final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort, sslProperties);
    final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel);

    final ListenGRPC listenGRPC = new ListenGRPC();
    final TestRunner runner = TestRunners.newTestRunner(listenGRPC);
    runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort));
    runner.setProperty(ListenGRPC.PROP_USE_SECURE, "true");
    useSSLContextService(runner, sslProperties);

    final ProcessContext processContext = runner.getProcessContext();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    try {
        // start the server. The order of the following statements shouldn't matter, because the
        // startServer() method waits for a processSessionFactory to be available to it.
        listenGRPC.startServer(processContext);
        listenGRPC.onTrigger(processContext, processSessionFactory);


        final FlowFileRequest ingestFile = FlowFileRequest.newBuilder()
                .putAttributes("FOO", "BAR")
                .setContent(ByteString.copyFrom("content".getBytes()))
                .build();
        final FlowFileReply reply = stub.send(ingestFile);
        assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS));
        assertThat(reply.getBody(), equalTo("FlowFile successfully received."));

        runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1);
        final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS);
        assertThat(successFiles.size(), equalTo(1));
        final MockFlowFile mockFlowFile = successFiles.get(0);
        assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_HOST), equalTo("127.0.0.1"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_USER_DN), equalTo(CERT_DN));

    } finally {
        // stop the server
        listenGRPC.stopServer(processContext);
        channel.shutdown();
    }
}

Example 12

Source File: ITListenGRPC.java From nifi with Apache License 2.0

4 votes

@Test(expected = io.grpc.StatusRuntimeException.class)
public void testExceedMaxMessageSize() throws UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException {
    final int randPort = TestGRPCClient.randomPort();
    final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort);
    final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel);

    final ListenGRPC listenGRPC = new ListenGRPC();
    final TestRunner runner = TestRunners.newTestRunner(listenGRPC);
    runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort));
    // set max message size to 1 byte to force exception to be thrown.
    runner.setProperty(ListenGRPC.PROP_MAX_MESSAGE_SIZE, "1B");

    final ProcessContext processContext = runner.getProcessContext();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    try {
        // start the server. The order of the following statements shouldn't matter, because the
        // startServer() method waits for a processSessionFactory to be available to it.
        listenGRPC.startServer(processContext);
        listenGRPC.onTrigger(processContext, processSessionFactory);

        final FlowFileRequest ingestFile = FlowFileRequest.newBuilder()
                .putAttributes("FOO", "BAR")
                .putAttributes(CoreAttributes.UUID.key(), SOURCE_SYSTEM_UUID)
                .setContent(ByteString.copyFrom("content".getBytes()))
                .build();
        // this should throw a runtime exception
        final FlowFileReply reply = stub.send(ingestFile);
        assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS));
        assertThat(reply.getBody(), equalTo("FlowFile successfully received."));

        runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1);
        final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS);
        assertThat(successFiles.size(), equalTo(1));
        final MockFlowFile mockFlowFile = successFiles.get(0);
        assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_HOST), equalTo("127.0.0.1"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_USER_DN), equalTo(FlowFileIngestServiceInterceptor.DEFAULT_FOUND_SUBJECT));

    } finally {
        // stop the server
        listenGRPC.stopServer(processContext);
        channel.shutdown();
    }
}

Example 13

Source File: ITListenGRPC.java From nifi with Apache License 2.0

4 votes

@Test
public void testSuccessfulRoundTrip() throws UnrecoverableKeyException, CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException {
    final int randPort = TestGRPCClient.randomPort();
    final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort);
    final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel);

    final ListenGRPC listenGRPC = new ListenGRPC();
    final TestRunner runner = TestRunners.newTestRunner(listenGRPC);
    runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort));

    final ProcessContext processContext = runner.getProcessContext();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();

    try {
        // start the server. The order of the following statements shouldn't matter, because the
        // startServer() method waits for a processSessionFactory to be available to it.
        listenGRPC.startServer(processContext);
        listenGRPC.onTrigger(processContext, processSessionFactory);

        final FlowFileRequest ingestFile = FlowFileRequest.newBuilder()
                .putAttributes("FOO", "BAR")
                .putAttributes(CoreAttributes.UUID.key(), SOURCE_SYSTEM_UUID)
                .setContent(ByteString.copyFrom("content".getBytes()))
                .build();
        final FlowFileReply reply = stub.send(ingestFile);
        assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS));
        assertThat(reply.getBody(), equalTo("FlowFile successfully received."));

        runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1);
        final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS);
        assertThat(successFiles.size(), equalTo(1));
        final MockFlowFile mockFlowFile = successFiles.get(0);
        assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_HOST), equalTo("127.0.0.1"));
        assertThat(mockFlowFile.getAttribute(ListenGRPC.REMOTE_USER_DN), equalTo(FlowFileIngestServiceInterceptor.DEFAULT_FOUND_SUBJECT));

    } finally {
        // stop the server
        listenGRPC.stopServer(processContext);
        channel.shutdown();
    }
}

Example 14

Source File: TestUpdateAttribute.java From nifi with Apache License 2.0

4 votes

@Test
public void testStateFailuresWithRulesUsingOriginal() throws Exception {
    final Criteria criteria = getCriteria();
    criteria.setFlowFilePolicy(FlowFilePolicy.USE_ORIGINAL);
    addRule(criteria, "rule", Collections.singletonList(
            // conditions
            "${getStateValue('maxValue'):lt(${value})}"), getMap(
            // actions
            "maxValue", "${value}"));
    addRule(criteria, "rule2", Collections.singletonList(
            // conditions
            "${getStateValue('maxValue2'):lt(${value})}"), getMap(
            // actions
            "maxValue2", "${value}"));

    TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
    final UpdateAttribute processor = (UpdateAttribute) runner.getProcessor();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    MockStateManager mockStateManager = runner.getStateManager();

    runner.setProperty(UpdateAttribute.STORE_STATE, STORE_STATE_LOCALLY);
    runner.setProperty(UpdateAttribute.STATEFUL_VARIABLES_INIT_VALUE, "0");
    runner.setAnnotationData(serialize(criteria));


    processor.onScheduled(runner.getProcessContext());

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("value", "1");
    runner.enqueue(new byte[0], attributes);

    mockStateManager.setFailOnStateGet(Scope.LOCAL, true);
    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueNotEmpty();
    mockStateManager.setFailOnStateGet(Scope.LOCAL, false);
    mockStateManager.setFailOnStateSet(Scope.LOCAL, true);

    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueEmpty();

    runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_FAILED_SET_STATE, 1);
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("maxValue", "1");
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("maxValue2", "1");
}

Example 15

Source File: TestUpdateAttribute.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testStateFailuresWithRulesUsingOriginal() throws Exception {
    final Criteria criteria = getCriteria();
    criteria.setFlowFilePolicy(FlowFilePolicy.USE_ORIGINAL);
    addRule(criteria, "rule", Collections.singletonList(
            // conditions
            "${getStateValue('maxValue'):lt(${value})}"), getMap(
            // actions
            "maxValue", "${value}"));
    addRule(criteria, "rule2", Collections.singletonList(
            // conditions
            "${getStateValue('maxValue2'):lt(${value})}"), getMap(
            // actions
            "maxValue2", "${value}"));

    TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute());
    final UpdateAttribute processor = (UpdateAttribute) runner.getProcessor();
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    MockStateManager mockStateManager = runner.getStateManager();

    runner.setProperty(UpdateAttribute.STORE_STATE, STORE_STATE_LOCALLY);
    runner.setProperty(UpdateAttribute.STATEFUL_VARIABLES_INIT_VALUE, "0");
    runner.setAnnotationData(serialize(criteria));


    processor.onScheduled(runner.getProcessContext());

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("value", "1");
    runner.enqueue(new byte[0], attributes);

    mockStateManager.setFailOnStateGet(Scope.LOCAL, true);
    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueNotEmpty();
    mockStateManager.setFailOnStateGet(Scope.LOCAL, false);
    mockStateManager.setFailOnStateSet(Scope.LOCAL, true);

    processor.onTrigger(runner.getProcessContext(), processSessionFactory.createSession());

    runner.assertQueueEmpty();

    runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_FAILED_SET_STATE, 1);
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("maxValue", "1");
    runner.getFlowFilesForRelationship(UpdateAttribute.REL_FAILED_SET_STATE).get(0).assertAttributeEquals("maxValue2", "1");
}

Example 16

Source File: ITListenSyslog.java From nifi with Apache License 2.0

4 votes

@Test
public void testTCPSingleConnectionWithNewLines() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    final int numMessages = 3;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // send 3 messages as 1
    final String multipleMessages = VALID_MESSAGE_TCP + "\n" + VALID_MESSAGE_TCP + "\n" + VALID_MESSAGE_TCP + "\n";
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, 1, 10, multipleMessages));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);

        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.TCP_VALUE.getValue());

        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());

        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("tcp"));

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 17

Source File: ITListenSyslog.java From nifi with Apache License 2.0

4 votes

@Test
public void testTCPSingleConnection() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    // Allow time for the processor to perform its scheduled start
    Thread.sleep(500);

    final int numMessages = 20;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // write some TCP messages to the port in the background
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, numMessages, 10, VALID_MESSAGE_TCP));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);

        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.TCP_VALUE.getValue());

        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());

        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("tcp"));

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 18

Source File: ITListenSyslog.java From nifi with Apache License 2.0

4 votes

@Test
public void testUDP() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.UDP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    final int numMessages = 20;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // write some UDP messages to the port in the background
    final Thread sender = new Thread(new DatagramSender(port, numMessages, 10, VALID_MESSAGE));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all datagrams, or 30 seconds passed
    try {
        int numTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (numTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            numTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the datagrams", numMessages, numTransferred);

        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.UDP_VALUE.getValue());

        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());

        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("udp"));

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 19

Source File: TestListenSyslog.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testTCPSingleConnectionWithNewLines() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    final int numMessages = 3;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // send 3 messages as 1
    final String multipleMessages = VALID_MESSAGE_TCP + "\n" + VALID_MESSAGE_TCP + "\n" + VALID_MESSAGE_TCP + "\n";
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, 1, 10, multipleMessages));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);

        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.TCP_VALUE.getValue());

        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());

        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("tcp"));

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 20

Source File: TestListenSyslog.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testTCPSingleConnection() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");

    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);

    // Allow time for the processor to perform its scheduled start
    Thread.sleep(500);

    final int numMessages = 20;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);

    // write some TCP messages to the port in the background
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, numMessages, 10, VALID_MESSAGE_TCP));
    sender.setDaemon(true);
    sender.start();

    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;

        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);

        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.TCP_VALUE.getValue());

        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());

        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("tcp"));

    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}