Java Code Examples for org.apache.nifi.util.MockFlowFile

The following examples show how to use org.apache.nifi.util.MockFlowFile. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: nifi   Source File: TestPutKinesisStream.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWithSizeGreaterThan1MB() {
    runner.setProperty(PutKinesisStream.BATCH_SIZE, "1");
    runner.assertValid();
    byte [] bytes = new byte[(PutKinesisStream.MAX_MESSAGE_SIZE + 1)];
    for (int i = 0; i < bytes.length; i++) {
        bytes[i] = 'a';
    }
    runner.enqueue(bytes);
    runner.run(1);

    runner.assertAllFlowFilesTransferred(PutKinesisStream.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutKinesisStream.REL_FAILURE);

    assertNotNull(flowFiles.get(0).getAttribute(PutKinesisStream.AWS_KINESIS_ERROR_MESSAGE));
}
 
Example 2
Source Project: nifi   Source File: TestFetchElasticsearchHttp.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFetchElasticsearchOnTriggerWithDocNotFound() throws IOException {
    runner = TestRunners.newTestRunner(new FetchElasticsearchHttpTestProcessor(false)); // simulate doc not found
    runner.setProperty(AbstractElasticsearchHttpProcessor.ES_URL, "http://127.0.0.1:9200");
    runner.setProperty(FetchElasticsearchHttp.INDEX, "doc");
    runner.setProperty(FetchElasticsearchHttp.DOC_ID, "${doc_id}");

    runner.setIncomingConnection(true);
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});
    runner.run(1, true, true);

    // This test generates a "document not found"
    runner.assertAllFlowFilesTransferred(FetchElasticsearchHttp.REL_NOT_FOUND, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(FetchElasticsearchHttp.REL_NOT_FOUND).get(0);
    assertNotNull(out);
    out.assertAttributeEquals("doc_id", "28039652140");
}
 
Example 3
@Test
public void testPutElasticSearchOnTriggerBadIndexOp() throws IOException {
    runner = TestRunners.newTestRunner(new PutElasticsearchTestProcessor(false)); // no failures
    runner.setValidateExpressionUsage(true);
    runner.setProperty(AbstractElasticsearchHttpProcessor.ES_URL, "http://127.0.0.1:9200");

    runner.setProperty(PutElasticsearchHttp.INDEX, "doc");
    runner.setProperty(PutElasticsearchHttp.TYPE, "status");
    runner.setProperty(PutElasticsearchHttp.BATCH_SIZE, "1");
    runner.setProperty(PutElasticsearchHttp.ID_ATTRIBUTE, "doc_id");
    runner.setProperty(PutElasticsearchHttp.INDEX_OP, "${no.attr}");
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(PutElasticsearchHttp.REL_FAILURE, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(PutElasticsearchHttp.REL_FAILURE).get(0);
    assertNotNull(out);
    out.assertAttributeEquals("doc_id", "28039652140");
}
 
Example 4
Source Project: nifi   Source File: PutGCSObjectTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAclAttributeProject() throws Exception {
    reset(storage, blob);
    final PutGCSObject processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.create(any(BlobInfo.class), any(InputStream.class), any(Storage.BlobWriteOption.class)))
            .thenReturn(blob);

    final Acl.Project mockProject = mock(Acl.Project.class);
    when(mockProject.getProjectId()).thenReturn(OWNER_PROJECT_ID);
    when(blob.getOwner()).thenReturn(mockProject);

    runner.enqueue("test");
    runner.run();

    runner.assertAllFlowFilesTransferred(PutGCSObject.REL_SUCCESS);
    runner.assertTransferCount(PutGCSObject.REL_SUCCESS, 1);

    final MockFlowFile mockFlowFile = runner.getFlowFilesForRelationship(PutGCSObject.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(OWNER_ATTR, OWNER_PROJECT_ID);
    mockFlowFile.assertAttributeEquals(OWNER_TYPE_ATTR, "project");
}
 
Example 5
Source Project: nifi   Source File: TestJoltTransformRecord.java    License: Apache License 2.0 6 votes vote down vote up
@Test
    public void testTransformInputWithDefaultrExpressionLanguage() throws IOException {
        generateTestData(1, null);
        final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformRecord/defaultrELOutputSchema.avsc")));
        runner.setProperty(writer, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
        runner.setProperty(writer, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
        runner.setProperty(writer, "Pretty Print JSON", "true");
        runner.enableControllerService(writer);
        final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformRecord/defaultrELSpec.json")));
        runner.setProperty(JoltTransformRecord.JOLT_SPEC, spec);
        runner.setProperty(JoltTransformRecord.JOLT_TRANSFORM, JoltTransformRecord.DEFAULTR);
        runner.setVariable("quota", "5");
        runner.enqueue(new byte[0]);
        runner.run();
        runner.assertTransferCount(JoltTransformRecord.REL_SUCCESS, 1);
runner.assertTransferCount(JoltTransformRecord.REL_ORIGINAL, 1);
        final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformRecord.REL_SUCCESS).get(0);
        assertEquals(new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformRecord/defaultrELOutput.json"))),
                new String(transformed.toByteArray()));

    }
 
Example 6
Source Project: localization_nifi   Source File: TestExtractText.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIgnoreZeroCaptureGroupProperty() throws Exception {
    final TestRunner testRunner = TestRunners.newTestRunner(new ExtractText());

    testRunner.setProperty(ExtractText.INCLUDE_CAPTURE_GROUP_ZERO, "false");

    final String attributeKey = "regex.result";

    testRunner.setProperty(attributeKey, "(?s)(.*)");

    testRunner.enqueue(SAMPLE_STRING.getBytes("UTF-8"));
    testRunner.run();

    testRunner.assertAllFlowFilesTransferred(ExtractText.REL_MATCH, 1);
    final MockFlowFile out = testRunner.getFlowFilesForRelationship(ExtractText.REL_MATCH).get(0);

    // Ensure the zero capture group is not in the resultant attributes
    out.assertAttributeNotExists(attributeKey + ".0");
    out.assertAttributeEquals(attributeKey, SAMPLE_STRING);
}
 
Example 7
Source Project: nifi   Source File: TestInferAvroSchema.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void inferAvroSchemaFromHeaderDefinitionOfCSVFile() throws Exception {

    runner.assertValid();

    Map<String, String> attributes = new HashMap<>();
    attributes.put(CoreAttributes.MIME_TYPE.key(), "text/csv");
    runner.enqueue(new File("src/test/resources/Shapes_Header.csv").toPath(), attributes);

    runner.run();
    runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0);
    runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0);
    runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1);
    runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1);

    MockFlowFile flowFile = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0);
    flowFile.assertContentEquals(unix2PlatformSpecificLineEndings(new File("src/test/resources/Shapes_header.csv.avro")));
    flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/avro-binary");
}
 
Example 8
Source Project: nifi   Source File: TestXMLReader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecordFormat2() throws IOException, InitializationException {
    TestRunner runner = setup("src/test/resources/xml/testschema");

    runner.setProperty(reader, XMLReader.RECORD_FORMAT, XMLReader.RECORD_ARRAY);

    runner.enableControllerService(reader);

    InputStream is = new FileInputStream("src/test/resources/xml/people.xml");
    runner.enqueue(is, Collections.singletonMap(EVALUATE_IS_ARRAY, "true"));
    runner.run();

    List<MockFlowFile> flowFile = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS);
    List<String> records = Arrays.asList((new String(runner.getContentAsByteArray(flowFile.get(0)))).split("\n"));

    assertEquals(4, records.size());
}
 
Example 9
Source Project: nifi   Source File: TestSplitAvro.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecordSplitDatafileOutputWithMultipleRecords() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new SplitAvro());
    runner.setProperty(SplitAvro.OUTPUT_SIZE, "20");

    runner.enqueue(users.toByteArray());
    runner.run();

    runner.assertTransferCount(SplitAvro.REL_SPLIT, 5);
    runner.assertTransferCount(SplitAvro.REL_ORIGINAL, 1);
    runner.assertTransferCount(SplitAvro.REL_FAILURE, 0);

    runner.getFlowFilesForRelationship(SplitAvro.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT.key(), "5");
    final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(SplitAvro.REL_SPLIT);
    checkDataFileSplitSize(flowFiles, 20, true);
}
 
Example 10
Source Project: nifi   Source File: TestInferAvroSchema.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void inferAvroSchemaFromHeaderDefinitionOfCSVTabDelimitedFile() throws Exception {

    runner.setProperty(InferAvroSchema.DELIMITER, "\\t");
    runner.assertValid();

    Map<String, String> attributes = new HashMap<>();
    attributes.put(CoreAttributes.MIME_TYPE.key(), "text/csv");
    runner.enqueue(new File("src/test/resources/Shapes_Header_TabDelimited.csv").toPath(), attributes);

    runner.run();
    runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0);
    runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0);
    runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1);
    runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1);

    MockFlowFile flowFile = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0);
    flowFile.assertContentEquals(unix2PlatformSpecificLineEndings(new File("src/test/resources/Shapes_header.csv.avro")));
    flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/avro-binary");
}
 
Example 11
Source Project: localization_nifi   Source File: TestInvokeGroovy.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests a script that has a Groovy Processor that that reads the first line of text from the flowfiles content and stores the value in an attribute of the outgoing flowfile.
 *
 * @throws Exception Any error encountered while testing
 */
@Test
public void testReadFlowFileContentAndStoreInFlowFileAttribute() throws Exception {
    runner.setValidateExpressionUsage(false);
    runner.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE, "Groovy");
    runner.setProperty(ScriptingComponentUtils.SCRIPT_FILE, "target/test/resources/groovy/test_reader.groovy");
    runner.setProperty(ScriptingComponentUtils.MODULES, "target/test/resources/groovy");

    runner.assertValid();
    runner.enqueue("test content".getBytes(StandardCharsets.UTF_8));
    runner.run();

    runner.assertAllFlowFilesTransferred("test", 1);
    final List<MockFlowFile> result = runner.getFlowFilesForRelationship("test");
    result.get(0).assertAttributeEquals("from-content", "test content");
}
 
Example 12
Source Project: nifi   Source File: TestCSVToAvroProcessor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(FAILURE_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals("Should set an error message",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 13
Source Project: nifi   Source File: TestReplaceText.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBackReferenceWithTooLargeOfIndexIsEscapedExceptLastLine() throws IOException {
    final TestRunner runner = getRunner();
    runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
    runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_LAST_LINE);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "(O)");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1$2");

    final Map<String, String> attributes = new HashMap<>();
    runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/ReplaceExceptLastLine.txt")));
}
 
Example 14
Source Project: nifi   Source File: TestSelectHive_1_1QL.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWithBadSQL() throws SQLException {
    final String BAD_SQL = "create table TEST_NO_ROWS (id integer)";

    // Test with incoming flow file (it should be routed to failure intact, i.e. same content and no parent)
    runner.setIncomingConnection(true);
    // Try a valid SQL statement that will generate an error (val1 does not exist, e.g.)
    runner.enqueue(BAD_SQL);
    runner.run();
    runner.assertAllFlowFilesTransferred(SelectHive_1_1QL.REL_FAILURE, 1);
    MockFlowFile flowFile = runner.getFlowFilesForRelationship(SelectHive_1_1QL.REL_FAILURE).get(0);
    flowFile.assertContentEquals(BAD_SQL);
    flowFile.assertAttributeEquals("parentIds", null);
    runner.clearTransferState();

    // Test with no incoming flow file (an empty flow file is transferred)
    runner.setIncomingConnection(false);
    // Try a valid SQL statement that will generate an error (val1 does not exist, e.g.)
    runner.setProperty(SelectHive_1_1QL.HIVEQL_SELECT_QUERY, BAD_SQL);
    runner.run();
    runner.assertAllFlowFilesTransferred(SelectHive_1_1QL.REL_FAILURE, 1);
    flowFile = runner.getFlowFilesForRelationship(SelectHive_1_1QL.REL_FAILURE).get(0);
    flowFile.assertContentEquals("");
}
 
Example 15
Source Project: localization_nifi   Source File: ITPutKinesisFirehose.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test5MessageWithBatch2MaxBufferSize10MBRunOnce2MessageSent() {
    runner = TestRunners.newTestRunner(PutKinesisFirehose.class);
    runner.setProperty(PutKinesisFirehose.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutKinesisFirehose.BATCH_SIZE, "2");
    runner.setProperty(PutKinesisFirehose.MAX_MESSAGE_BUFFER_SIZE_MB, "1 MB");
    runner.setProperty(PutKinesisFirehose.KINESIS_FIREHOSE_DELIVERY_STREAM_NAME, "testkinesis");
    runner.assertValid();
    byte [] bytes = new byte[10];
    for (int i = 0; i < bytes.length; i++) {
        bytes[i] = 'a';
    }
    runner.enqueue(bytes);
    runner.enqueue(bytes.clone());
    runner.enqueue(bytes.clone());
    runner.enqueue(bytes);
    runner.enqueue(bytes.clone());
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(PutKinesisFirehose.REL_SUCCESS, 2);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutKinesisFirehose.REL_SUCCESS);
    assertEquals(2,flowFiles.size());
    for (MockFlowFile flowFile : flowFiles) {
        flowFile.assertAttributeExists(PutKinesisFirehose.AWS_KINESIS_FIREHOSE_RECORD_ID);
    }
}
 
Example 16
Source Project: localization_nifi   Source File: TestCSVToAvroProcessor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(FAILURE_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals("Should set an error message",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 17
Source Project: nifi   Source File: TestLogAttribute.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLogPropertyRegexWithIgnoreRegex() {
    final LogAttribute logAttribute = new LogAttribute();
    final TestRunner runner = TestRunners.newTestRunner(logAttribute);
    final ProcessContext context = runner.getProcessContext();
    final ProcessSession session = runner.getProcessSessionFactory().createSession();
    final MockComponentLog LOG = runner.getLogger();

    runner.setProperty(LogAttribute.ATTRIBUTES_TO_LOG_REGEX, "foo.*"); // includes foo,foobaz
    runner.setProperty(LogAttribute.ATTRIBUTES_TO_IGNORE_REGEX, "foobaz.*"); // includes foobaz

    final Map<String,String> attrs = Maps.newHashMap();
    attrs.put("foo", "foo-value");
    attrs.put("bar", "bar-value");
    attrs.put("foobaz", "foobaz-value");

    final MockFlowFile flowFile = runner.enqueue("content", attrs);

    final String logMessage = logAttribute.processFlowFile(LOG, LogAttribute.DebugLevels.info, flowFile, session, context);
    assertThat(logMessage, not(containsString("foobaz-value")));
    assertThat(logMessage, containsString("foo-value"));
    assertThat(logMessage, not(containsString("bar-value")));
}
 
Example 18
Source Project: nifi   Source File: TestEvaluateJsonPath.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleAsciiControlCharacters() throws Exception {
    final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
    testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_ATTRIBUTE);
    testRunner.setProperty(EvaluateJsonPath.RETURN_TYPE, EvaluateJsonPath.RETURN_TYPE_JSON);

    final String jsonPathControlCharKey = "evaluatejson.controlcharacterpath";

    testRunner.setProperty(jsonPathControlCharKey, "$.jinxing_json.object.property");

    testRunner.enqueue(Paths.get("src/test/resources/TestJson/control-characters.json"));
    testRunner.run();

    final Relationship expectedRel = EvaluateJsonPath.REL_MATCH;

    testRunner.assertAllFlowFilesTransferred(expectedRel, 1);
    final MockFlowFile out = testRunner.getFlowFilesForRelationship(expectedRel).get(0);
    Assert.assertNotNull("Transferred flow file did not have the correct result for id attribute", out.getAttribute(jsonPathControlCharKey));
}
 
Example 19
Source Project: nifi   Source File: TestGetCouchbaseKey.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPutToAttributeNoTargetAttribute() throws Exception {

    Bucket bucket = mock(Bucket.class);
    String inFileDataStr = "doc-in";
    String content = "some-value";
    when(bucket.get(inFileDataStr, RawJsonDocument.class))
        .thenReturn(RawJsonDocument.create(inFileDataStr, content));
    setupMockBucket(bucket);

    byte[] inFileData = inFileDataStr.getBytes(StandardCharsets.UTF_8);
    testRunner.setProperty(PUT_VALUE_TO_ATTRIBUTE, "${expressionReturningNoValue}");
    testRunner.enqueue(inFileData);
    testRunner.run();

    testRunner.assertTransferCount(REL_SUCCESS, 0);
    testRunner.assertTransferCount(REL_ORIGINAL, 0);
    testRunner.assertTransferCount(REL_RETRY, 0);
    testRunner.assertTransferCount(REL_FAILURE, 1);
    MockFlowFile outFile = testRunner.getFlowFilesForRelationship(REL_FAILURE).get(0);
    outFile.assertContentEquals(inFileDataStr);
}
 
Example 20
Source Project: nifi   Source File: TestRouteText.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleCaseInsensitiveStartsMatch() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new RouteText());
    runner.setProperty(RouteText.MATCH_STRATEGY, RouteText.STARTS_WITH);
    runner.setProperty(RouteText.IGNORE_CASE, "true");
    runner.setProperty("simple", "start");

    runner.enqueue("start middle end\nSTart middle end".getBytes("UTF-8"));
    runner.run();

    runner.assertTransferCount("simple", 1);
    runner.assertTransferCount("unmatched", 0);
    runner.assertTransferCount("original", 1);
    final MockFlowFile outMatched = runner.getFlowFilesForRelationship("simple").get(0);
    outMatched.assertContentEquals("start middle end\nSTart middle end".getBytes("UTF-8"));
}
 
Example 21
Source Project: localization_nifi   Source File: TestGetHDFSEvents.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldProperlyHandleAnEmptyEventBatch() throws Exception {
    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(new Event[]{});

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path");
    runner.setProperty(GetHDFSEvents.NUMBER_OF_RETRIES_FOR_POLL, "5");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(0, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example 22
Source Project: nifi   Source File: TestLogAttribute.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLogPropertyWithIgnoreCSV() {
    final LogAttribute logAttribute = new LogAttribute();
    final TestRunner runner = TestRunners.newTestRunner(logAttribute);
    final ProcessContext context = runner.getProcessContext();
    final ProcessSession session = runner.getProcessSessionFactory().createSession();
    final MockComponentLog LOG = runner.getLogger();

    runner.setProperty(LogAttribute.ATTRIBUTES_TO_IGNORE_CSV, "bar");

    final Map<String,String> attrs = Maps.newHashMap();
    attrs.put("foo", "foo-value");
    attrs.put("bar", "bar-value");
    attrs.put("foobaz", "foobaz-value");

    final MockFlowFile flowFile = runner.enqueue("content", attrs);

    final String logMessage = logAttribute.processFlowFile(LOG, LogAttribute.DebugLevels.info, flowFile, session, context);
    assertThat(logMessage, containsString("foobaz-value"));
    assertThat(logMessage, containsString("foo-value"));
    assertThat(logMessage, not(containsString("bar-value")));
}
 
Example 23
Source Project: nifi   Source File: TestInFlightMessageTracker.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000L)
public void testAwaitCompletionWhenComplete() throws InterruptedException, TimeoutException {
    final MockFlowFile flowFile = new MockFlowFile(1L);

    final InFlightMessageTracker tracker = new InFlightMessageTracker();
    tracker.incrementSentCount(flowFile);

    verifyNotComplete(tracker);

    tracker.incrementSentCount(flowFile);
    verifyNotComplete(tracker);

    tracker.incrementAcknowledgedCount(flowFile);
    verifyNotComplete(tracker);

    tracker.incrementAcknowledgedCount(flowFile);
    tracker.awaitCompletion(1L);
}
 
Example 24
@Test
public void testWithMatchingGroupAndContext() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceTextWithMapping());
    runner.setProperty(ReplaceTextWithMapping.REGEX, "-(.*?)-");
    runner.setProperty(ReplaceTextWithMapping.MATCHING_GROUP_FOR_LOOKUP_KEY, "1");
    runner.setProperty(ReplaceTextWithMapping.MAPPING_FILE, Paths.get("src/test/resources/TestReplaceTextWithMapping/color-fruit-mapping.txt").toFile().getAbsolutePath());

    runner.enqueue(Paths.get("src/test/resources/TestReplaceTextWithMapping/colors.txt"));
    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceTextWithMapping.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceTextWithMapping.REL_SUCCESS).get(0);
    String outputString = new String(out.toByteArray());
    String expected = "-roses- are -apple-\n"
            + "violets are -blueberry-\n"
            + "something else is -grape-\n"
            + "I'm not good at writing poems";
    assertEquals(expected, outputString);
}
 
Example 25
Source Project: localization_nifi   Source File: TestFetchHDFS.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAutomaticDecompression() throws IOException {
    FetchHDFS proc = new TestableFetchHDFS(kerberosProperties);
    TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(FetchHDFS.FILENAME, "src/test/resources/testdata/randombytes-1.gz");
    runner.setProperty(FetchHDFS.COMPRESSION_CODEC, "AUTOMATIC");
    runner.enqueue(new String("trigger flow file"));
    runner.run();

    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(FetchHDFS.REL_SUCCESS);
    assertEquals(1, flowFiles.size());

    MockFlowFile flowFile = flowFiles.get(0);
    assertTrue(flowFile.getAttribute(CoreAttributes.FILENAME.key()).equals("randombytes-1"));
    InputStream expected = getClass().getResourceAsStream("/testdata/randombytes-1");
    flowFile.assertContentEquals(expected);
}
 
Example 26
Source Project: nifi   Source File: TestSplitText.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveLineSplitWithHeaderNotRetainNewline() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new SplitText());
    runner.setProperty(SplitText.HEADER_LINE_COUNT, "1");
    runner.setProperty(SplitText.LINE_SPLIT_COUNT, "5");
    runner.setProperty(SplitText.REMOVE_TRAILING_NEWLINES, "true");

    runner.enqueue(TEST_INPUT_DATA);
    runner.run();

    runner.assertTransferCount(SplitText.REL_FAILURE, 0);
    runner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    runner.assertTransferCount(SplitText.REL_SPLITS, 10);

    final List<MockFlowFile> splits = runner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splits.get(0).assertContentEquals("HeaderLine1\nLine2SpacesAtEnd  \nLine3\nLine4");
    splits.get(1).assertContentEquals("HeaderLine1\nLine8\nLine9\n\n\n13");
    splits.get(2).assertContentEquals("HeaderLine1\n14\n15    EndofLine15\n16");
    splits.get(3).assertContentEquals("HeaderLine1");
    splits.get(4).assertContentEquals("HeaderLine1");
    splits.get(5).assertContentEquals("HeaderLine1");
    splits.get(6).assertContentEquals("HeaderLine1");
    splits.get(7).assertContentEquals("HeaderLine1");
    splits.get(8).assertContentEquals("HeaderLine1");
    splits.get(9).assertContentEquals("HeaderLine1\n\nLastLine");
}
 
Example 27
Source Project: nifi   Source File: ConsumeAMQPTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void validateSuccessfullConsumeAndTransferToSuccess() throws Exception {
    final Map<String, List<String>> routingMap = Collections.singletonMap("key1", Arrays.asList("queue1", "queue2"));
    final Map<String, String> exchangeToRoutingKeymap = Collections.singletonMap("myExchange", "key1");

    final Connection connection = new TestConnection(exchangeToRoutingKeymap, routingMap);

    try (AMQPPublisher sender = new AMQPPublisher(connection, mock(ComponentLog.class))) {
        sender.publish("hello".getBytes(), MessageProperties.PERSISTENT_TEXT_PLAIN, "key1", "myExchange");

        ConsumeAMQP proc = new LocalConsumeAMQP(connection);
        TestRunner runner = TestRunners.newTestRunner(proc);
        runner.setProperty(ConsumeAMQP.HOST, "injvm");
        runner.setProperty(ConsumeAMQP.QUEUE, "queue1");

        runner.run();
        final MockFlowFile successFF = runner.getFlowFilesForRelationship(PublishAMQP.REL_SUCCESS).get(0);
        assertNotNull(successFF);
    }
}
 
Example 28
Source Project: localization_nifi   Source File: PutGCSObjectIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEncryptedPut() throws Exception {
    final TestRunner runner = buildNewRunner(new PutGCSObject());
    runner.setProperty(PutGCSObject.BUCKET, BUCKET);
    runner.setProperty(PutGCSObject.KEY, KEY);
    runner.setProperty(PutGCSObject.ENCRYPTION_KEY, ENCRYPTION_KEY);

    runner.enqueue(CONTENT);

    runner.run();

    runner.assertAllFlowFilesTransferred(ListGCSBucket.REL_SUCCESS, 1);
    assertTrue(fileEqualsEncrypted(KEY, CONTENT));

    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListGCSBucket.REL_SUCCESS).get(0);
    flowFile.assertAttributeExists(ENCRYPTION_ALGORITHM_ATTR);

    for (Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) {
        System.out.println(entry.getKey() + ":" + entry.getValue());
    }
}
 
Example 29
Source Project: nifi   Source File: TestCompareFuzzyHash.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSsdeepCompareFuzzyHashWithBlankHashList() {
    double matchingSimilarity = 80;
    runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue());
    runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
    runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/blank_ssdeep.list");
    runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));

    Map<String, String> attributes = new HashMap<>();
    attributes.put("fuzzyhash.value", "6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS");

    runner.enqueue("bogus".getBytes(), attributes);
    runner.run();

    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1);

    final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0);
}
 
Example 30
@Test
public void testFetchElasticsearchOnTrigger() throws IOException {
    runner = TestRunners.newTestRunner(QueryElasticsearchHttp.class); // all docs are found
    runner.setValidateExpressionUsage(true);
    runner.setProperty(AbstractElasticsearchHttpProcessor.ES_URL,
            "http://localhost.internal:9200");

    runner.setProperty(QueryElasticsearchHttp.INDEX, "prod-accounting");
    runner.assertNotValid();
    runner.setProperty(QueryElasticsearchHttp.TYPE, "provenance");
    runner.assertNotValid();
    runner.setProperty(QueryElasticsearchHttp.QUERY,
            "identifier:2f79eba8839f5976cd0b1e16a0e7fe8d7dd0ceca");
    runner.setProperty(QueryElasticsearchHttp.SORT, "timestamp:asc");
    runner.setProperty(QueryElasticsearchHttp.FIELDS, "transit_uri,version");
    runner.setProperty(QueryElasticsearchHttp.PAGE_SIZE, "1");
    runner.assertValid();

    runner.setIncomingConnection(false);
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(QueryElasticsearchHttp.REL_SUCCESS, 3);
    final MockFlowFile out = runner.getFlowFilesForRelationship(
            QueryElasticsearchHttp.REL_SUCCESS).get(0);
    assertNotNull(out);
}