org.apache.nifi.util.MockFlowFile Java Examples

The following examples show how to use org.apache.nifi.util.MockFlowFile. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestReplaceTextWithMapping.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testWithMatchingGroupAndContext() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceTextWithMapping());
    runner.setProperty(ReplaceTextWithMapping.REGEX, "-(.*?)-");
    runner.setProperty(ReplaceTextWithMapping.MATCHING_GROUP_FOR_LOOKUP_KEY, "1");
    runner.setProperty(ReplaceTextWithMapping.MAPPING_FILE, Paths.get("src/test/resources/TestReplaceTextWithMapping/color-fruit-mapping.txt").toFile().getAbsolutePath());

    runner.enqueue(Paths.get("src/test/resources/TestReplaceTextWithMapping/colors.txt"));
    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceTextWithMapping.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceTextWithMapping.REL_SUCCESS).get(0);
    String outputString = new String(out.toByteArray());
    String expected = "-roses- are -apple-\n"
            + "violets are -blueberry-\n"
            + "something else is -grape-\n"
            + "I'm not good at writing poems";
    assertEquals(expected, outputString);
}
 
Example #2
Source File: PutGCSObjectTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAclAttributeProject() throws Exception {
    reset(storage, blob);
    final PutGCSObject processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.create(any(BlobInfo.class), any(InputStream.class), any(Storage.BlobWriteOption.class)))
            .thenReturn(blob);

    final Acl.Project mockProject = mock(Acl.Project.class);
    when(mockProject.getProjectId()).thenReturn(OWNER_PROJECT_ID);
    when(blob.getOwner()).thenReturn(mockProject);

    runner.enqueue("test");
    runner.run();

    runner.assertAllFlowFilesTransferred(PutGCSObject.REL_SUCCESS);
    runner.assertTransferCount(PutGCSObject.REL_SUCCESS, 1);

    final MockFlowFile mockFlowFile = runner.getFlowFilesForRelationship(PutGCSObject.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(OWNER_ATTR, OWNER_PROJECT_ID);
    mockFlowFile.assertAttributeEquals(OWNER_TYPE_ATTR, "project");
}
 
Example #3
Source File: TestInferAvroSchema.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void inferAvroSchemaFromHeaderDefinitionOfCSVTabDelimitedFile() throws Exception {

    runner.setProperty(InferAvroSchema.DELIMITER, "\\t");
    runner.assertValid();

    Map<String, String> attributes = new HashMap<>();
    attributes.put(CoreAttributes.MIME_TYPE.key(), "text/csv");
    runner.enqueue(new File("src/test/resources/Shapes_Header_TabDelimited.csv").toPath(), attributes);

    runner.run();
    runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0);
    runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0);
    runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1);
    runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1);

    MockFlowFile flowFile = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0);
    flowFile.assertContentEquals(unix2PlatformSpecificLineEndings(new File("src/test/resources/Shapes_header.csv.avro")));
    flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/avro-binary");
}
 
Example #4
Source File: TestInvokeGroovy.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a script that has a Groovy Processor that that reads the first line of text from the flowfiles content and stores the value in an attribute of the outgoing flowfile.
 *
 * @throws Exception Any error encountered while testing
 */
@Test
public void testReadFlowFileContentAndStoreInFlowFileAttribute() throws Exception {
    runner.setValidateExpressionUsage(false);
    runner.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE, "Groovy");
    runner.setProperty(ScriptingComponentUtils.SCRIPT_FILE, "target/test/resources/groovy/test_reader.groovy");
    runner.setProperty(ScriptingComponentUtils.MODULES, "target/test/resources/groovy");

    runner.assertValid();
    runner.enqueue("test content".getBytes(StandardCharsets.UTF_8));
    runner.run();

    runner.assertAllFlowFilesTransferred("test", 1);
    final List<MockFlowFile> result = runner.getFlowFilesForRelationship("test");
    result.get(0).assertAttributeEquals("from-content", "test content");
}
 
Example #5
Source File: TestExtractText.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testIgnoreZeroCaptureGroupProperty() throws Exception {
    final TestRunner testRunner = TestRunners.newTestRunner(new ExtractText());

    testRunner.setProperty(ExtractText.INCLUDE_CAPTURE_GROUP_ZERO, "false");

    final String attributeKey = "regex.result";

    testRunner.setProperty(attributeKey, "(?s)(.*)");

    testRunner.enqueue(SAMPLE_STRING.getBytes("UTF-8"));
    testRunner.run();

    testRunner.assertAllFlowFilesTransferred(ExtractText.REL_MATCH, 1);
    final MockFlowFile out = testRunner.getFlowFilesForRelationship(ExtractText.REL_MATCH).get(0);

    // Ensure the zero capture group is not in the resultant attributes
    out.assertAttributeNotExists(attributeKey + ".0");
    out.assertAttributeEquals(attributeKey, SAMPLE_STRING);
}
 
Example #6
Source File: TestInferAvroSchema.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void inferAvroSchemaFromHeaderDefinitionOfCSVFile() throws Exception {

    runner.assertValid();

    Map<String, String> attributes = new HashMap<>();
    attributes.put(CoreAttributes.MIME_TYPE.key(), "text/csv");
    runner.enqueue(new File("src/test/resources/Shapes_Header.csv").toPath(), attributes);

    runner.run();
    runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0);
    runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0);
    runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1);
    runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1);

    MockFlowFile flowFile = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0);
    flowFile.assertContentEquals(unix2PlatformSpecificLineEndings(new File("src/test/resources/Shapes_header.csv.avro")));
    flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/avro-binary");
}
 
Example #7
Source File: ITPutKinesisFirehose.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void test5MessageWithBatch2MaxBufferSize10MBRunOnce2MessageSent() {
    runner = TestRunners.newTestRunner(PutKinesisFirehose.class);
    runner.setProperty(PutKinesisFirehose.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutKinesisFirehose.BATCH_SIZE, "2");
    runner.setProperty(PutKinesisFirehose.MAX_MESSAGE_BUFFER_SIZE_MB, "1 MB");
    runner.setProperty(PutKinesisFirehose.KINESIS_FIREHOSE_DELIVERY_STREAM_NAME, "testkinesis");
    runner.assertValid();
    byte [] bytes = new byte[10];
    for (int i = 0; i < bytes.length; i++) {
        bytes[i] = 'a';
    }
    runner.enqueue(bytes);
    runner.enqueue(bytes.clone());
    runner.enqueue(bytes.clone());
    runner.enqueue(bytes);
    runner.enqueue(bytes.clone());
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(PutKinesisFirehose.REL_SUCCESS, 2);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutKinesisFirehose.REL_SUCCESS);
    assertEquals(2,flowFiles.size());
    for (MockFlowFile flowFile : flowFiles) {
        flowFile.assertAttributeExists(PutKinesisFirehose.AWS_KINESIS_FIREHOSE_RECORD_ID);
    }
}
 
Example #8
Source File: TestCSVToAvroProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(FAILURE_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals("Should set an error message",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example #9
Source File: TestLogAttribute.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testLogPropertyRegexWithIgnoreRegex() {
    final LogAttribute logAttribute = new LogAttribute();
    final TestRunner runner = TestRunners.newTestRunner(logAttribute);
    final ProcessContext context = runner.getProcessContext();
    final ProcessSession session = runner.getProcessSessionFactory().createSession();
    final MockComponentLog LOG = runner.getLogger();

    runner.setProperty(LogAttribute.ATTRIBUTES_TO_LOG_REGEX, "foo.*"); // includes foo,foobaz
    runner.setProperty(LogAttribute.ATTRIBUTES_TO_IGNORE_REGEX, "foobaz.*"); // includes foobaz

    final Map<String,String> attrs = Maps.newHashMap();
    attrs.put("foo", "foo-value");
    attrs.put("bar", "bar-value");
    attrs.put("foobaz", "foobaz-value");

    final MockFlowFile flowFile = runner.enqueue("content", attrs);

    final String logMessage = logAttribute.processFlowFile(LOG, LogAttribute.DebugLevels.info, flowFile, session, context);
    assertThat(logMessage, not(containsString("foobaz-value")));
    assertThat(logMessage, containsString("foo-value"));
    assertThat(logMessage, not(containsString("bar-value")));
}
 
Example #10
Source File: TestEvaluateJsonPath.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleAsciiControlCharacters() throws Exception {
    final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateJsonPath());
    testRunner.setProperty(EvaluateJsonPath.DESTINATION, EvaluateJsonPath.DESTINATION_ATTRIBUTE);
    testRunner.setProperty(EvaluateJsonPath.RETURN_TYPE, EvaluateJsonPath.RETURN_TYPE_JSON);

    final String jsonPathControlCharKey = "evaluatejson.controlcharacterpath";

    testRunner.setProperty(jsonPathControlCharKey, "$.jinxing_json.object.property");

    testRunner.enqueue(Paths.get("src/test/resources/TestJson/control-characters.json"));
    testRunner.run();

    final Relationship expectedRel = EvaluateJsonPath.REL_MATCH;

    testRunner.assertAllFlowFilesTransferred(expectedRel, 1);
    final MockFlowFile out = testRunner.getFlowFilesForRelationship(expectedRel).get(0);
    Assert.assertNotNull("Transferred flow file did not have the correct result for id attribute", out.getAttribute(jsonPathControlCharKey));
}
 
Example #11
Source File: TestGetCouchbaseKey.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testPutToAttributeNoTargetAttribute() throws Exception {

    Bucket bucket = mock(Bucket.class);
    String inFileDataStr = "doc-in";
    String content = "some-value";
    when(bucket.get(inFileDataStr, RawJsonDocument.class))
        .thenReturn(RawJsonDocument.create(inFileDataStr, content));
    setupMockBucket(bucket);

    byte[] inFileData = inFileDataStr.getBytes(StandardCharsets.UTF_8);
    testRunner.setProperty(PUT_VALUE_TO_ATTRIBUTE, "${expressionReturningNoValue}");
    testRunner.enqueue(inFileData);
    testRunner.run();

    testRunner.assertTransferCount(REL_SUCCESS, 0);
    testRunner.assertTransferCount(REL_ORIGINAL, 0);
    testRunner.assertTransferCount(REL_RETRY, 0);
    testRunner.assertTransferCount(REL_FAILURE, 1);
    MockFlowFile outFile = testRunner.getFlowFilesForRelationship(REL_FAILURE).get(0);
    outFile.assertContentEquals(inFileDataStr);
}
 
Example #12
Source File: TestRouteText.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleCaseInsensitiveStartsMatch() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new RouteText());
    runner.setProperty(RouteText.MATCH_STRATEGY, RouteText.STARTS_WITH);
    runner.setProperty(RouteText.IGNORE_CASE, "true");
    runner.setProperty("simple", "start");

    runner.enqueue("start middle end\nSTart middle end".getBytes("UTF-8"));
    runner.run();

    runner.assertTransferCount("simple", 1);
    runner.assertTransferCount("unmatched", 0);
    runner.assertTransferCount("original", 1);
    final MockFlowFile outMatched = runner.getFlowFilesForRelationship("simple").get(0);
    outMatched.assertContentEquals("start middle end\nSTart middle end".getBytes("UTF-8"));
}
 
Example #13
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldProperlyHandleAnEmptyEventBatch() throws Exception {
    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(new Event[]{});

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path");
    runner.setProperty(GetHDFSEvents.NUMBER_OF_RETRIES_FOR_POLL, "5");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(0, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #14
Source File: TestLogAttribute.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testLogPropertyWithIgnoreCSV() {
    final LogAttribute logAttribute = new LogAttribute();
    final TestRunner runner = TestRunners.newTestRunner(logAttribute);
    final ProcessContext context = runner.getProcessContext();
    final ProcessSession session = runner.getProcessSessionFactory().createSession();
    final MockComponentLog LOG = runner.getLogger();

    runner.setProperty(LogAttribute.ATTRIBUTES_TO_IGNORE_CSV, "bar");

    final Map<String,String> attrs = Maps.newHashMap();
    attrs.put("foo", "foo-value");
    attrs.put("bar", "bar-value");
    attrs.put("foobaz", "foobaz-value");

    final MockFlowFile flowFile = runner.enqueue("content", attrs);

    final String logMessage = logAttribute.processFlowFile(LOG, LogAttribute.DebugLevels.info, flowFile, session, context);
    assertThat(logMessage, containsString("foobaz-value"));
    assertThat(logMessage, containsString("foo-value"));
    assertThat(logMessage, not(containsString("bar-value")));
}
 
Example #15
Source File: TestInFlightMessageTracker.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000L)
public void testAwaitCompletionWhenComplete() throws InterruptedException, TimeoutException {
    final MockFlowFile flowFile = new MockFlowFile(1L);

    final InFlightMessageTracker tracker = new InFlightMessageTracker();
    tracker.incrementSentCount(flowFile);

    verifyNotComplete(tracker);

    tracker.incrementSentCount(flowFile);
    verifyNotComplete(tracker);

    tracker.incrementAcknowledgedCount(flowFile);
    verifyNotComplete(tracker);

    tracker.incrementAcknowledgedCount(flowFile);
    tracker.awaitCompletion(1L);
}
 
Example #16
Source File: TestFetchElasticsearchHttp.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testFetchElasticsearchOnTriggerWithDocNotFound() throws IOException {
    runner = TestRunners.newTestRunner(new FetchElasticsearchHttpTestProcessor(false)); // simulate doc not found
    runner.setProperty(AbstractElasticsearchHttpProcessor.ES_URL, "http://127.0.0.1:9200");
    runner.setProperty(FetchElasticsearchHttp.INDEX, "doc");
    runner.setProperty(FetchElasticsearchHttp.DOC_ID, "${doc_id}");

    runner.setIncomingConnection(true);
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});
    runner.run(1, true, true);

    // This test generates a "document not found"
    runner.assertAllFlowFilesTransferred(FetchElasticsearchHttp.REL_NOT_FOUND, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(FetchElasticsearchHttp.REL_NOT_FOUND).get(0);
    assertNotNull(out);
    out.assertAttributeEquals("doc_id", "28039652140");
}
 
Example #17
Source File: TestFetchHDFS.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAutomaticDecompression() throws IOException {
    FetchHDFS proc = new TestableFetchHDFS(kerberosProperties);
    TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(FetchHDFS.FILENAME, "src/test/resources/testdata/randombytes-1.gz");
    runner.setProperty(FetchHDFS.COMPRESSION_CODEC, "AUTOMATIC");
    runner.enqueue(new String("trigger flow file"));
    runner.run();

    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(FetchHDFS.REL_SUCCESS);
    assertEquals(1, flowFiles.size());

    MockFlowFile flowFile = flowFiles.get(0);
    assertTrue(flowFile.getAttribute(CoreAttributes.FILENAME.key()).equals("randombytes-1"));
    InputStream expected = getClass().getResourceAsStream("/testdata/randombytes-1");
    flowFile.assertContentEquals(expected);
}
 
Example #18
Source File: TestSplitText.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveLineSplitWithHeaderNotRetainNewline() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new SplitText());
    runner.setProperty(SplitText.HEADER_LINE_COUNT, "1");
    runner.setProperty(SplitText.LINE_SPLIT_COUNT, "5");
    runner.setProperty(SplitText.REMOVE_TRAILING_NEWLINES, "true");

    runner.enqueue(TEST_INPUT_DATA);
    runner.run();

    runner.assertTransferCount(SplitText.REL_FAILURE, 0);
    runner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    runner.assertTransferCount(SplitText.REL_SPLITS, 10);

    final List<MockFlowFile> splits = runner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splits.get(0).assertContentEquals("HeaderLine1\nLine2SpacesAtEnd  \nLine3\nLine4");
    splits.get(1).assertContentEquals("HeaderLine1\nLine8\nLine9\n\n\n13");
    splits.get(2).assertContentEquals("HeaderLine1\n14\n15    EndofLine15\n16");
    splits.get(3).assertContentEquals("HeaderLine1");
    splits.get(4).assertContentEquals("HeaderLine1");
    splits.get(5).assertContentEquals("HeaderLine1");
    splits.get(6).assertContentEquals("HeaderLine1");
    splits.get(7).assertContentEquals("HeaderLine1");
    splits.get(8).assertContentEquals("HeaderLine1");
    splits.get(9).assertContentEquals("HeaderLine1\n\nLastLine");
}
 
Example #19
Source File: ConsumeAMQPTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void validateSuccessfullConsumeAndTransferToSuccess() throws Exception {
    final Map<String, List<String>> routingMap = Collections.singletonMap("key1", Arrays.asList("queue1", "queue2"));
    final Map<String, String> exchangeToRoutingKeymap = Collections.singletonMap("myExchange", "key1");

    final Connection connection = new TestConnection(exchangeToRoutingKeymap, routingMap);

    try (AMQPPublisher sender = new AMQPPublisher(connection, mock(ComponentLog.class))) {
        sender.publish("hello".getBytes(), MessageProperties.PERSISTENT_TEXT_PLAIN, "key1", "myExchange");

        ConsumeAMQP proc = new LocalConsumeAMQP(connection);
        TestRunner runner = TestRunners.newTestRunner(proc);
        runner.setProperty(ConsumeAMQP.HOST, "injvm");
        runner.setProperty(ConsumeAMQP.QUEUE, "queue1");

        runner.run();
        final MockFlowFile successFF = runner.getFlowFilesForRelationship(PublishAMQP.REL_SUCCESS).get(0);
        assertNotNull(successFF);
    }
}
 
Example #20
Source File: PutGCSObjectIT.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testEncryptedPut() throws Exception {
    final TestRunner runner = buildNewRunner(new PutGCSObject());
    runner.setProperty(PutGCSObject.BUCKET, BUCKET);
    runner.setProperty(PutGCSObject.KEY, KEY);
    runner.setProperty(PutGCSObject.ENCRYPTION_KEY, ENCRYPTION_KEY);

    runner.enqueue(CONTENT);

    runner.run();

    runner.assertAllFlowFilesTransferred(ListGCSBucket.REL_SUCCESS, 1);
    assertTrue(fileEqualsEncrypted(KEY, CONTENT));

    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListGCSBucket.REL_SUCCESS).get(0);
    flowFile.assertAttributeExists(ENCRYPTION_ALGORITHM_ATTR);

    for (Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) {
        System.out.println(entry.getKey() + ":" + entry.getValue());
    }
}
 
Example #21
Source File: TestCompareFuzzyHash.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSsdeepCompareFuzzyHashWithBlankHashList() {
    double matchingSimilarity = 80;
    runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue());
    runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
    runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/blank_ssdeep.list");
    runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));

    Map<String, String> attributes = new HashMap<>();
    attributes.put("fuzzyhash.value", "6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS");

    runner.enqueue("bogus".getBytes(), attributes);
    runner.run();

    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1);

    final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0);
}
 
Example #22
Source File: ITQueryElasticsearchHttp.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testFetchElasticsearchOnTrigger() throws IOException {
    runner = TestRunners.newTestRunner(QueryElasticsearchHttp.class); // all docs are found
    runner.setValidateExpressionUsage(true);
    runner.setProperty(AbstractElasticsearchHttpProcessor.ES_URL,
            "http://localhost.internal:9200");

    runner.setProperty(QueryElasticsearchHttp.INDEX, "prod-accounting");
    runner.assertNotValid();
    runner.setProperty(QueryElasticsearchHttp.TYPE, "provenance");
    runner.assertNotValid();
    runner.setProperty(QueryElasticsearchHttp.QUERY,
            "identifier:2f79eba8839f5976cd0b1e16a0e7fe8d7dd0ceca");
    runner.setProperty(QueryElasticsearchHttp.SORT, "timestamp:asc");
    runner.setProperty(QueryElasticsearchHttp.FIELDS, "transit_uri,version");
    runner.setProperty(QueryElasticsearchHttp.PAGE_SIZE, "1");
    runner.assertValid();

    runner.setIncomingConnection(false);
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(QueryElasticsearchHttp.REL_SUCCESS, 3);
    final MockFlowFile out = runner.getFlowFilesForRelationship(
            QueryElasticsearchHttp.REL_SUCCESS).get(0);
    assertNotNull(out);
}
 
Example #23
Source File: TestPutDistributedMapCache.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxCacheEntrySize() throws InitializationException, IOException {

    runner.setProperty(PutDistributedMapCache.CACHE_ENTRY_IDENTIFIER, "${uuid}");
    runner.setProperty(PutDistributedMapCache.CACHE_ENTRY_MAX_BYTES, "10 B");

    // max length is 10 bytes, flow file content is 20 bytes
    String flowFileContent = "contentwhichistoobig";
    runner.enqueue(flowFileContent.getBytes("UTF-8"));

    runner.run();

    // no cache key attribute
    runner.assertAllFlowFilesTransferred(PutDistributedMapCache.REL_FAILURE, 1);
    runner.assertTransferCount(PutDistributedMapCache.REL_FAILURE, 1);

    final MockFlowFile outputFlowFile = runner.getFlowFilesForRelationship(PutDistributedMapCache.REL_FAILURE).get(0);
    outputFlowFile.assertAttributeNotExists("cached");
    outputFlowFile.assertContentEquals(flowFileContent);


    runner.clearTransferState();
    runner.setProperty(PutDistributedMapCache.CACHE_ENTRY_MAX_BYTES, "1 MB");
}
 
Example #24
Source File: GetHDFSTest.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testDirectoryUsesValidEL() throws IOException {
    GetHDFS proc = new TestableGetHDFS(kerberosProperties);
    TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(PutHDFS.DIRECTORY, "src/test/resources/${literal('testdata'):substring(0,8)}");
    runner.setProperty(GetHDFS.FILE_FILTER_REGEX, ".*.zip");
    runner.setProperty(GetHDFS.KEEP_SOURCE_FILE, "true");
    runner.setProperty(GetHDFS.COMPRESSION_CODEC, "AUTOMATIC");
    runner.run();

    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetHDFS.REL_SUCCESS);
    assertEquals(1, flowFiles.size());

    MockFlowFile flowFile = flowFiles.get(0);
    assertTrue(flowFile.getAttribute(CoreAttributes.FILENAME.key()).equals("13545423550275052.zip"));
    InputStream expected = getClass().getResourceAsStream("/testdata/13545423550275052.zip");
    flowFile.assertContentEquals(expected);
}
 
Example #25
Source File: TestDeleteElasticsearch5.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeleteSuccessful() throws IOException {
    restStatus = RestStatus.OK;
    deleteResponse = new DeleteResponse(null, TYPE1, documentId, 1, true) {

        @Override
        public RestStatus status() {
            return restStatus;
        }

    };
    runner.enqueue(new byte [] {}, new HashMap<String, String>() {{
        put("documentId", documentId);
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(DeleteElasticsearch5.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(DeleteElasticsearch5.REL_SUCCESS).get(0);
    assertNotNull(out);
    assertEquals(null,out.getAttribute(DeleteElasticsearch5.ES_ERROR_MESSAGE));
    out.assertAttributeEquals(DeleteElasticsearch5.ES_FILENAME, documentId);
    out.assertAttributeEquals(DeleteElasticsearch5.ES_INDEX, INDEX1);
    out.assertAttributeEquals(DeleteElasticsearch5.ES_TYPE, TYPE1);
}
 
Example #26
Source File: PutDynamoDBTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringHashStringRangePutOnlyHashWithRangeNameNoRangeValueFailure() {
    final TestRunner putRunner = TestRunners.newTestRunner(PutDynamoDB.class);

    putRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    putRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    putRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    putRunner.enqueue(new byte[] {});

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);

    List<MockFlowFile> flowFiles = putRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertNotNull(flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_RANGE_KEY_VALUE_ERROR));
    }
}
 
Example #27
Source File: TestFetchElasticsearch.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testFetchElasticsearchOnTriggerWithFailures() throws IOException {
    runner = TestRunners.newTestRunner(new FetchElasticsearchTestProcessor(false)); // simulate doc not found
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.CLUSTER_NAME, "elasticsearch");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(FetchElasticsearch.INDEX, "doc");
    runner.setProperty(FetchElasticsearch.TYPE, "status");
    runner.setProperty(FetchElasticsearch.DOC_ID, "${doc_id}");

    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});
    runner.run(1, true, true);

    // This test generates a "document not found"
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_NOT_FOUND, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(FetchElasticsearch.REL_NOT_FOUND).get(0);
    assertNotNull(out);
    out.assertAttributeEquals("doc_id", "28039652140");
}
 
Example #28
Source File: ExecuteGroovyScriptTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void test_filter_01() throws Exception {
    runner.setProperty(proc.SCRIPT_BODY, "def ff = session.get{it.FILTER=='3'}; if(!ff)return; REL_SUCCESS << ff;");
    //runner.setProperty(proc.FAIL_STRATEGY, "rollback");

    runner.assertValid();

    runner.enqueue("01".getBytes("UTF-8"), map("FILTER", "1"));
    runner.enqueue("31".getBytes("UTF-8"), map("FILTER", "3"));
    runner.enqueue("03".getBytes("UTF-8"), map("FILTER", "2"));
    runner.enqueue("32".getBytes("UTF-8"), map("FILTER", "3"));
    runner.run();

    runner.assertAllFlowFilesTransferred(proc.REL_SUCCESS.getName(), 2);
    final List<MockFlowFile> result = runner.getFlowFilesForRelationship(proc.REL_SUCCESS.getName());

    result.get(0).assertContentEquals("31", "UTF-8");
    result.get(1).assertContentEquals("32", "UTF-8");
}
 
Example #29
Source File: ITPutLambda.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Comment out ignore for integration tests (requires creds files)
 */
@Test
@Ignore
public void testIntegrationClientErrorBadMessageBody() throws Exception {
    runner = TestRunners.newTestRunner(PutLambda.class);
    runner.setProperty(PutLambda.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutLambda.AWS_LAMBDA_FUNCTION_NAME, "hello");
    runner.assertValid();

    runner.enqueue("badbod".getBytes());
    runner.run(1);

    runner.assertAllFlowFilesTransferred(PutLambda.REL_FAILURE, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(PutLambda.REL_FAILURE);
    final MockFlowFile out = ffs.iterator().next();
    assertNull("Function error should be null since there is exception"
        + out.getAttribute(PutLambda.AWS_LAMBDA_RESULT_FUNCTION_ERROR), out.getAttribute(PutLambda.AWS_LAMBDA_RESULT_FUNCTION_ERROR));
    assertNull("log should not be null", out.getAttribute(PutLambda.AWS_LAMBDA_RESULT_LOG));
    assertEquals("Status should be equal", null,out.getAttribute(PutLambda.AWS_LAMBDA_RESULT_STATUS_CODE));
    assertEquals("exception error code should be equal", "InvalidRequestContentException",out.getAttribute(PutLambda.AWS_LAMBDA_EXCEPTION_ERROR_CODE));
    assertEquals("exception exception error type should be equal", "Client",out.getAttribute(PutLambda.AWS_LAMBDA_EXCEPTION_ERROR_TYPE));
    assertEquals("exception exception error code should be equal", "400",out.getAttribute(PutLambda.AWS_LAMBDA_EXCEPTION_STATUS_CODE));
    assertTrue("exception exception error message should be start with",out.getAttribute(PutLambda.AWS_LAMBDA_EXCEPTION_MESSAGE)
           .startsWith("Could not parse request body into json: Unrecognized token 'badbod': was expecting ('true', 'false' or 'null')"));
}
 
Example #30
Source File: TestConvertJSONToSQL.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testUpdateBasedOnPrimaryKey() throws InitializationException, ProcessException, SQLException, IOException {
    final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class);

    runner.addControllerService("dbcp", service);
    runner.enableControllerService(service);
    runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp");
    runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS");
    runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE");
    runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json"));
    runner.run();

    runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1);
    runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT.key(), "1");
    runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0);
    out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.VARCHAR));
    out.assertAttributeEquals("sql.args.1.value", "Mark");
    out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER));
    out.assertAttributeEquals("sql.args.2.value", "48");
    out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER));
    out.assertAttributeEquals("sql.args.3.value", "1");

    out.assertContentEquals("UPDATE PERSONS SET NAME = ?, CODE = ? WHERE ID = ?");
}