Java Code Examples for org.apache.nifi.util.MockFlowFile#getAttribute()

@Test
public void testDataFormatting() throws Exception {
    testRunner.enqueue(new File("src/test/resources/dataformatting.xlsx").toPath());

    testRunner.setProperty(ConvertExcelToCSVProcessor.FORMAT_VALUES, "false");

    testRunner.run();

    testRunner.assertTransferCount(ConvertExcelToCSVProcessor.SUCCESS, 1);
    testRunner.assertTransferCount(ConvertExcelToCSVProcessor.ORIGINAL, 1);
    testRunner.assertTransferCount(ConvertExcelToCSVProcessor.FAILURE, 0);

    MockFlowFile ff = testRunner.getFlowFilesForRelationship(ConvertExcelToCSVProcessor.SUCCESS).get(0);
    Long rowsSheet = new Long(ff.getAttribute(ConvertExcelToCSVProcessor.ROW_NUM));
    assertTrue(rowsSheet == 9);

    ff.assertContentEquals("Numbers,Timestamps,Money\n" +
            "1234.4559999999999,42736.5,123.45\n" +
            "1234.4559999999999,42736.5,123.45\n" +
            "1234.4559999999999,42736.5,123.45\n" +
            "1234.4559999999999,42736.5,1023.45\n" +
            "1234.4559999999999,42736.5,1023.45\n" +
            "987654321,42736.5,1023.45\n" +
            "987654321,,\n" +
            "987654321,,\n");
}
 

@Test
public void testMatchesMultipleXmlAttribute() throws XPathFactoryConfigurationException, IOException {
    final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
    testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
    testRunner.setProperty("some.property", "//fruit/name");

    testRunner.enqueue(XML_SNIPPET);
    testRunner.run();

    testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);

    final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);

    for (int i = 0; i < fruitNames.length; i++) {
        final String outXml = out.getAttribute("some.property." + ((int) i + 1));
        String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><name xmlns:ns=\"http://namespace/1\">" + fruitNames[i] + "</name>";
        assertEquals(expectedXml, outXml.trim());
    }
    testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0).assertContentEquals(XML_SNIPPET);
}
 

@Test
public void testFlowFileStreamV2() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new UnpackContent());
    runner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.FLOWFILE_STREAM_FORMAT_V2.toString());
    runner.enqueue(dataPath.resolve("data.flowfilev2"));
    runner.enqueue(dataPath.resolve("data.flowfilev2"));

    runner.run(2);

    runner.assertTransferCount(UnpackContent.REL_SUCCESS, 4);
    runner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2);
    runner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2");
    runner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2");
    runner.assertTransferCount(UnpackContent.REL_FAILURE, 0);

    final List<MockFlowFile> unpacked = runner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS);
    for (final MockFlowFile flowFile : unpacked) {
        final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
        final String folder = flowFile.getAttribute(CoreAttributes.PATH.key());
        final Path path = dataPath.resolve(folder).resolve(filename);
        assertTrue(Files.exists(path));

        flowFile.assertContentEquals(path.toFile());
    }
}
 

@Test
public void inferAvroSchemaFromJSONFile() throws Exception {

    runner.assertValid();

    runner.setProperty(InferAvroSchema.INPUT_CONTENT_TYPE, InferAvroSchema.USE_MIME_TYPE);

    // Purposely set to True to test that none of the JSON file is read which would cause issues.
    runner.setProperty(InferAvroSchema.GET_CSV_HEADER_DEFINITION_FROM_INPUT, "true");
    runner.setProperty(InferAvroSchema.SCHEMA_DESTINATION, InferAvroSchema.DESTINATION_ATTRIBUTE);

    Map<String, String> attributes = new HashMap<>();
    attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json");
    runner.enqueue(new File("src/test/resources/Shapes.json").toPath(), attributes);

    runner.run();
    runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0);
    runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0);
    runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1);
    runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1);

    MockFlowFile data = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0);
    String avroSchema = data.getAttribute(InferAvroSchema.AVRO_SCHEMA_ATTRIBUTE_NAME);
    String knownSchema = new String(unix2PlatformSpecificLineEndings(new File("src/test/resources/Shapes.json.avro")), StandardCharsets.UTF_8);
    Assert.assertEquals(avroSchema, knownSchema);

    // Since that avro schema is written to an attribute this should be teh same as the original
    data.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json");
}
 

@Test
public void testWriteXmlToAttribute() throws XPathFactoryConfigurationException, IOException {
    final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
    testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
    testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name");

    testRunner.enqueue(XML_SNIPPET);
    testRunner.run();

    testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);
    final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
    final String outXml = out.getAttribute("some.property");
    assertTrue(outXml.contains("<name xmlns:ns=\"http://namespace/1\">apple</name>"));
    testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0).assertContentEquals(XML_SNIPPET);
}
 

@Test
public void testExecuteJarPutToAttribute() throws Exception {
    File exJar = new File("src/test/resources/ExecuteCommand/TestSuccess.jar");
    File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
    String jarPath = exJar.getAbsolutePath();
    exJar.setExecutable(true);
    final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
    controller.enqueue(dummy.toPath());
    controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
    controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
    controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");
    controller.run(1);
    controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
    controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);

    List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
    MockFlowFile outputFlowFile = flowFiles.get(0);
    String result = outputFlowFile.getAttribute("executeStreamCommand.output");
    outputFlowFile.assertContentEquals(dummy);
    assertTrue(Pattern.compile("Test was a success\r?\n").matcher(result).find());
    assertEquals("0", outputFlowFile.getAttribute("execution.status"));
    assertEquals("java", outputFlowFile.getAttribute("execution.command"));
    assertEquals("-jar;", outputFlowFile.getAttribute("execution.command.args").substring(0, 5));
    String attribute = outputFlowFile.getAttribute("execution.command.args");
    String expected = "src" + File.separator + "test" + File.separator + "resources" + File.separator + "ExecuteCommand" + File.separator + "TestSuccess.jar";
    assertEquals(expected, attribute.substring(attribute.length() - expected.length()));
}
 

@Test
public void test401NotAuth() throws Exception {
    addHandler(new BasicAuthHandler());

    final String username = "basic_user";
    final String password = "basic_password";

    runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/401");
    runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_USERNAME, username);
    runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_PASSWORD, password);

    createFlowFiles(runner);

    runner.run();

    runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0);
    runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0);
    runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1);
    runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
    runner.assertPenalizeCount(0);

    // expected in request status.code and status.message
    // original flow file (+attributes)
    final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0);
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "401");
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Unauthorized");
    bundle.assertAttributeEquals("Foo", "Bar");
    final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8);
    final String expected = "Hello";
    Assert.assertEquals(expected, actual);

    final String response = bundle.getAttribute(InvokeHTTP.RESPONSE_BODY);
    assertEquals(response, "Get off my lawn!"+System.lineSeparator());
}
 

@Test
public void makeSureExpressionLanguageIsWorkingProperlyWithinTheHdfsPathToWatch() throws Exception {
    Event[] events = new Event[] {
            new Event.CreateEvent.Builder().path("/some/path/1/2/3/t.txt").build(),
            new Event.CreateEvent.Builder().path("/some/path/1/2/4/t.txt").build(),
            new Event.CreateEvent.Builder().path("/some/path/1/2/3/.t.txt").build()
    };

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/${literal(1)}/${literal(2)}/${literal(3)}/.*.txt");
    runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create");
    runner.setProperty(GetHDFSEvents.IGNORE_HIDDEN_FILES, "true");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(1, successfulFlowFiles.size());

    for (MockFlowFile f : successfulFlowFiles) {
        String eventType = f.getAttribute(EventAttributes.EVENT_TYPE);
        assertTrue(eventType.equals("CREATE"));
    }

    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 

@Test
public void validateSuccessfulConsumeAndTransferToSuccess() throws Exception {
    final String destinationName = "cooQueue";
    JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false);
    JMSPublisher sender = new JMSPublisher(jmsTemplate, mock(ComponentLog.class));
    final Map<String, String> senderAttributes = new HashMap<>();
    senderAttributes.put("filename", "message.txt");
    senderAttributes.put("attribute_from_sender", "some value");
    sender.publish(destinationName, "Hey dude!".getBytes(), senderAttributes);
    TestRunner runner = TestRunners.newTestRunner(new ConsumeJMS());
    JMSConnectionFactoryProviderDefinition cs = mock(JMSConnectionFactoryProviderDefinition.class);
    when(cs.getIdentifier()).thenReturn("cfProvider");
    when(cs.getConnectionFactory()).thenReturn(jmsTemplate.getConnectionFactory());
    runner.addControllerService("cfProvider", cs);
    runner.enableControllerService(cs);

    runner.setProperty(PublishJMS.CF_SERVICE, "cfProvider");
    runner.setProperty(ConsumeJMS.DESTINATION, destinationName);
    runner.setProperty(ConsumeJMS.DESTINATION_TYPE, ConsumeJMS.QUEUE);
    runner.run(1, false);
    //
    final MockFlowFile successFF = runner.getFlowFilesForRelationship(PublishJMS.REL_SUCCESS).get(0);
    assertNotNull(successFF);
    successFF.assertAttributeExists(JmsHeaders.DESTINATION);
    successFF.assertAttributeEquals(JmsHeaders.DESTINATION, destinationName);
    successFF.assertAttributeExists("filename");
    successFF.assertAttributeEquals("filename", "message.txt");
    successFF.assertAttributeExists("attribute_from_sender");
    successFF.assertAttributeEquals("attribute_from_sender", "some value");
    successFF.assertContentEquals("Hey dude!".getBytes());
    String sourceDestination = successFF.getAttribute(ConsumeJMS.JMS_SOURCE_DESTINATION_NAME);
    assertNotNull(sourceDestination);

    ((CachingConnectionFactory) jmsTemplate.getConnectionFactory()).destroy();
}
 

@Test
public void testNoAttrListCoreNullOffToAttribute() throws IOException {
    final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV());
    testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "true");
    testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false");

    Map<String, String> attrs = new HashMap<String, String>(){{
        put("beach-name", "Malibu Beach");
        put("beach-location", "California, US");
        put("beach-endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim");
    }};

    testRunner.enqueue(new byte[0], attrs);
    testRunner.run();

    List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS);

    testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0);
    testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1);

    MockFlowFile flowFile = flowFilesForRelationship.get(0);

    assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));

    final String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME);

    Set<String> csvAttributeValues = new HashSet<>(getStrings(attributeData));

    assertEquals(6, csvAttributeValues.size());

    assertTrue(csvAttributeValues.contains("Malibu Beach"));
    assertTrue(csvAttributeValues.contains("\"California, US\""));
    assertTrue(csvAttributeValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\""));

    assertTrue(csvAttributeValues.contains(flowFile.getAttribute("filename")));
    assertTrue(csvAttributeValues.contains(flowFile.getAttribute("path")));
    assertTrue(csvAttributeValues.contains(flowFile.getAttribute("uuid")));
}
 

@Test
public void testAttrListNoCoreNullOffToAttribute() throws IOException {
    final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV());
    testRunner.setProperty(AttributesToCSV.DESTINATION, OUTPUT_NEW_ATTRIBUTE);
    testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "false");
    testRunner.setProperty(AttributesToCSV.ATTRIBUTES_LIST, "beach-name,beach-location,beach-endorsement");
    testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false");

    Map<String, String> attrs = new HashMap<String, String>(){{
        put("beach-name", "Malibu Beach");
        put("beach-location", "California, US");
        put("beach-endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim");
        put("attribute-should-be-eliminated", "This should not be in CSVAttribute!");
    }};

    testRunner.enqueue(new byte[0], attrs);
    testRunner.run();

    List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS);

    testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0);
    testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1);

    MockFlowFile flowFile = flowFilesForRelationship.get(0);

    assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));

    final String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME);

    Set<String> CSVDataValues = new HashSet<>(getStrings(attributeData));

    assertEquals(3, CSVDataValues.size());

    assertTrue(CSVDataValues.contains("Malibu Beach"));
    assertTrue(CSVDataValues.contains("\"California, US\""));
    assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\""));

}
 

@Test
public void testRecursiveWithDefaultFilterAndFilterMode() throws InterruptedException {
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/.testFile.txt")));
    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/testFile.txt")));

    proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/testDir")));
    proc.fileSystem.addFileStatus(new Path("/test/testDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/testDir/1.txt")));

    // first iteration will not pick up files because it has to instead check timestamps.
    // We must then wait long enough to ensure that the listing can be performed safely and
    // run the Processor again.
    runner.run();
    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 2);

    final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListHDFS.REL_SUCCESS);
    for (int i=0; i < 2; i++) {
        final MockFlowFile ff = flowFiles.get(i);
        final String filename = ff.getAttribute("filename");

        if (filename.equals("testFile.txt")) {
            ff.assertAttributeEquals("path", "/test");
        } else if ( filename.equals("1.txt")) {
            ff.assertAttributeEquals("path", "/test/testDir");
        } else {
            Assert.fail("filename was " + filename);
        }
    }
}
 

@Test
public void testZip() throws IOException {
    final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
    final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent());
    unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
    autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
    unpackRunner.enqueue(dataPath.resolve("data.zip"));
    unpackRunner.enqueue(dataPath.resolve("data.zip"));
    Map<String, String> attributes = new HashMap<>(1);
    attributes.put("mime.type", "application/zip");
    autoUnpackRunner.enqueue(dataPath.resolve("data.zip"), attributes);
    autoUnpackRunner.enqueue(dataPath.resolve("data.zip"), attributes);
    unpackRunner.run(2);
    autoUnpackRunner.run(2);

    unpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4);
    unpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2);
    unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2");
    unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2");
    unpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0);

    autoUnpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4);
    autoUnpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2);
    autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2");
    autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2");
    autoUnpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0);

    final List<MockFlowFile> unpacked = unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS);
    for (final MockFlowFile flowFile : unpacked) {
        final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
        final String folder = flowFile.getAttribute(CoreAttributes.PATH.key());
        final Path path = dataPath.resolve(folder).resolve(filename);
        assertTrue(Files.exists(path));

        flowFile.assertContentEquals(path.toFile());
    }
}
 

@Test
public void testTar() throws IOException {
    final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
    final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent());
    unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.TAR_FORMAT.toString());
    autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
    unpackRunner.enqueue(dataPath.resolve("data.tar"));
    unpackRunner.enqueue(dataPath.resolve("data.tar"));
    Map<String, String> attributes = new HashMap<>(1);
    Map<String, String> attributes2 = new HashMap<>(1);
    attributes.put("mime.type", UnpackContent.PackageFormat.TAR_FORMAT.getMimeType());
    attributes2.put("mime.type", UnpackContent.PackageFormat.X_TAR_FORMAT.getMimeType());
    autoUnpackRunner.enqueue(dataPath.resolve("data.tar"), attributes);
    autoUnpackRunner.enqueue(dataPath.resolve("data.tar"), attributes2);
    unpackRunner.run(2);
    autoUnpackRunner.run(2);

    unpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4);
    unpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2);
    unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2");
    unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2");
    unpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0);

    autoUnpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4);
    autoUnpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2);
    autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2");
    autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2");
    autoUnpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0);

    final List<MockFlowFile> unpacked = unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS);
    for (final MockFlowFile flowFile : unpacked) {
        final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
        final String folder = flowFile.getAttribute(CoreAttributes.PATH.key());
        final Path path = dataPath.resolve(folder).resolve(filename);
        assertTrue(Files.exists(path));

        flowFile.assertContentEquals(path.toFile());
    }
}
 

@Test
public void testSmallEchoPutToAttribute() throws Exception {
    File dummy = new File("src/test/resources/hello.txt");
    assertTrue(dummy.exists());
    final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
    controller.enqueue("".getBytes());

    if(isWindows()) {
        controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "cmd.exe");
        controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "/c;echo Hello");
        controller.setProperty(ExecuteStreamCommand.ARG_DELIMITER, ";");
    } else{
        controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "echo");
        controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "Hello");
    }
    controller.setProperty(ExecuteStreamCommand.IGNORE_STDIN, "true");
    controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");

    controller.run(1);
    controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
    controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);

    List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
    MockFlowFile outputFlowFile = flowFiles.get(0);
    outputFlowFile.assertContentEquals("");
    String ouput = outputFlowFile.getAttribute("executeStreamCommand.output");
    assertTrue(ouput.startsWith("Hello"));
    assertEquals("0", outputFlowFile.getAttribute("execution.status"));
    assertEquals(isWindows() ? "cmd.exe" : "echo", outputFlowFile.getAttribute("execution.command"));
}
 

@Test
public void specifyJsonParametersInExpressionLanguage() throws Exception {
    runner.assertValid();
    runner.setProperty(InferAvroSchema.INPUT_CONTENT_TYPE, InferAvroSchema.USE_MIME_TYPE);

    // Purposely set to True to test that none of the JSON file is read which would cause issues.
    runner.setProperty(InferAvroSchema.GET_CSV_HEADER_DEFINITION_FROM_INPUT, "true");
    runner.setProperty(InferAvroSchema.SCHEMA_DESTINATION, InferAvroSchema.DESTINATION_ATTRIBUTE);
    runner.setProperty(InferAvroSchema.RECORD_NAME, "${record.name}");
    runner.setProperty(InferAvroSchema.NUM_RECORDS_TO_ANALYZE, "${records.analyze}");

    Map<String, String> attributes = new HashMap<>();
    attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json");
    attributes.put("record.name", "myrecord");
    attributes.put("records.analyze", "2");
    runner.enqueue(new File("src/test/resources/Shapes.json").toPath(), attributes);

    runner.run();
    runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0);
    runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0);
    runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1);
    runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1);

    MockFlowFile data = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0);
    String avroSchema = data.getAttribute(InferAvroSchema.AVRO_SCHEMA_ATTRIBUTE_NAME);
    Assert.assertTrue(avroSchema.contains("\"name\" : \"myrecord\""));

    // Since that avro schema is written to an attribute this should be teh same as the original
    data.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json");
}
 

@Test
public void testAttributesRegex() throws IOException {
    final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV());
    testRunner.setProperty(AttributesToCSV.DESTINATION, OUTPUT_NEW_ATTRIBUTE);
    testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "false");
    testRunner.setProperty(AttributesToCSV.ATTRIBUTES_REGEX, "${myRegEx}");
    testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false");

    Map<String, String> attrs = new HashMap<String, String>(){{
        put("beach-name", "Malibu Beach");
        put("beach-location", "California, US");
        put("beach-endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim");
        put("attribute-should-be-eliminated", "This should not be in CSVData!");
        put("myRegEx", "beach-.*");
    }};

    testRunner.enqueue(new byte[0], attrs);
    testRunner.run();

    List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS);

    testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0);
    testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1);

    MockFlowFile flowFile = flowFilesForRelationship.get(0);

    assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));

    final String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME);

    Set<String> CSVDataValues = new HashSet<>(getStrings(attributeData));

    assertEquals(3, CSVDataValues.size());

    assertTrue(CSVDataValues.contains("Malibu Beach"));
    assertTrue(CSVDataValues.contains("\"California, US\""));
    assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\""));


    assertTrue(!CSVDataValues.contains(flowFile.getAttribute("filename")));
    assertTrue(!CSVDataValues.contains(flowFile.getAttribute("path")));
    assertTrue(!CSVDataValues.contains(flowFile.getAttribute("uuid")));
}
 

@Test
public void testExecuteJarToAttributeConfigurationDyanmicProperties() throws Exception {
    File exJar = new File("src/test/resources/ExecuteCommand/TestSuccess.jar");
    String jarPath = exJar.getAbsolutePath();
    exJar.setExecutable(true);
    final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
    controller.enqueue("small test".getBytes());
    controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
    controller.setProperty(ExecuteStreamCommand.ARGUMENTS_STRATEGY, ExecuteStreamCommand.DYNAMIC_PROPERTY_ARGUMENTS_STRATEGY.getValue());
    PropertyDescriptor dynamicProp1 = new PropertyDescriptor.Builder()
        .dynamic(true)
        .name("command.argument.1")
        .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
        .build();
    controller.setProperty(dynamicProp1, "-jar");
    PropertyDescriptor dynamicProp2 = new PropertyDescriptor.Builder()
        .dynamic(true)
        .name("command.argument.2")
        .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
        .build();
    controller.setProperty(dynamicProp2, jarPath);
    controller.setProperty(ExecuteStreamCommand.PUT_ATTRIBUTE_MAX_LENGTH, "10");
    controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "outputDest");
    assertEquals(1, controller.getProcessContext().getAvailableRelationships().size());
    controller.run(1);
    controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
    controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0);
    controller.assertTransferCount(ExecuteStreamCommand.NONZERO_STATUS_RELATIONSHIP, 0);

    List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
    MockFlowFile outputFlowFile = flowFiles.get(0);
    outputFlowFile.assertContentEquals("small test".getBytes());
    String result = outputFlowFile.getAttribute("outputDest");
    assertTrue(Pattern.compile("Test was a").matcher(result).find());
    assertEquals("0", outputFlowFile.getAttribute("execution.status"));
    assertEquals("java", outputFlowFile.getAttribute("execution.command"));
    assertEquals("-jar", outputFlowFile.getAttribute("execution.command.args").substring(0,4));
    String attribute = outputFlowFile.getAttribute("execution.command.args");
    String expected = "src" + File.separator + "test" + File.separator + "resources" + File.separator + "ExecuteCommand" + File.separator + "TestSuccess.jar";
    assertEquals(expected, attribute.substring(attribute.length() - expected.length()));
}
 

@Test
public void testAttrListWithCommasInNameFromExpCoreNullOffToAttribute() throws IOException {
    final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV());
    testRunner.setProperty(AttributesToCSV.DESTINATION, OUTPUT_NEW_ATTRIBUTE);
    testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "true");
    testRunner.setProperty(AttributesToCSV.ATTRIBUTES_LIST, "${myAttribs}");
    testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false");


    Map<String, String> attrsCommaInName = new HashMap<String, String>(){{
        put("beach,name", "Malibu Beach");
        put("beach,location", "California, US");
        put("beach,endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim");
        put("attribute-should-be-eliminated", "This should not be in CSVData!");
        put("myAttribs", "\"beach,name\",\"beach,location\",\"beach,endorsement\"");
    }};

    testRunner.enqueue(new byte[0], attrsCommaInName);
    testRunner.run();

    List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS);

    testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0);
    testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1);

    //Test flow file 0 with ATTRIBUTE_LIST populated from expression language
    MockFlowFile flowFile = flowFilesForRelationship.get(0);

    assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));

    String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME);

    Set<String> CSVDataValues = new HashSet<>(getStrings(attributeData));

    assertEquals(6, CSVDataValues.size());

    assertTrue(CSVDataValues.contains("Malibu Beach"));
    assertTrue(CSVDataValues.contains("\"California, US\""));
    assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\""));

    assertTrue(CSVDataValues.contains(flowFile.getAttribute("filename")));
    assertTrue(CSVDataValues.contains(flowFile.getAttribute("path")));
    assertTrue(CSVDataValues.contains(flowFile.getAttribute("uuid")));

    //Test flow file 1 with ATTRIBUTE_LIST populated from expression language containing commas (output should be he same)
    flowFile = flowFilesForRelationship.get(0);

    assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));

    attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME);

    CSVDataValues = new HashSet<>(getStrings(attributeData));

    assertEquals(6, CSVDataValues.size());

    assertTrue(CSVDataValues.contains("Malibu Beach"));
    assertTrue(CSVDataValues.contains("\"California, US\""));
    assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\""));

    assertTrue(CSVDataValues.contains(flowFile.getAttribute("filename")));
    assertTrue(CSVDataValues.contains(flowFile.getAttribute("path")));
    assertTrue(CSVDataValues.contains(flowFile.getAttribute("uuid")));

}
 

@Test
public void testRecursiveWithCustomFilterFullPathWithSchemeAndAuthority() throws InterruptedException, IOException {
    // set custom regex filter and filter mode
    runner.setProperty(ListHDFS.FILE_FILTER, "hdfs://hdfscluster:8020(/.*/)*anotherDir/1\\..*");
    runner.setProperty(ListHDFS.FILE_FILTER_MODE, FILTER_FULL_PATH_VALUE.getValue());

    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testFile.out")));
    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testFile.txt")));

    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test"),
            new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir")));
    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/1.txt")));

    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir"),
            new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir")));
    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/1.out")));
    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/1.txt")));
    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/2.out")));
    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/2.txt")));

    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir"),
            new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/someDir")));
    proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/someDir"),
            new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/someDir/1.out")));

    // first iteration will not pick up files because it has to instead check timestamps.
    // We must then wait long enough to ensure that the listing can be performed safely and
    // run the Processor again.
    runner.run();
    Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS));
    runner.run();

    runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 2);

    final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListHDFS.REL_SUCCESS);
    for (int i = 0; i < 2; i++) {
        final MockFlowFile ff = flowFiles.get(i);
        final String filename = ff.getAttribute("filename");

        if (filename.equals("1.out")) {
            ff.assertAttributeEquals("path", "/test/testDir/anotherDir");
        } else if (filename.equals("1.txt")) {
            ff.assertAttributeEquals("path", "/test/testDir/anotherDir");
        } else {
            Assert.fail("filename was " + filename);
        }
    }
}