Java Code Examples for org.apache.nifi.util.MockFlowFile#getAttribute()
The following examples show how to use
org.apache.nifi.util.MockFlowFile#getAttribute() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ConvertExcelToCSVProcessorTest.java From nifi with Apache License 2.0 | 6 votes |
@Test public void testDataFormatting() throws Exception { testRunner.enqueue(new File("src/test/resources/dataformatting.xlsx").toPath()); testRunner.setProperty(ConvertExcelToCSVProcessor.FORMAT_VALUES, "false"); testRunner.run(); testRunner.assertTransferCount(ConvertExcelToCSVProcessor.SUCCESS, 1); testRunner.assertTransferCount(ConvertExcelToCSVProcessor.ORIGINAL, 1); testRunner.assertTransferCount(ConvertExcelToCSVProcessor.FAILURE, 0); MockFlowFile ff = testRunner.getFlowFilesForRelationship(ConvertExcelToCSVProcessor.SUCCESS).get(0); Long rowsSheet = new Long(ff.getAttribute(ConvertExcelToCSVProcessor.ROW_NUM)); assertTrue(rowsSheet == 9); ff.assertContentEquals("Numbers,Timestamps,Money\n" + "1234.4559999999999,42736.5,123.45\n" + "1234.4559999999999,42736.5,123.45\n" + "1234.4559999999999,42736.5,123.45\n" + "1234.4559999999999,42736.5,1023.45\n" + "1234.4559999999999,42736.5,1023.45\n" + "987654321,42736.5,1023.45\n" + "987654321,,\n" + "987654321,,\n"); }
Example 2
Source File: TestEvaluateXQuery.java From nifi with Apache License 2.0 | 6 votes |
@Test public void testMatchesMultipleXmlAttribute() throws XPathFactoryConfigurationException, IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("some.property", "//fruit/name"); testRunner.enqueue(XML_SNIPPET); testRunner.run(); testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); for (int i = 0; i < fruitNames.length; i++) { final String outXml = out.getAttribute("some.property." + ((int) i + 1)); String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><name xmlns:ns=\"http://namespace/1\">" + fruitNames[i] + "</name>"; assertEquals(expectedXml, outXml.trim()); } testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0).assertContentEquals(XML_SNIPPET); }
Example 3
Source File: TestUnpackContent.java From localization_nifi with Apache License 2.0 | 6 votes |
@Test public void testFlowFileStreamV2() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new UnpackContent()); runner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.FLOWFILE_STREAM_FORMAT_V2.toString()); runner.enqueue(dataPath.resolve("data.flowfilev2")); runner.enqueue(dataPath.resolve("data.flowfilev2")); runner.run(2); runner.assertTransferCount(UnpackContent.REL_SUCCESS, 4); runner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2); runner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2"); runner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2"); runner.assertTransferCount(UnpackContent.REL_FAILURE, 0); final List<MockFlowFile> unpacked = runner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS); for (final MockFlowFile flowFile : unpacked) { final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key()); final String folder = flowFile.getAttribute(CoreAttributes.PATH.key()); final Path path = dataPath.resolve(folder).resolve(filename); assertTrue(Files.exists(path)); flowFile.assertContentEquals(path.toFile()); } }
Example 4
Source File: TestInferAvroSchema.java From nifi with Apache License 2.0 | 5 votes |
@Test public void inferAvroSchemaFromJSONFile() throws Exception { runner.assertValid(); runner.setProperty(InferAvroSchema.INPUT_CONTENT_TYPE, InferAvroSchema.USE_MIME_TYPE); // Purposely set to True to test that none of the JSON file is read which would cause issues. runner.setProperty(InferAvroSchema.GET_CSV_HEADER_DEFINITION_FROM_INPUT, "true"); runner.setProperty(InferAvroSchema.SCHEMA_DESTINATION, InferAvroSchema.DESTINATION_ATTRIBUTE); Map<String, String> attributes = new HashMap<>(); attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json"); runner.enqueue(new File("src/test/resources/Shapes.json").toPath(), attributes); runner.run(); runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0); runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0); runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1); runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1); MockFlowFile data = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0); String avroSchema = data.getAttribute(InferAvroSchema.AVRO_SCHEMA_ATTRIBUTE_NAME); String knownSchema = new String(unix2PlatformSpecificLineEndings(new File("src/test/resources/Shapes.json.avro")), StandardCharsets.UTF_8); Assert.assertEquals(avroSchema, knownSchema); // Since that avro schema is written to an attribute this should be teh same as the original data.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json"); }
Example 5
Source File: TestEvaluateXQuery.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testWriteXmlToAttribute() throws XPathFactoryConfigurationException, IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name"); testRunner.enqueue(XML_SNIPPET); testRunner.run(); testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); final String outXml = out.getAttribute("some.property"); assertTrue(outXml.contains("<name xmlns:ns=\"http://namespace/1\">apple</name>")); testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0).assertContentEquals(XML_SNIPPET); }
Example 6
Source File: TestExecuteStreamCommand.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testExecuteJarPutToAttribute() throws Exception { File exJar = new File("src/test/resources/ExecuteCommand/TestSuccess.jar"); File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt"); String jarPath = exJar.getAbsolutePath(); exJar.setExecutable(true); final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class); controller.enqueue(dummy.toPath()); controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java"); controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath); controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output"); controller.run(1); controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1); controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0); List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP); MockFlowFile outputFlowFile = flowFiles.get(0); String result = outputFlowFile.getAttribute("executeStreamCommand.output"); outputFlowFile.assertContentEquals(dummy); assertTrue(Pattern.compile("Test was a success\r?\n").matcher(result).find()); assertEquals("0", outputFlowFile.getAttribute("execution.status")); assertEquals("java", outputFlowFile.getAttribute("execution.command")); assertEquals("-jar;", outputFlowFile.getAttribute("execution.command.args").substring(0, 5)); String attribute = outputFlowFile.getAttribute("execution.command.args"); String expected = "src" + File.separator + "test" + File.separator + "resources" + File.separator + "ExecuteCommand" + File.separator + "TestSuccess.jar"; assertEquals(expected, attribute.substring(attribute.length() - expected.length())); }
Example 7
Source File: TestInvokeHttpCommon.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void test401NotAuth() throws Exception { addHandler(new BasicAuthHandler()); final String username = "basic_user"; final String password = "basic_password"; runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/401"); runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_USERNAME, username); runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_PASSWORD, password); createFlowFiles(runner); runner.run(); runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 0); runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 0); runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0); runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 1); runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0); runner.assertPenalizeCount(0); // expected in request status.code and status.message // original flow file (+attributes) final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_NO_RETRY).get(0); bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "401"); bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "Unauthorized"); bundle.assertAttributeEquals("Foo", "Bar"); final String actual = new String(bundle.toByteArray(), StandardCharsets.UTF_8); final String expected = "Hello"; Assert.assertEquals(expected, actual); final String response = bundle.getAttribute(InvokeHTTP.RESPONSE_BODY); assertEquals(response, "Get off my lawn!"+System.lineSeparator()); }
Example 8
Source File: TestGetHDFSEvents.java From nifi with Apache License 2.0 | 5 votes |
@Test public void makeSureExpressionLanguageIsWorkingProperlyWithinTheHdfsPathToWatch() throws Exception { Event[] events = new Event[] { new Event.CreateEvent.Builder().path("/some/path/1/2/3/t.txt").build(), new Event.CreateEvent.Builder().path("/some/path/1/2/4/t.txt").build(), new Event.CreateEvent.Builder().path("/some/path/1/2/3/.t.txt").build() }; EventBatch eventBatch = mock(EventBatch.class); when(eventBatch.getEvents()).thenReturn(events); when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch); when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream); when(eventBatch.getTxid()).thenReturn(100L); GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin); TestRunner runner = TestRunners.newTestRunner(processor); runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/${literal(1)}/${literal(2)}/${literal(3)}/.*.txt"); runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create"); runner.setProperty(GetHDFSEvents.IGNORE_HIDDEN_FILES, "true"); runner.run(); List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS); assertEquals(1, successfulFlowFiles.size()); for (MockFlowFile f : successfulFlowFiles) { String eventType = f.getAttribute(EventAttributes.EVENT_TYPE); assertTrue(eventType.equals("CREATE")); } verify(eventBatch).getTxid(); assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id")); }
Example 9
Source File: ConsumeJMSTest.java From solace-integration-guides with Apache License 2.0 | 5 votes |
@Test public void validateSuccessfulConsumeAndTransferToSuccess() throws Exception { final String destinationName = "cooQueue"; JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false); JMSPublisher sender = new JMSPublisher(jmsTemplate, mock(ComponentLog.class)); final Map<String, String> senderAttributes = new HashMap<>(); senderAttributes.put("filename", "message.txt"); senderAttributes.put("attribute_from_sender", "some value"); sender.publish(destinationName, "Hey dude!".getBytes(), senderAttributes); TestRunner runner = TestRunners.newTestRunner(new ConsumeJMS()); JMSConnectionFactoryProviderDefinition cs = mock(JMSConnectionFactoryProviderDefinition.class); when(cs.getIdentifier()).thenReturn("cfProvider"); when(cs.getConnectionFactory()).thenReturn(jmsTemplate.getConnectionFactory()); runner.addControllerService("cfProvider", cs); runner.enableControllerService(cs); runner.setProperty(PublishJMS.CF_SERVICE, "cfProvider"); runner.setProperty(ConsumeJMS.DESTINATION, destinationName); runner.setProperty(ConsumeJMS.DESTINATION_TYPE, ConsumeJMS.QUEUE); runner.run(1, false); // final MockFlowFile successFF = runner.getFlowFilesForRelationship(PublishJMS.REL_SUCCESS).get(0); assertNotNull(successFF); successFF.assertAttributeExists(JmsHeaders.DESTINATION); successFF.assertAttributeEquals(JmsHeaders.DESTINATION, destinationName); successFF.assertAttributeExists("filename"); successFF.assertAttributeEquals("filename", "message.txt"); successFF.assertAttributeExists("attribute_from_sender"); successFF.assertAttributeEquals("attribute_from_sender", "some value"); successFF.assertContentEquals("Hey dude!".getBytes()); String sourceDestination = successFF.getAttribute(ConsumeJMS.JMS_SOURCE_DESTINATION_NAME); assertNotNull(sourceDestination); ((CachingConnectionFactory) jmsTemplate.getConnectionFactory()).destroy(); }
Example 10
Source File: TestAttributesToCSV.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testNoAttrListCoreNullOffToAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV()); testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "true"); testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false"); Map<String, String> attrs = new HashMap<String, String>(){{ put("beach-name", "Malibu Beach"); put("beach-location", "California, US"); put("beach-endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim"); }}; testRunner.enqueue(new byte[0], attrs); testRunner.run(); List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS); testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0); testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1); MockFlowFile flowFile = flowFilesForRelationship.get(0); assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key())); final String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME); Set<String> csvAttributeValues = new HashSet<>(getStrings(attributeData)); assertEquals(6, csvAttributeValues.size()); assertTrue(csvAttributeValues.contains("Malibu Beach")); assertTrue(csvAttributeValues.contains("\"California, US\"")); assertTrue(csvAttributeValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\"")); assertTrue(csvAttributeValues.contains(flowFile.getAttribute("filename"))); assertTrue(csvAttributeValues.contains(flowFile.getAttribute("path"))); assertTrue(csvAttributeValues.contains(flowFile.getAttribute("uuid"))); }
Example 11
Source File: TestAttributesToCSV.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testAttrListNoCoreNullOffToAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV()); testRunner.setProperty(AttributesToCSV.DESTINATION, OUTPUT_NEW_ATTRIBUTE); testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "false"); testRunner.setProperty(AttributesToCSV.ATTRIBUTES_LIST, "beach-name,beach-location,beach-endorsement"); testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false"); Map<String, String> attrs = new HashMap<String, String>(){{ put("beach-name", "Malibu Beach"); put("beach-location", "California, US"); put("beach-endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim"); put("attribute-should-be-eliminated", "This should not be in CSVAttribute!"); }}; testRunner.enqueue(new byte[0], attrs); testRunner.run(); List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS); testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0); testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1); MockFlowFile flowFile = flowFilesForRelationship.get(0); assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key())); final String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME); Set<String> CSVDataValues = new HashSet<>(getStrings(attributeData)); assertEquals(3, CSVDataValues.size()); assertTrue(CSVDataValues.contains("Malibu Beach")); assertTrue(CSVDataValues.contains("\"California, US\"")); assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\"")); }
Example 12
Source File: TestListHDFS.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testRecursiveWithDefaultFilterAndFilterMode() throws InterruptedException { proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/.testFile.txt"))); proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/testFile.txt"))); proc.fileSystem.addFileStatus(new Path("/test"), new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/testDir"))); proc.fileSystem.addFileStatus(new Path("/test/testDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("/test/testDir/1.txt"))); // first iteration will not pick up files because it has to instead check timestamps. // We must then wait long enough to ensure that the listing can be performed safely and // run the Processor again. runner.run(); Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS)); runner.run(); runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 2); final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListHDFS.REL_SUCCESS); for (int i=0; i < 2; i++) { final MockFlowFile ff = flowFiles.get(i); final String filename = ff.getAttribute("filename"); if (filename.equals("testFile.txt")) { ff.assertAttributeEquals("path", "/test"); } else if ( filename.equals("1.txt")) { ff.assertAttributeEquals("path", "/test/testDir"); } else { Assert.fail("filename was " + filename); } } }
Example 13
Source File: TestUnpackContent.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testZip() throws IOException { final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent()); final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent()); unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString()); autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString()); unpackRunner.enqueue(dataPath.resolve("data.zip")); unpackRunner.enqueue(dataPath.resolve("data.zip")); Map<String, String> attributes = new HashMap<>(1); attributes.put("mime.type", "application/zip"); autoUnpackRunner.enqueue(dataPath.resolve("data.zip"), attributes); autoUnpackRunner.enqueue(dataPath.resolve("data.zip"), attributes); unpackRunner.run(2); autoUnpackRunner.run(2); unpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4); unpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2); unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2"); unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2"); unpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0); autoUnpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4); autoUnpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2); autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2"); autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2"); autoUnpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0); final List<MockFlowFile> unpacked = unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS); for (final MockFlowFile flowFile : unpacked) { final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key()); final String folder = flowFile.getAttribute(CoreAttributes.PATH.key()); final Path path = dataPath.resolve(folder).resolve(filename); assertTrue(Files.exists(path)); flowFile.assertContentEquals(path.toFile()); } }
Example 14
Source File: TestUnpackContent.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testTar() throws IOException { final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent()); final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent()); unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.TAR_FORMAT.toString()); autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString()); unpackRunner.enqueue(dataPath.resolve("data.tar")); unpackRunner.enqueue(dataPath.resolve("data.tar")); Map<String, String> attributes = new HashMap<>(1); Map<String, String> attributes2 = new HashMap<>(1); attributes.put("mime.type", UnpackContent.PackageFormat.TAR_FORMAT.getMimeType()); attributes2.put("mime.type", UnpackContent.PackageFormat.X_TAR_FORMAT.getMimeType()); autoUnpackRunner.enqueue(dataPath.resolve("data.tar"), attributes); autoUnpackRunner.enqueue(dataPath.resolve("data.tar"), attributes2); unpackRunner.run(2); autoUnpackRunner.run(2); unpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4); unpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2); unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2"); unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2"); unpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0); autoUnpackRunner.assertTransferCount(UnpackContent.REL_SUCCESS, 4); autoUnpackRunner.assertTransferCount(UnpackContent.REL_ORIGINAL, 2); autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(0).assertAttributeEquals(FRAGMENT_COUNT, "2"); autoUnpackRunner.getFlowFilesForRelationship(UnpackContent.REL_ORIGINAL).get(1).assertAttributeEquals(FRAGMENT_COUNT, "2"); autoUnpackRunner.assertTransferCount(UnpackContent.REL_FAILURE, 0); final List<MockFlowFile> unpacked = unpackRunner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS); for (final MockFlowFile flowFile : unpacked) { final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key()); final String folder = flowFile.getAttribute(CoreAttributes.PATH.key()); final Path path = dataPath.resolve(folder).resolve(filename); assertTrue(Files.exists(path)); flowFile.assertContentEquals(path.toFile()); } }
Example 15
Source File: TestExecuteStreamCommand.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testSmallEchoPutToAttribute() throws Exception { File dummy = new File("src/test/resources/hello.txt"); assertTrue(dummy.exists()); final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class); controller.enqueue("".getBytes()); if(isWindows()) { controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "cmd.exe"); controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "/c;echo Hello"); controller.setProperty(ExecuteStreamCommand.ARG_DELIMITER, ";"); } else{ controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "echo"); controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "Hello"); } controller.setProperty(ExecuteStreamCommand.IGNORE_STDIN, "true"); controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output"); controller.run(1); controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1); controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0); List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP); MockFlowFile outputFlowFile = flowFiles.get(0); outputFlowFile.assertContentEquals(""); String ouput = outputFlowFile.getAttribute("executeStreamCommand.output"); assertTrue(ouput.startsWith("Hello")); assertEquals("0", outputFlowFile.getAttribute("execution.status")); assertEquals(isWindows() ? "cmd.exe" : "echo", outputFlowFile.getAttribute("execution.command")); }
Example 16
Source File: TestInferAvroSchema.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void specifyJsonParametersInExpressionLanguage() throws Exception { runner.assertValid(); runner.setProperty(InferAvroSchema.INPUT_CONTENT_TYPE, InferAvroSchema.USE_MIME_TYPE); // Purposely set to True to test that none of the JSON file is read which would cause issues. runner.setProperty(InferAvroSchema.GET_CSV_HEADER_DEFINITION_FROM_INPUT, "true"); runner.setProperty(InferAvroSchema.SCHEMA_DESTINATION, InferAvroSchema.DESTINATION_ATTRIBUTE); runner.setProperty(InferAvroSchema.RECORD_NAME, "${record.name}"); runner.setProperty(InferAvroSchema.NUM_RECORDS_TO_ANALYZE, "${records.analyze}"); Map<String, String> attributes = new HashMap<>(); attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json"); attributes.put("record.name", "myrecord"); attributes.put("records.analyze", "2"); runner.enqueue(new File("src/test/resources/Shapes.json").toPath(), attributes); runner.run(); runner.assertTransferCount(InferAvroSchema.REL_UNSUPPORTED_CONTENT, 0); runner.assertTransferCount(InferAvroSchema.REL_FAILURE, 0); runner.assertTransferCount(InferAvroSchema.REL_ORIGINAL, 1); runner.assertTransferCount(InferAvroSchema.REL_SUCCESS, 1); MockFlowFile data = runner.getFlowFilesForRelationship(InferAvroSchema.REL_SUCCESS).get(0); String avroSchema = data.getAttribute(InferAvroSchema.AVRO_SCHEMA_ATTRIBUTE_NAME); Assert.assertTrue(avroSchema.contains("\"name\" : \"myrecord\"")); // Since that avro schema is written to an attribute this should be teh same as the original data.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json"); }
Example 17
Source File: TestAttributesToCSV.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testAttributesRegex() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV()); testRunner.setProperty(AttributesToCSV.DESTINATION, OUTPUT_NEW_ATTRIBUTE); testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "false"); testRunner.setProperty(AttributesToCSV.ATTRIBUTES_REGEX, "${myRegEx}"); testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false"); Map<String, String> attrs = new HashMap<String, String>(){{ put("beach-name", "Malibu Beach"); put("beach-location", "California, US"); put("beach-endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim"); put("attribute-should-be-eliminated", "This should not be in CSVData!"); put("myRegEx", "beach-.*"); }}; testRunner.enqueue(new byte[0], attrs); testRunner.run(); List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS); testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0); testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1); MockFlowFile flowFile = flowFilesForRelationship.get(0); assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key())); final String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME); Set<String> CSVDataValues = new HashSet<>(getStrings(attributeData)); assertEquals(3, CSVDataValues.size()); assertTrue(CSVDataValues.contains("Malibu Beach")); assertTrue(CSVDataValues.contains("\"California, US\"")); assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\"")); assertTrue(!CSVDataValues.contains(flowFile.getAttribute("filename"))); assertTrue(!CSVDataValues.contains(flowFile.getAttribute("path"))); assertTrue(!CSVDataValues.contains(flowFile.getAttribute("uuid"))); }
Example 18
Source File: TestExecuteStreamCommand.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testExecuteJarToAttributeConfigurationDyanmicProperties() throws Exception { File exJar = new File("src/test/resources/ExecuteCommand/TestSuccess.jar"); String jarPath = exJar.getAbsolutePath(); exJar.setExecutable(true); final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class); controller.enqueue("small test".getBytes()); controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java"); controller.setProperty(ExecuteStreamCommand.ARGUMENTS_STRATEGY, ExecuteStreamCommand.DYNAMIC_PROPERTY_ARGUMENTS_STRATEGY.getValue()); PropertyDescriptor dynamicProp1 = new PropertyDescriptor.Builder() .dynamic(true) .name("command.argument.1") .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .build(); controller.setProperty(dynamicProp1, "-jar"); PropertyDescriptor dynamicProp2 = new PropertyDescriptor.Builder() .dynamic(true) .name("command.argument.2") .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .build(); controller.setProperty(dynamicProp2, jarPath); controller.setProperty(ExecuteStreamCommand.PUT_ATTRIBUTE_MAX_LENGTH, "10"); controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "outputDest"); assertEquals(1, controller.getProcessContext().getAvailableRelationships().size()); controller.run(1); controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1); controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 0); controller.assertTransferCount(ExecuteStreamCommand.NONZERO_STATUS_RELATIONSHIP, 0); List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP); MockFlowFile outputFlowFile = flowFiles.get(0); outputFlowFile.assertContentEquals("small test".getBytes()); String result = outputFlowFile.getAttribute("outputDest"); assertTrue(Pattern.compile("Test was a").matcher(result).find()); assertEquals("0", outputFlowFile.getAttribute("execution.status")); assertEquals("java", outputFlowFile.getAttribute("execution.command")); assertEquals("-jar", outputFlowFile.getAttribute("execution.command.args").substring(0,4)); String attribute = outputFlowFile.getAttribute("execution.command.args"); String expected = "src" + File.separator + "test" + File.separator + "resources" + File.separator + "ExecuteCommand" + File.separator + "TestSuccess.jar"; assertEquals(expected, attribute.substring(attribute.length() - expected.length())); }
Example 19
Source File: TestAttributesToCSV.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testAttrListWithCommasInNameFromExpCoreNullOffToAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToCSV()); testRunner.setProperty(AttributesToCSV.DESTINATION, OUTPUT_NEW_ATTRIBUTE); testRunner.setProperty(AttributesToCSV.INCLUDE_CORE_ATTRIBUTES, "true"); testRunner.setProperty(AttributesToCSV.ATTRIBUTES_LIST, "${myAttribs}"); testRunner.setProperty(AttributesToCSV.NULL_VALUE_FOR_EMPTY_STRING, "false"); Map<String, String> attrsCommaInName = new HashMap<String, String>(){{ put("beach,name", "Malibu Beach"); put("beach,location", "California, US"); put("beach,endorsement", "This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim"); put("attribute-should-be-eliminated", "This should not be in CSVData!"); put("myAttribs", "\"beach,name\",\"beach,location\",\"beach,endorsement\""); }}; testRunner.enqueue(new byte[0], attrsCommaInName); testRunner.run(); List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToCSV.REL_SUCCESS); testRunner.assertTransferCount(AttributesToCSV.REL_FAILURE, 0); testRunner.assertTransferCount(AttributesToCSV.REL_SUCCESS, 1); //Test flow file 0 with ATTRIBUTE_LIST populated from expression language MockFlowFile flowFile = flowFilesForRelationship.get(0); assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key())); String attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME); Set<String> CSVDataValues = new HashSet<>(getStrings(attributeData)); assertEquals(6, CSVDataValues.size()); assertTrue(CSVDataValues.contains("Malibu Beach")); assertTrue(CSVDataValues.contains("\"California, US\"")); assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\"")); assertTrue(CSVDataValues.contains(flowFile.getAttribute("filename"))); assertTrue(CSVDataValues.contains(flowFile.getAttribute("path"))); assertTrue(CSVDataValues.contains(flowFile.getAttribute("uuid"))); //Test flow file 1 with ATTRIBUTE_LIST populated from expression language containing commas (output should be he same) flowFile = flowFilesForRelationship.get(0); assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key())); attributeData = flowFile.getAttribute(OUTPUT_ATTRIBUTE_NAME); CSVDataValues = new HashSet<>(getStrings(attributeData)); assertEquals(6, CSVDataValues.size()); assertTrue(CSVDataValues.contains("Malibu Beach")); assertTrue(CSVDataValues.contains("\"California, US\"")); assertTrue(CSVDataValues.contains("\"This is our family's favorite beach. We highly recommend it. \n\nThanks, Jim\"")); assertTrue(CSVDataValues.contains(flowFile.getAttribute("filename"))); assertTrue(CSVDataValues.contains(flowFile.getAttribute("path"))); assertTrue(CSVDataValues.contains(flowFile.getAttribute("uuid"))); }
Example 20
Source File: TestListHDFS.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testRecursiveWithCustomFilterFullPathWithSchemeAndAuthority() throws InterruptedException, IOException { // set custom regex filter and filter mode runner.setProperty(ListHDFS.FILE_FILTER, "hdfs://hdfscluster:8020(/.*/)*anotherDir/1\\..*"); runner.setProperty(ListHDFS.FILE_FILTER_MODE, FILTER_FULL_PATH_VALUE.getValue()); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testFile.out"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testFile.txt"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test"), new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/1.txt"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir"), new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/1.out"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/1.txt"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/2.out"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/anotherDir/2.txt"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir"), new FileStatus(1L, true, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/someDir"))); proc.fileSystem.addFileStatus(new Path("hdfs", "hdfscluster:8020", "/test/testDir/someDir"), new FileStatus(1L, false, 1, 1L, 0L, 0L, create777(), "owner", "group", new Path("hdfs", "hdfscluster:8020", "/test/testDir/someDir/1.out"))); // first iteration will not pick up files because it has to instead check timestamps. // We must then wait long enough to ensure that the listing can be performed safely and // run the Processor again. runner.run(); Thread.sleep(TimeUnit.NANOSECONDS.toMillis(2 * ListHDFS.LISTING_LAG_NANOS)); runner.run(); runner.assertAllFlowFilesTransferred(ListHDFS.REL_SUCCESS, 2); final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListHDFS.REL_SUCCESS); for (int i = 0; i < 2; i++) { final MockFlowFile ff = flowFiles.get(i); final String filename = ff.getAttribute("filename"); if (filename.equals("1.out")) { ff.assertAttributeEquals("path", "/test/testDir/anotherDir"); } else if (filename.equals("1.txt")) { ff.assertAttributeEquals("path", "/test/testDir/anotherDir"); } else { Assert.fail("filename was " + filename); } } }