Java Code Examples for org.apache.nifi.util.TestRunner#assertAllFlowFilesContainAttribute()

The following examples show how to use org.apache.nifi.util.TestRunner#assertAllFlowFilesContainAttribute() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestGetSolr.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testInitialDateFilter() throws IOException, SolrServerException, ParseException {
    final Date dateToFilter = DATE_FORMAT.parse(DATE_STRING_LATER);
    final org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor proc = new org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor(solrClient);

    TestRunner runner = createDefaultTestRunner(proc);
    runner.setProperty(GetSolr.DATE_FILTER, DATE_FORMAT.format(dateToFilter));
    runner.setProperty(GetSolr.BATCH_SIZE, "1");

    SolrInputDocument doc10 = new SolrInputDocument();
    doc10.addField("id", "doc10");
    doc10.addField("created", dateToFilter);
    SolrInputDocument doc11 = new SolrInputDocument();
    doc11.addField("id", "doc11");
    doc11.addField("created", dateToFilter);

    solrClient.add(doc10);
    solrClient.add(doc11);
    solrClient.commit();

    runner.run(1,true, true);
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 2);
    runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key());
}
Example 2
Source File: TestGetSolr.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testForValidXml() throws IOException, SolrServerException, InitializationException {
    final org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor proc = new org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor(solrClient);

    TestRunner runner = createDefaultTestRunner(proc);
    runner.setProperty(GetSolr.SOLR_QUERY, "id:doc1");
    runner.setProperty(GetSolr.RETURN_FIELDS, "id");
    runner.setProperty(GetSolr.BATCH_SIZE, "10");

    runner.run(1,true, true);
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 1);
    runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key());

    String expectedXml = "<docs><doc><field name=\"id\">doc1</field></doc></docs>";
    assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(GetSolr.REL_SUCCESS).get(0)))));
}
Example 3
Source File: TestGetSolr.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCompletenessDespiteUpdates() throws IOException, SolrServerException {
    final org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor proc = new org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor(solrClient);

    TestRunner runner = createDefaultTestRunner(proc);
    runner.setProperty(GetSolr.BATCH_SIZE, "1");

    runner.run(1,false, true);
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 10);
    runner.clearTransferState();

    SolrInputDocument doc0 = new SolrInputDocument();
    doc0.addField("id", "doc0");
    doc0.addField("created", new Date());
    SolrInputDocument doc1 = new SolrInputDocument();
    doc1.addField("id", "doc1");
    doc1.addField("created", new Date());

    solrClient.add(doc0);
    solrClient.add(doc1);
    solrClient.commit();

    runner.run(1,true, false);
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 2);
    runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key());
}
Example 4
Source File: TestGetSolr.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCompletenessDespiteDeletions() throws IOException, SolrServerException {
    final org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor proc = new org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor(solrClient);

    TestRunner runner = createDefaultTestRunner(proc);
    runner.setProperty(GetSolr.BATCH_SIZE, "1");

    runner.run(1,false, true);
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 10);
    runner.clearTransferState();

    SolrInputDocument doc10 = new SolrInputDocument();
    doc10.addField("id", "doc10");
    doc10.addField("created", new Date());
    SolrInputDocument doc11 = new SolrInputDocument();
    doc11.addField("id", "doc11");
    doc11.addField("created", new Date());

    solrClient.add(doc10);
    solrClient.add(doc11);
    solrClient.deleteById("doc0");
    solrClient.deleteById("doc1");
    solrClient.deleteById("doc2");
    solrClient.commit();

    runner.run(1,true, false);
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 2);
    runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key());
}
Example 5
Source File: TestFTP.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void basicFileList() throws IOException, InterruptedException {
    FileSystem results = fakeFtpServer.getFileSystem();

    FileEntry sampleFile = new FileEntry("c:\\data\\randombytes-2");
    sampleFile.setContents("Just some random test test test chocolate");
    results.add(sampleFile);

    // Check file exists
    Assert.assertTrue(results.exists("c:\\data\\randombytes-2"));

    TestRunner runner = TestRunners.newTestRunner(ListFTP.class);
    runner.setProperty(ListFTP.HOSTNAME, "localhost");
    runner.setProperty(ListFTP.USERNAME, username);
    runner.setProperty(FTPTransfer.PASSWORD, password);
    runner.setProperty(FTPTransfer.PORT, Integer.toString(ftpPort));
    runner.setProperty(ListFTP.REMOTE_PATH, "/");
    // FakeFTPServer has timestamp precision in minutes.
    // Specify milliseconds precision so that test does not need to wait for minutes.
    runner.setProperty(ListFile.TARGET_SYSTEM_TIMESTAMP_PRECISION, ListFile.PRECISION_MILLIS);
    runner.assertValid();

    // Ensure wait for enough lag time.
    Thread.sleep(AbstractListProcessor.LISTING_LAG_MILLIS.get(TimeUnit.MILLISECONDS) * 2);

    runner.run();

    runner.assertTransferCount(FetchFTP.REL_SUCCESS, 1);
    final MockFlowFile retrievedFile = runner.getFlowFilesForRelationship(FetchFTP.REL_SUCCESS).get(0);
    runner.assertAllFlowFilesContainAttribute("ftp.remote.host");
    runner.assertAllFlowFilesContainAttribute("ftp.remote.port");
    runner.assertAllFlowFilesContainAttribute("ftp.listing.user");
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_OWNER_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_GROUP_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_PERMISSIONS_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_SIZE_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_LAST_MODIFY_TIME_ATTRIBUTE);
    retrievedFile.assertAttributeEquals("ftp.listing.user", username);
    retrievedFile.assertAttributeEquals("filename", "randombytes-2");
}
Example 6
Source File: TestValidateXml.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testInvalid() throws IOException, SAXException {
    final TestRunner runner = TestRunners.newTestRunner(new ValidateXml());
    runner.setProperty(ValidateXml.SCHEMA_FILE, "src/test/resources/TestXml/XmlBundle.xsd");

    runner.enqueue("<this>is an invalid</xml>");
    runner.run();

    runner.assertAllFlowFilesTransferred(ValidateXml.REL_INVALID, 1);
    runner.assertAllFlowFilesContainAttribute(ValidateXml.REL_INVALID, ValidateXml.ERROR_ATTRIBUTE_KEY);
}
Example 7
Source File: TestValidateXml.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test(expected = AssertionError.class)
public void testInvalidEL() throws IOException, SAXException {
    final TestRunner runner = TestRunners.newTestRunner(new ValidateXml());
    runner.setProperty(ValidateXml.SCHEMA_FILE, "${my.schema}");

    runner.enqueue("<this>is an invalid</xml>");
    runner.run();

    runner.assertAllFlowFilesTransferred(ValidateXml.REL_INVALID, 1);
    runner.assertAllFlowFilesContainAttribute(ValidateXml.REL_INVALID, ValidateXml.ERROR_ATTRIBUTE_KEY);
}
Example 8
Source File: TestListSFTP.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void basicFileList() throws InterruptedException {
    TestRunner runner = TestRunners.newTestRunner(ListSFTP.class);
    runner.setProperty(ListSFTP.HOSTNAME, "localhost");
    runner.setProperty(ListSFTP.USERNAME, username);
    runner.setProperty(SFTPTransfer.PASSWORD, password);
    runner.setProperty(FTPTransfer.PORT, Integer.toString(port));
    runner.setProperty(ListSFTP.REMOTE_PATH, "/directory/");

    runner.setProperty(ListFile.TARGET_SYSTEM_TIMESTAMP_PRECISION, ListFile.PRECISION_MILLIS);
    runner.assertValid();

    // Ensure wait for enough lag time.
    Thread.sleep(AbstractListProcessor.LISTING_LAG_MILLIS.get(TimeUnit.MILLISECONDS) * 2);

    runner.run();

    runner.assertTransferCount(ListSFTP.REL_SUCCESS, 3);

    runner.assertAllFlowFilesContainAttribute("sftp.remote.host");
    runner.assertAllFlowFilesContainAttribute("sftp.remote.port");
    runner.assertAllFlowFilesContainAttribute("sftp.listing.user");
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_OWNER_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_GROUP_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_PERMISSIONS_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_SIZE_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute(ListFile.FILE_LAST_MODIFY_TIME_ATTRIBUTE);
    runner.assertAllFlowFilesContainAttribute( "filename");

    final MockFlowFile retrievedFile = runner.getFlowFilesForRelationship(ListSFTP.REL_SUCCESS).get(0);
    retrievedFile.assertAttributeEquals("sftp.listing.user", username);
}
Example 9
Source File: TestGetSolr.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRecordWriter() throws IOException, InitializationException {
    final org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor proc = new org.apache.nifi.processors.solr.TestGetSolr.TestableProcessor(solrClient);

    TestRunner runner = createDefaultTestRunner(proc);
    runner.setProperty(GetSolr.RETURN_TYPE, GetSolr.MODE_REC.getValue());
    runner.setProperty(GetSolr.RETURN_FIELDS, "id,created,integer_single");
    runner.setProperty(GetSolr.BATCH_SIZE, "10");

    final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/test-schema.avsc")));

    final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
    runner.addControllerService("writer", jsonWriter);
    runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
    runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
    runner.setProperty(jsonWriter, "Pretty Print JSON", "true");
    runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute");
    runner.enableControllerService(jsonWriter);
    runner.setProperty(SolrUtils.RECORD_WRITER, "writer");

    runner.run(1,true, true);
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 1);
    runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key());

    // Check for valid json
    JsonReader reader = new JsonReader(new InputStreamReader(new ByteArrayInputStream(
            runner.getContentAsByteArray(runner.getFlowFilesForRelationship(GetSolr.REL_SUCCESS).get(0)))));
    reader.beginArray();
    int controlScore = 0;
    while (reader.hasNext()) {
        reader.beginObject();
        while (reader.hasNext()) {
            if (reader.nextName().equals("integer_single"))
                controlScore += reader.nextInt();
            else
                reader.skipValue();
        }
        reader.endObject();
    }
    assertEquals(controlScore, 45);
}
Example 10
Source File: QuerySolrIT.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRelationshipRoutings() throws IOException {
    SolrClient solrClient = createSolrClient();
    TestRunner runner = createRunnerWithSolrClient(solrClient);

    runner.setProperty("facet", "true");
    runner.setProperty("stats", "true");

    // Set request handler for request failure
    runner.setProperty(QuerySolr.SOLR_PARAM_REQUEST_HANDLER, "/nonexistentrequesthandler");

    // Processor has no input connection and fails
    runner.setNonLoopConnection(false);
    runner.run(1, false);
    runner.assertAllFlowFilesTransferred(QuerySolr.FAILURE, 1);

    MockFlowFile flowFile = runner.getFlowFilesForRelationship(QuerySolr.FAILURE).get(0);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION_MESSAGE);
    runner.clearTransferState();

    // Processor has an input connection and fails
    runner.setNonLoopConnection(true);
    runner.enqueue(new byte[0]);
    runner.run(1, false);
    runner.assertAllFlowFilesTransferred(QuerySolr.FAILURE, 1);

    flowFile = runner.getFlowFilesForRelationship(QuerySolr.FAILURE).get(0);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION_MESSAGE);
    runner.clearTransferState();

    // Set request handler for successful request
    runner.setProperty(QuerySolr.SOLR_PARAM_REQUEST_HANDLER, "/select");

    // Processor has no input connection and succeeds
    runner.setNonLoopConnection(false);
    runner.run(1, false);
    runner.assertTransferCount(QuerySolr.RESULTS, 1);
    runner.assertTransferCount(QuerySolr.FACETS, 1);
    runner.assertTransferCount(QuerySolr.STATS, 1);

    flowFile = runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    runner.clearTransferState();

    // Processor has an input connection and succeeds
    runner.setNonLoopConnection(true);
    runner.enqueue(new byte[0]);
    runner.run(1, true);
    runner.assertTransferCount(QuerySolr.RESULTS, 1);
    runner.assertTransferCount(QuerySolr.FACETS, 1);
    runner.assertTransferCount(QuerySolr.STATS, 1);
    runner.assertTransferCount(QuerySolr.ORIGINAL, 1);
    runner.assertAllFlowFilesContainAttribute(QuerySolr.ATTRIBUTE_SOLR_CONNECT);

    flowFile = runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    flowFile = runner.getFlowFilesForRelationship(QuerySolr.FACETS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    flowFile = runner.getFlowFilesForRelationship(QuerySolr.STATS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    runner.clearTransferState();

    solrClient.close();
}
Example 11
Source File: TestQuerySolr.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRelationshipRoutings() throws IOException {
    SolrClient solrClient = createSolrClient();
    TestRunner runner = createRunnerWithSolrClient(solrClient);

    runner.setProperty("facet", "true");
    runner.setProperty("stats", "true");

    // Set request handler for request failure
    runner.setProperty(QuerySolr.SOLR_PARAM_REQUEST_HANDLER, "/nonexistentrequesthandler");

    // Processor has no input connection and fails
    runner.setNonLoopConnection(false);
    runner.run(1, false);
    runner.assertAllFlowFilesTransferred(QuerySolr.FAILURE, 1);

    MockFlowFile flowFile = runner.getFlowFilesForRelationship(QuerySolr.FAILURE).get(0);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION_MESSAGE);
    runner.clearTransferState();

    // Processor has an input connection and fails
    runner.setNonLoopConnection(true);
    runner.enqueue(new byte[0]);
    runner.run(1, false);
    runner.assertAllFlowFilesTransferred(QuerySolr.FAILURE, 1);

    flowFile = runner.getFlowFilesForRelationship(QuerySolr.FAILURE).get(0);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION);
    flowFile.assertAttributeExists(QuerySolr.EXCEPTION_MESSAGE);
    runner.clearTransferState();

    // Set request handler for successful request
    runner.setProperty(QuerySolr.SOLR_PARAM_REQUEST_HANDLER, "/select");

    // Processor has no input connection and succeeds
    runner.setNonLoopConnection(false);
    runner.run(1, false);
    runner.assertTransferCount(QuerySolr.RESULTS, 1);
    runner.assertTransferCount(QuerySolr.FACETS, 1);
    runner.assertTransferCount(QuerySolr.STATS, 1);

    flowFile = runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    runner.clearTransferState();

    // Processor has an input connection and succeeds
    runner.setNonLoopConnection(true);
    runner.enqueue(new byte[0]);
    runner.run(1, true);
    runner.assertTransferCount(QuerySolr.RESULTS, 1);
    runner.assertTransferCount(QuerySolr.FACETS, 1);
    runner.assertTransferCount(QuerySolr.STATS, 1);
    runner.assertTransferCount(QuerySolr.ORIGINAL, 1);
    runner.assertAllFlowFilesContainAttribute(QuerySolr.ATTRIBUTE_SOLR_CONNECT);

    flowFile = runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    flowFile = runner.getFlowFilesForRelationship(QuerySolr.FACETS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    flowFile = runner.getFlowFilesForRelationship(QuerySolr.STATS).get(0);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_CONNECT);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_CURSOR_MARK);
    flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
    runner.clearTransferState();

    solrClient.close();
}