Java Code Examples for org.apache.nifi.components.state.Scope

The following examples show how to use org.apache.nifi.components.state.Scope. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: nifi   Source File: TestGetHBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnRemovedClearsState() throws IOException {
    final long now = System.currentTimeMillis();

    final Map<String, String> cells = new HashMap<>();
    cells.put("greeting", "hello");
    cells.put("name", "nifi");

    hBaseClient.addResult("row0", cells, now - 2);
    hBaseClient.addResult("row1", cells, now - 1);
    hBaseClient.addResult("row2", cells, now - 1);
    hBaseClient.addResult("row3", cells, now);

    runner.run(100);
    runner.assertAllFlowFilesTransferred(GetHBase.REL_SUCCESS, 4);

    // should have a local state file and a cache entry before removing
    runner.getStateManager().assertStateSet(Scope.CLUSTER);

    proc.onRemoved(runner.getProcessContext());

    // onRemoved should have cleared both
    Assert.assertFalse(proc.getStateFile().exists());
    Assert.assertFalse(cacheClient.containsKey(proc.getKey(), new StringSerDe()));
}
 
Example 2
Source Project: nifi   Source File: TestGetHDFSEvents.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldProperlyHandleANullEventBatch() throws Exception {
    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(null);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path${now()}");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(0, successfulFlowFiles.size());
    assertEquals("-1", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example 3
Source Project: nifi   Source File: TestMonitorActivity.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testClusterMonitorActiveCopyAttribute() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new TestableProcessor(TimeUnit.MINUTES.toMillis(120)));
    runner.setClustered(true);
    runner.setPrimaryNode(false);
    runner.setProperty(MonitorActivity.MONITORING_SCOPE, MonitorActivity.SCOPE_CLUSTER);
    // This has to be very small threshold, otherwise, MonitorActivity skip persisting state.
    runner.setProperty(MonitorActivity.THRESHOLD, "1 ms");
    runner.setProperty(MonitorActivity.COPY_ATTRIBUTES, "true");

    final HashMap<String, String> attributes = new HashMap<>();
    attributes.put("key1", "value1");
    attributes.put("key2", "value2");
    runner.enqueue("Incoming data", attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(MonitorActivity.REL_SUCCESS);

    final StateMap updatedState = runner.getStateManager().getState(Scope.CLUSTER);
    assertNotNull("Latest timestamp should be persisted", updatedState.get(MonitorActivity.STATE_KEY_LATEST_SUCCESS_TRANSFER));
    assertEquals("value1", updatedState.get("key1"));
    assertEquals("value2", updatedState.get("key2"));
}
 
Example 4
Source Project: nifi   Source File: ListGCSBucketTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyList() throws Exception {
    reset(storage, mockBlobPage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Iterable<Blob> mockList = ImmutableList.of();

    when(mockBlobPage.getValues()).thenReturn(mockList);
    when(mockBlobPage.getNextPage()).thenReturn(null);
    when(storage.list(anyString(), any(Storage.BlobListOption[].class))).thenReturn(mockBlobPage);

    runner.enqueue("test");
    runner.run();

    runner.assertTransferCount(ListGCSBucket.REL_SUCCESS, 0);

    assertEquals("No state should be persisted on an empty return", -1L, runner.getStateManager().getState(Scope.CLUSTER).getVersion());
}
 
Example 5
Source Project: localization_nifi   Source File: TestGetHBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnRemovedClearsState() throws IOException {
    final long now = System.currentTimeMillis();

    final Map<String, String> cells = new HashMap<>();
    cells.put("greeting", "hello");
    cells.put("name", "nifi");

    hBaseClient.addResult("row0", cells, now - 2);
    hBaseClient.addResult("row1", cells, now - 1);
    hBaseClient.addResult("row2", cells, now - 1);
    hBaseClient.addResult("row3", cells, now);

    runner.run(100);
    runner.assertAllFlowFilesTransferred(GetHBase.REL_SUCCESS, 4);

    // should have a local state file and a cache entry before removing
    runner.getStateManager().assertStateSet(Scope.CLUSTER);

    proc.onRemoved(runner.getProcessContext());

    // onRemoved should have cleared both
    Assert.assertFalse(proc.getStateFile().exists());
    Assert.assertFalse(cacheClient.containsKey(proc.getKey(), new StringSerDe()));
}
 
Example 6
Source Project: localization_nifi   Source File: TestGetHDFSEvents.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldOnlyProcessEventsWithSpecificPath() throws Exception {
    Event[] events = getEvents();

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/create(/)?");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(1, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example 7
Source Project: localization_nifi   Source File: TestMonitorActivity.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testClusterMonitorActive() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new TestableProcessor(100));
    runner.setClustered(true);
    runner.setPrimaryNode(false);
    runner.setProperty(MonitorActivity.MONITORING_SCOPE, MonitorActivity.SCOPE_CLUSTER);
    // This has to be very small threshold, otherwise, MonitorActivity skip persisting state.
    runner.setProperty(MonitorActivity.THRESHOLD, "1 ms");

    runner.enqueue("Incoming data");

    runner.run();

    runner.assertAllFlowFilesTransferred(MonitorActivity.REL_SUCCESS);

    final StateMap updatedState = runner.getStateManager().getState(Scope.CLUSTER);
    assertNotNull("Latest timestamp should be persisted", updatedState.get(MonitorActivity.STATE_KEY_LATEST_SUCCESS_TRANSFER));
    // Should be null because COPY_ATTRIBUTES is null.
    assertNull(updatedState.get("key1"));
    assertNull(updatedState.get("key2"));
}
 
Example 8
Source Project: nifi-scripting-samples   Source File: TestState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Demonstrates reading and writing processor state values
 * @throws Exception
 */
@Test
public void testStatePython() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new ExecuteScript());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(SCRIPT_ENGINE, "python");
    runner.setProperty(ScriptingComponentUtils.SCRIPT_FILE, "src/test/resources/executescript/state/state.py");
    runner.setProperty(ScriptingComponentUtils.MODULES, "src/test/resources/executescript");
    runner.assertValid();

    StateManager stateManager = runner.getStateManager();
    stateManager.clear(Scope.CLUSTER);
    Map<String, String> initialStateValues = new HashMap<>();
    initialStateValues.put("some-state", "foo");
    stateManager.setState(initialStateValues, Scope.CLUSTER);

    runner.enqueue("sample text".getBytes(StandardCharsets.UTF_8));
    runner.run();

    runner.assertAllFlowFilesTransferred("success", 1);
    StateMap resultStateValues = stateManager.getState(Scope.CLUSTER);
    Assert.assertEquals("foobar", resultStateValues.get("some-state"));
}
 
Example 9
Source Project: nifi   Source File: GetSplunk.java    License: Apache License 2.0 6 votes vote down vote up
private TimeRange loadState(StateManager stateManager) throws IOException {
    final StateMap stateMap = stateManager.getState(Scope.CLUSTER);

    if (stateMap.getVersion() < 0) {
        getLogger().debug("No previous state found");
        return null;
    }

    final String earliest = stateMap.get(EARLIEST_TIME_KEY);
    final String latest = stateMap.get(LATEST_TIME_KEY);
    getLogger().debug("Loaded state with earliestTime of {} and latestTime of {}", new Object[] {earliest, latest});

    if (StringUtils.isBlank(earliest) && StringUtils.isBlank(latest)) {
        return null;
    } else {
        return new TimeRange(earliest, latest);
    }
}
 
Example 10
Source Project: nifi   Source File: ListGCSBucket.java    License: Apache License 2.0 6 votes vote down vote up
void persistState(final ProcessContext context, final long timestamp, final Set<String> keys) {
    final Map<String, String> state = new HashMap<>();
    state.put(CURRENT_TIMESTAMP, String.valueOf(timestamp));

    int i = 0;
    for (final String key : keys) {
        state.put(CURRENT_KEY_PREFIX+i, key);
        i++;
    }

    try {
        context.getStateManager().setState(state, Scope.CLUSTER);
    } catch (IOException ioe) {
        getLogger().error("Failed to save cluster-wide state. If NiFi is restarted, data duplication may occur", ioe);
    }
}
 
Example 11
Source Project: localization_nifi   Source File: ListGCSBucketTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRestoreFreshState() throws Exception {
    reset(storage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    assertEquals("Cluster StateMap should be fresh (version -1L)",
            -1L,
            runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).getVersion()
    );

    assertNull(processor.currentKeys);

    processor.restoreState(runner.getProcessContext());

    assertNotNull(processor.currentKeys);
    assertEquals(
            0L,
            processor.currentTimestamp
    );

    assertTrue(processor.currentKeys.isEmpty());

}
 
Example 12
Source Project: localization_nifi   Source File: GetHBase.java    License: Apache License 2.0 5 votes vote down vote up
private ScanResult getState(final StateManager stateManager) throws IOException {
    final StateMap stateMap = stateManager.getState(Scope.CLUSTER);
    if (stateMap.getVersion() < 0) {
        return null;
    }

    return ScanResult.fromFlatMap(stateMap.toMap());
}
 
Example 13
Source Project: nifi   Source File: GetHTTP.java    License: Apache License 2.0 5 votes vote down vote up
@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
    if (clearState.getAndSet(false)) {
        context.getStateManager().clear(Scope.LOCAL);
    }
    if (customHeaders.size() == 0) {
        for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
            // only add the custom defined Headers (i.e. dynamic properties)
            if (!getSupportedPropertyDescriptors().contains(property.getKey())) {
                customHeaders.add(property.getKey());
            }
        }
    }
}
 
Example 14
Source Project: localization_nifi   Source File: ScrollElasticsearchHttp.java    License: Apache License 2.0 5 votes vote down vote up
private String loadScrollId(StateManager stateManager) throws IOException {
    final StateMap stateMap = stateManager.getState(Scope.LOCAL);

    if (stateMap.getVersion() < 0) {
        getLogger().debug("No previous state found");
        return null;
    }

    final String scrollId = stateMap.get(SCROLL_ID_STATE);
    getLogger().debug("Loaded state with scrollId {}", new Object[] { scrollId });

    return scrollId;
}
 
Example 15
Source Project: localization_nifi   Source File: MockStateManager.java    License: Apache License 2.0 5 votes vote down vote up
private void verifyAnnotation(final Scope scope) {
    // ensure that the @Stateful annotation is present with the appropriate Scope
    if ((scope == Scope.LOCAL && !usesLocalState) || (scope == Scope.CLUSTER && !usesClusterState)) {
        Assert.fail("Component is attempting to set or retrieve state with a scope of " + scope + " but does not declare that it will use "
            + scope + " state. A @Stateful annotation should be added to the component with a scope of " + scope);
    }
}
 
Example 16
Source Project: nifi   Source File: ScrollElasticsearchHttp.java    License: Apache License 2.0 5 votes vote down vote up
private String loadScrollId(StateManager stateManager) throws IOException {
    final StateMap stateMap = stateManager.getState(Scope.LOCAL);

    if (stateMap.getVersion() < 0) {
        getLogger().debug("No previous state found");
        return null;
    }

    final String scrollId = stateMap.get(SCROLL_ID_STATE);
    getLogger().debug("Loaded state with scrollId {}", new Object[] { scrollId });

    return scrollId;
}
 
Example 17
Source Project: localization_nifi   Source File: ListS3.java    License: Apache License 2.0 5 votes vote down vote up
private void persistState(final ProcessContext context) {
    Map<String, String> state = new HashMap<>();
    state.put(CURRENT_TIMESTAMP, String.valueOf(currentTimestamp));
    int i = 0;
    for (String key : currentKeys) {
        state.put(CURRENT_KEY_PREFIX+i, key);
        i++;
    }
    try {
        context.getStateManager().setState(state, Scope.CLUSTER);
    } catch (IOException ioe) {
        getLogger().error("Failed to save cluster-wide state. If NiFi is restarted, data duplication may occur", ioe);
    }
}
 
Example 18
Source Project: localization_nifi   Source File: MockStateManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Specifies whether or not the State Manager should throw an IOException when state is retrieved for the given scope.
 *
 * @param scope the scope that should (or should not) fail
 * @param fail whether or not retrieving state should fail
 */
public void setFailOnStateGet(final Scope scope, final boolean fail) {
    if (scope == Scope.LOCAL) {
        failToGetLocalState = fail;
    } else {
        failToGetClusterState = fail;
    }
}
 
Example 19
Source Project: localization_nifi   Source File: GetSplunk.java    License: Apache License 2.0 5 votes vote down vote up
@OnRemoved
public void onRemoved(final ProcessContext context) {
    try {
        context.getStateManager().clear(Scope.CLUSTER);
    } catch (IOException e) {
       getLogger().error("Unable to clear processor state due to {}", new Object[] {e.getMessage()}, e);
    }
}
 
Example 20
Source Project: nifi   Source File: QueryDatabaseTableTest.java    License: Apache License 2.0 5 votes vote down vote up
@After
public void teardown() throws IOException {
    runner.getStateManager().clear(Scope.CLUSTER);
    runner = null;
    QueryDatabaseTable.dbAdapters.clear();
    QueryDatabaseTable.dbAdapters.putAll(origDbAdapters);
}
 
Example 21
Source Project: localization_nifi   Source File: GetHTTP.java    License: Apache License 2.0 5 votes vote down vote up
@OnScheduled
public void onScheduled(final ProcessContext context) throws IOException {
    if (clearState.getAndSet(false)) {
        context.getStateManager().clear(Scope.LOCAL);
    }
    if (customHeaders.size() == 0) {
        for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
            // only add the custom defined Headers (i.e. dynamic properties)
            if (!getSupportedPropertyDescriptors().contains(property.getKey())) {
                customHeaders.add(property.getKey());
            }
        }
    }
}
 
Example 22
Source Project: localization_nifi   Source File: ListFile.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Scope getStateScope(final ProcessContext context) {
    final String location = context.getProperty(DIRECTORY_LOCATION).getValue();
    if (LOCATION_REMOTE.getValue().equalsIgnoreCase(location)) {
        return Scope.CLUSTER;
    }

    return Scope.LOCAL;
}
 
Example 23
Source Project: localization_nifi   Source File: MonitorActivity.java    License: Apache License 2.0 5 votes vote down vote up
@OnStopped
public void onStopped(final ProcessContext context) {
    if (getNodeTypeProvider().isPrimary()) {
        final StateManager stateManager = context.getStateManager();
        try {
            stateManager.clear(Scope.CLUSTER);
        } catch (IOException e) {
            getLogger().error("Failed to clear cluster state due to " + e, e);
        }
    }
}
 
Example 24
@Test
public void testStateStoredInClusterStateManagement() throws Exception {
    final ConcreteListProcessor proc = new ConcreteListProcessor();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    final DistributedCache cache = new DistributedCache();
    runner.addControllerService("cache", cache);
    runner.enableControllerService(cache);
    runner.setProperty(AbstractListProcessor.DISTRIBUTED_CACHE_SERVICE, "cache");

    final long initialTimestamp = System.nanoTime();

    proc.addEntity("name", "id", initialTimestamp);
    runner.run();

    final Map<String, String> expectedState = new HashMap<>();
    // Ensure only timestamp is migrated
    expectedState.put(AbstractListProcessor.LISTING_TIMESTAMP_KEY, String.valueOf(initialTimestamp));
    expectedState.put(AbstractListProcessor.PROCESSED_TIMESTAMP_KEY, "0");
    runner.getStateManager().assertStateEquals(expectedState, Scope.CLUSTER);

    Thread.sleep(DEFAULT_SLEEP_MILLIS);

    runner.run();
    // Ensure only timestamp is migrated
    expectedState.put(AbstractListProcessor.LISTING_TIMESTAMP_KEY, String.valueOf(initialTimestamp));
    expectedState.put(AbstractListProcessor.PROCESSED_TIMESTAMP_KEY, String.valueOf(initialTimestamp));
    runner.getStateManager().assertStateEquals(expectedState, Scope.CLUSTER);
}
 
Example 25
@Test
public void testStateMigratedFromLocalFile() throws Exception {
    final ConcreteListProcessor proc = new ConcreteListProcessor();
    final TestRunner runner = TestRunners.newTestRunner(proc);

    // Create a file that we will populate with the desired state
    File persistenceFile = testFolder.newFile(proc.persistenceFilename);
    // Override the processor's internal persistence file
    proc.persistenceFile = persistenceFile;

    // Local File persistence was a properties file format of <key>=<JSON entity listing representation>
    // Our ConcreteListProcessor is centered around files which are provided for a given path
    final String serviceState = proc.getPath(runner.getProcessContext()) + "={\"latestTimestamp\":1492,\"matchingIdentifiers\":[\"id\"]}";

    // Create a persistence file of the format anticipated
    try (FileOutputStream fos = new FileOutputStream(persistenceFile);) {
        fos.write(serviceState.getBytes(Charsets.UTF_8));
    }

    runner.run();

    // Verify the local persistence file is removed
    Assert.assertTrue("Failed to remove persistence file", !persistenceFile.exists());

    // Verify the state manager now maintains the associated state
    final Map<String, String> expectedState = new HashMap<>();
    // Ensure only timestamp is migrated
    expectedState.put(AbstractListProcessor.LISTING_TIMESTAMP_KEY, "1492");
    expectedState.put(AbstractListProcessor.PROCESSED_TIMESTAMP_KEY, "1492");
    runner.getStateManager().assertStateEquals(expectedState, Scope.CLUSTER);
}
 
Example 26
Source Project: nifi   Source File: StandardStateManager.java    License: Apache License 2.0 5 votes vote down vote up
private StateProvider getProvider(final Scope scope) {
    if (scope == Scope.LOCAL || clusterProvider == null || !clusterProvider.isEnabled()) {
        return localProvider;
    }

    return clusterProvider;
}
 
Example 27
Source Project: nifi   Source File: ListGCSBucketTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPersistState() throws Exception {
    reset(storage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    assertEquals("Cluster StateMap should be fresh (version -1L)",
            -1L,
            runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).getVersion()
    );

    final Set<String> keys = ImmutableSet.of("test-key-0", "test-key-1");
    processor.persistState(runner.getProcessContext(), 4L, keys);

    final StateMap stateMap = runner.getStateManager().getState(Scope.CLUSTER);
    assertEquals("Cluster StateMap should have been written to", 1L, stateMap.getVersion());

    assertEquals(
            ImmutableMap.of(
                    ListGCSBucket.CURRENT_TIMESTAMP, String.valueOf(4L),
                    ListGCSBucket.CURRENT_KEY_PREFIX+"0", "test-key-0",
                    ListGCSBucket.CURRENT_KEY_PREFIX+"1", "test-key-1"
            ),
            stateMap.toMap()
    );
}
 
Example 28
Source Project: nifi   Source File: TestAbstractListProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNoStateToMigrate() throws Exception {

    runner.run();

    final MockStateManager stateManager = runner.getStateManager();
    final Map<String, String> expectedState = new HashMap<>();
    stateManager.assertStateEquals(expectedState, Scope.CLUSTER);
}
 
Example 29
Source Project: nifi   Source File: TestEnforceOrder.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMaxOrder() {
    final TestRunner runner = TestRunners.newTestRunner(EnforceOrder.class);

    runner.setProperty(EnforceOrder.GROUP_IDENTIFIER, "${fragment.identifier}");
    runner.setProperty(EnforceOrder.ORDER_ATTRIBUTE, "index");
    runner.setProperty(EnforceOrder.INITIAL_ORDER, "1");
    runner.setProperty(EnforceOrder.MAX_ORDER, "${fragment.count}");
    runner.assertValid();
    runner.enqueue("b.1", Ordered.i(1).put("fragment.identifier", "b").put("fragment.count", "3").map());
    runner.enqueue("a.2", Ordered.i(2).put("fragment.identifier", "a").put("fragment.count", "2").map());
    runner.enqueue("without max order", Ordered.i(1).put("fragment.identifier", "c").map());
    runner.enqueue("illegal max order", Ordered.i(1).put("fragment.identifier", "d").put("fragment.count", "X").map());
    runner.enqueue("a.1", Ordered.i(1).put("fragment.identifier", "a").put("fragment.count", "2").map());
    runner.enqueue("a.3", Ordered.i(3).put("fragment.identifier", "a").put("fragment.count", "2").map()); // Exceed max

    runner.run();

    final List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    succeeded.sort(new FirstInFirstOutPrioritizer());
    assertEquals(3, succeeded.size());
    succeeded.get(0).assertContentEquals("a.1");
    succeeded.get(1).assertContentEquals("a.2");
    succeeded.get(2).assertContentEquals("b.1");

    final List<MockFlowFile> failed = runner.getFlowFilesForRelationship(EnforceOrder.REL_FAILURE);
    assertEquals(3, failed.size());
    failed.get(0).assertContentEquals("without max order");
    failed.get(1).assertContentEquals("illegal max order");
    failed.get(2).assertContentEquals("a.3"); // exceeds max order

    final MockStateManager stateManager = runner.getStateManager();
    stateManager.assertStateEquals("a.target", "2", Scope.LOCAL);
    stateManager.assertStateEquals("a.max", "2", Scope.LOCAL);
}
 
Example 30
Source Project: localization_nifi   Source File: TestGenerateTableFetch.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBackwardsCompatibilityStateKeyStaticTableDynamicMaxValues() throws Exception {
    // load test data to database
    final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection();
    Statement stmt = con.createStatement();

    try {
        stmt.execute("drop table TEST_QUERY_DB_TABLE");
    } catch (final SQLException sqle) {
        // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842]
    }

    stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, bucket integer not null)");
    stmt.execute("insert into TEST_QUERY_DB_TABLE (id, bucket) VALUES (0, 0)");
    stmt.execute("insert into TEST_QUERY_DB_TABLE (id, bucket) VALUES (1, 0)");

    runner.setProperty(GenerateTableFetch.TABLE_NAME, "TEST_QUERY_DB_TABLE");
    runner.setIncomingConnection(true);
    runner.setProperty(GenerateTableFetch.MAX_VALUE_COLUMN_NAMES, "${maxValueCol}");
    runner.enqueue("".getBytes(), new HashMap<String, String>() {{
        put("maxValueCol", "id");
    }});

    // Pre-populate the state with a key for column name (not fully-qualified)
    StateManager stateManager = runner.getStateManager();
    stateManager.setState(new HashMap<String, String>() {{
        put("id", "0");
    }}, Scope.CLUSTER);

    // Pre-populate the column type map with an entry for id (not fully-qualified)
    processor.columnTypeMap.put("id", 4);

    runner.run();

    runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
    MockFlowFile flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(0);
    assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE WHERE id > 0 ORDER BY id FETCH NEXT 10000 ROWS ONLY", new String(flowFile.toByteArray()));
}