Java Code Examples for org.apache.nifi.events.EventReporter#NO_OP

The following examples show how to use org.apache.nifi.events.EventReporter#NO_OP . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestPartitionedWriteAheadEventStore.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEventsWithStartOffsetAndCountWithNothingAuthorized() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        events.add(event);
    }

    final EventAuthorizer allowEventNumberedEventIds = EventAuthorizer.DENY_ALL;
    final List<ProvenanceEventRecord> storedEvents = store.getEvents(0, 20, allowEventNumberedEventIds, EventTransformer.EMPTY_TRANSFORMER);
    assertTrue(storedEvents.isEmpty());
}
 
Example 2
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxEventIdRestored() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    final int numEvents = 20;
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
    }

    assertEquals(19, store.getMaxEventId());
    store.close();

    final PartitionedWriteAheadEventStore recoveredStore = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    recoveredStore.initialize();
    assertEquals(19, recoveredStore.getMaxEventId());
}
 
Example 3
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleWritesThenReads() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();
    assertEquals(-1, store.getMaxEventId());

    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        assertEquals(i, store.getMaxEventId());

        events.add(event);
    }

    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord read = store.getEvent(i).get();
        assertEquals(events.get(i), read);
    }
}
 
Example 4
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleWriteThenRead() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    assertEquals(-1, store.getMaxEventId());
    final ProvenanceEventRecord event1 = createEvent();
    final StorageResult result = store.addEvents(Collections.singleton(event1));

    final StorageSummary summary = result.getStorageLocations().values().iterator().next();
    final long eventId = summary.getEventId();
    final ProvenanceEventRecord eventWithId = addId(event1, eventId);

    assertEquals(0, store.getMaxEventId());

    final ProvenanceEventRecord read = store.getEvent(eventId).get();
    assertEquals(eventWithId, read);
}
 
Example 5
Source File: TestWriteAheadStorePartition.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testInitEmptyFile() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testInitEmptyFile");
    repoConfig.setMaxEventFileCount(5);

    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();

    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };

    final RecordReaderFactory recordReaderFactory = RecordReaders::newRecordReader;

    WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
            recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }

    long maxEventId = partition.getMaxEventId();
    assertTrue(maxEventId > 0);
    partition.close();

    final List<File> fileList = Arrays.asList(storageDirectory.listFiles(DirectoryUtils.EVENT_FILE_FILTER));
    Collections.sort(fileList, DirectoryUtils.LARGEST_ID_FIRST);

    // Create new empty prov file with largest id
    assertTrue(new File(storageDirectory, "1" + fileList.get(0).getName()).createNewFile());

    partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
            recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    partition.initialize();

    assertEquals(maxEventId, partition.getMaxEventId());
}
 
Example 6
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testExpiration() throws InterruptedException, IOException {
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final List<ProvenanceEventRecord> events = new ArrayList<>();
    events.add(createEvent(500000L));
    events.add(createEvent());

    final EventStore eventStore = Mockito.mock(EventStore.class);
    Mockito.doAnswer(new Answer<List<ProvenanceEventRecord>>() {
        @Override
        public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) throws Throwable {
            final Long eventId = invocation.getArgumentAt(0, Long.class);
            assertEquals(0, eventId.longValue());
            assertEquals(1, invocation.getArgumentAt(1, Integer.class).intValue());
            return Collections.singletonList(events.get(0));
        }
    }).when(eventStore).getEvents(Mockito.anyLong(), Mockito.anyInt());

    index.initialize(eventStore);
    index.addEvent(events.get(0), createStorageSummary(events.get(0).getEventId()));

    // Add the first event to the index and wait for it to be indexed, since indexing is asynchronous.
    List<File> allDirectories = Collections.emptyList();
    while (allDirectories.isEmpty()) {
        allDirectories = index.getDirectoryManager().getDirectories(null, null);
    }

    events.remove(0); // Remove the first event from the store
    index.performMaintenance();
    assertEquals(1, index.getDirectoryManager().getDirectories(null, null).size());
}
 
Example 7
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testUnauthorizedEventsGetPlaceholdersForLineage() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);

    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }

    final NiFiUser user = createUser();

    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 3) {
        final ComputeLineageSubmission submission = index.submitLineageComputation(1L, user, EventAuthorizer.DENY_ALL);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));

        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }

    assertEquals(3, nodes.size());

    for (final LineageNode node : nodes) {
        assertEquals(LineageNodeType.PROVENANCE_EVENT_NODE, node.getNodeType());
        final ProvenanceEventLineageNode eventNode = (ProvenanceEventLineageNode) node;
        assertEquals(ProvenanceEventType.UNKNOWN, eventNode.getEventType());
    }
}
 
Example 8
Source File: TestPartitionedWriteAheadEventStore.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSize() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    long storeSize = 0L;
    final int numEvents = 20;
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        final long newSize = store.getSize();
        assertTrue(newSize > storeSize);
        storeSize = newSize;
    }
}
 
Example 9
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSize() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    long storeSize = 0L;
    final int numEvents = 20;
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        final long newSize = store.getSize();
        assertTrue(newSize > storeSize);
        storeSize = newSize;
    }
}
 
Example 10
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteAfterRecoveringRepo() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    for (int i = 0; i < 4; i++) {
        store.addEvents(Collections.singleton(createEvent()));
    }

    store.close();

    final PartitionedWriteAheadEventStore recoveredStore = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    recoveredStore.initialize();

    List<ProvenanceEventRecord> recoveredEvents = recoveredStore.getEvents(0, 10);
    assertEquals(4, recoveredEvents.size());

    // ensure that we can still write to the store
    for (int i = 0; i < 4; i++) {
        recoveredStore.addEvents(Collections.singleton(createEvent()));
    }

    recoveredEvents = recoveredStore.getEvents(0, 10);
    assertEquals(8, recoveredEvents.size());

    for (int i = 0; i < 8; i++) {
        assertEquals(i, recoveredEvents.get(i).getEventId());
    }
}
 
Example 11
Source File: ClusteredSwapFileIT.java    From nifi with Apache License 2.0 5 votes vote down vote up
protected FlowFileQueue createFlowFileQueue(final String uuid) {
    final ProcessScheduler processScheduler = getFlowController().getProcessScheduler();
    final ResourceClaimManager resourceClaimManager = getFlowController().getResourceClaimManager();
    final FlowFileSwapManager swapManager = getFlowController().createSwapManager();

    final AsyncLoadBalanceClientRegistry clientRegistry = Mockito.mock(AsyncLoadBalanceClientRegistry.class);

    return new SocketLoadBalancedFlowFileQueue(uuid, ConnectionEventListener.NOP_EVENT_LISTENER, processScheduler, getFlowFileRepository(), getProvenanceRepository(),
        getContentRepository(), resourceClaimManager, getClusterCoordinator(), clientRegistry, swapManager, 20000, EventReporter.NO_OP);
}
 
Example 12
Source File: SiteInfoProvider.java    From nifi with Apache License 2.0 5 votes vote down vote up
protected SiteToSiteRestApiClient createSiteToSiteRestApiClient(final SSLContext sslContext, final HttpProxy proxy) {
    final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(sslContext, proxy, EventReporter.NO_OP);
    apiClient.setConnectTimeoutMillis(connectTimeoutMillis);
    apiClient.setReadTimeoutMillis(readTimeoutMillis);
    apiClient.setLocalAddress(localAddress);
    apiClient.setCacheExpirationMillis(cachedContentExpirationMillis);
    return apiClient;
}
 
Example 13
Source File: SiteToSiteRestApiClient.java    From nifi with Apache License 2.0 5 votes vote down vote up
private void startExtendingTtl(final String transactionUrl, final Closeable stream, final CloseableHttpResponse response) {
    if (ttlExtendingFuture != null) {
        // Already started.
        return;
    }

    logger.debug("Starting extending TTL thread...");

    extendingApiClient = new SiteToSiteRestApiClient(sslContext, proxy, EventReporter.NO_OP);
    extendingApiClient.transportProtocolVersionNegotiator = this.transportProtocolVersionNegotiator;
    extendingApiClient.connectTimeoutMillis = this.connectTimeoutMillis;
    extendingApiClient.readTimeoutMillis = this.readTimeoutMillis;
    extendingApiClient.localAddress = this.localAddress;

    final int extendFrequency = serverTransactionTtl / 2;

    ttlExtendingFuture = ttlExtendTaskExecutor.scheduleWithFixedDelay(() -> {
        try {
            extendingApiClient.extendTransaction(transactionUrl);
        } catch (final Exception e) {
            logger.warn("Failed to extend transaction ttl", e);

            try {
                // Without disconnecting, Site-to-Site client keep reading data packet,
                // while server has already rollback.
                this.close();
            } catch (final IOException ec) {
                logger.warn("Failed to close", e);
            }
        }
    }, extendFrequency, extendFrequency, TimeUnit.SECONDS);
}
 
Example 14
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForLineage() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);

    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }

    final NiFiUser user = createUser();

    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 3) {
        final ComputeLineageSubmission submission = index.submitLineageComputation(1L, user, EventAuthorizer.DENY_ALL);
        assertTrue(submission.getResult().awaitCompletion(15, TimeUnit.SECONDS));

        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }

    assertEquals(3, nodes.size());

    for (final LineageNode node : nodes) {
        assertEquals(LineageNodeType.PROVENANCE_EVENT_NODE, node.getNodeType());
        final ProvenanceEventLineageNode eventNode = (ProvenanceEventLineageNode) node;
        assertEquals(ProvenanceEventType.UNKNOWN, eventNode.getEventType());
    }
}
 
Example 15
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testExpiration() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final List<ProvenanceEventRecord> events = new ArrayList<>();
    events.add(createEvent(500000L));
    events.add(createEvent());

    final EventStore eventStore = Mockito.mock(EventStore.class);
    Mockito.doAnswer(new Answer<List<ProvenanceEventRecord>>() {
        @Override
        public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) {
            final Long eventId = invocation.getArgument(0);
            assertEquals(0, eventId.longValue());
            assertEquals(1, invocation.<Integer>getArgument(1).intValue());
            return Collections.singletonList(events.get(0));
        }
    }).when(eventStore).getEvents(Mockito.anyLong(), Mockito.anyInt());

    index.initialize(eventStore);
    index.addEvent(events.get(0), createStorageSummary(events.get(0).getEventId()));

    // Add the first event to the index and wait for it to be indexed, since indexing is asynchronous.
    List<File> allDirectories = Collections.emptyList();
    while (allDirectories.isEmpty()) {
        allDirectories = index.getDirectoryManager().getDirectories(null, null);
    }

    events.remove(0); // Remove the first event from the store
    index.performMaintenance();
    assertEquals(1, index.getDirectoryManager().getDirectories(null, null).size());
}
 
Example 16
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final ProvenanceEventRecord event = createEvent();

    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));

    final Query query = new Query(UUID.randomUUID().toString());

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(4000, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example 17
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 50000)
public void testQuerySpecificField() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP);

    // add 2 events, one of which we will query for.
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L));

    // Create a query that searches for the event with the FlowFile UUID equal to the first event's.
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid()));

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example 18
Source File: SiteInfoProvider.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
protected SiteToSiteRestApiClient createSiteToSiteRestApiClient(final SSLContext sslContext, final HttpProxy proxy) {
    return new SiteToSiteRestApiClient(sslContext, proxy, EventReporter.NO_OP);
}
 
Example 19
Source File: TestWriteAheadStorePartition.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testReindex() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testReindex");
    repoConfig.setMaxEventFileCount(5);

    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();

    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };

    final RecordReaderFactory recordReaderFactory = RecordReaders::newRecordReader;

    final WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
        recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }

    final Map<ProvenanceEventRecord, StorageSummary> reindexedEvents = new ConcurrentHashMap<>();
    final EventIndex eventIndex = Mockito.mock(EventIndex.class);
    Mockito.doAnswer(new Answer<Object>() {
        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            final Map<ProvenanceEventRecord, StorageSummary> events = invocation.getArgument(0);
            reindexedEvents.putAll(events);
            return null;
        }
    }).when(eventIndex).reindexEvents(Mockito.anyMap());

    Mockito.doReturn(18L).when(eventIndex).getMinimumEventIdToReindex("1");
    partition.reindexLatestEvents(eventIndex);

    final List<Long> eventIdsReindexed = reindexedEvents.values().stream()
        .map(StorageSummary::getEventId)
        .sorted()
        .collect(Collectors.toList());

    assertEquals(82, eventIdsReindexed.size());
    for (int i = 0; i < eventIdsReindexed.size(); i++) {
        assertEquals(18 + i, eventIdsReindexed.get(i).intValue());
    }
}
 
Example 20
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 50000)
public void testQuerySpecificField() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP);

    // add 2 events, one of which we will query for.
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L));

    // Create a query that searches for the event with the FlowFile UUID equal to the first event's.
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid()));

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(4000, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}