org.apache.nifi.provenance.search.QueryResult Java Examples

The following examples show how to use org.apache.nifi.provenance.search.QueryResult. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    logger.info("{} got {} hits", query, result.getTotalHitCount());
    return result;
}
 
Example #2
Source File: VolatileProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }

    return result;
}
 
Example #3
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of not having any event
 * records while having correct headers
 */
@Test
public void testWithWithEventFileMissingRecord() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);

    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.writeUTF("BlahBlah");
    in.writeInt(4);
    in.close();
    assertTrue(eventFile.exists());
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #4
Source File: PersistentProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    logger.info("{} got {} hits", query, result.getTotalHitCount());
    return result;
}
 
Example #5
Source File: MiNiFiPersistentProvenanceRepository.java    From nifi-minifi with Apache License 2.0 6 votes vote down vote up
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    logger.info("{} got {} hits", query, result.getTotalHitCount());
    return result;
}
 
Example #6
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of not having any event
 * records while having correct headers
 */
@Test
public void testWithWithEventFileMissingRecord() throws Exception {
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);

    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.writeUTF("BlahBlah");
    in.writeInt(4);
    in.close();
    assertTrue(eventFile.exists());
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #7
Source File: VolatileProvenanceRepository.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }

    return result;
}
 
Example #8
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final ProvenanceEventRecord event = createEvent();

    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));

    final Query query = new Query(UUID.randomUUID().toString());

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example #9
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final ProvenanceEventRecord event = createEvent();

    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));

    final Query query = new Query(UUID.randomUUID().toString());

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(4000, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example #10
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #11
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #12
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L * 10);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L * 10);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "10000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
 
Example #13
Source File: ControllerFacade.java    From nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Retrieves the results of a provenance query.
 *
 * @param provenanceId id
 * @return the results of a provenance query
 */
public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) {
    try {
        // get the query to the provenance repository
        final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
        final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser());

        // ensure the query results could be found
        if (querySubmission == null) {
            throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged.");
        }

        // get the original query and the results
        final Query query = querySubmission.getQuery();
        final QueryResult queryResult = querySubmission.getResult();

        // build the response
        final ProvenanceDTO provenanceDto = new ProvenanceDTO();
        final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
        final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO();

        // include the original request and results
        provenanceDto.setRequest(requestDto);
        provenanceDto.setResults(resultsDto);

        // convert the original request
        requestDto.setStartDate(query.getStartDate());
        requestDto.setEndDate(query.getEndDate());
        requestDto.setMinimumFileSize(query.getMinFileSize());
        requestDto.setMaximumFileSize(query.getMaxFileSize());
        requestDto.setMaxResults(query.getMaxResults());
        if (query.getSearchTerms() != null) {
            final Map<String, String> searchTerms = new HashMap<>();
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
            }
            requestDto.setSearchTerms(searchTerms);
        }

        // convert the provenance
        provenanceDto.setId(query.getIdentifier());
        provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime());
        provenanceDto.setExpiration(queryResult.getExpiration());
        provenanceDto.setFinished(queryResult.isFinished());
        provenanceDto.setPercentCompleted(queryResult.getPercentComplete());

        // convert each event
        final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults);
        if (includeResults || queryResult.isFinished()) {
            final List<ProvenanceEventDTO> events = new ArrayList<>();
            for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) {
                events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize)));
            }
            resultsDto.setProvenanceEvents(events);
        }

        if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
            resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
            resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
        } else {
            resultsDto.setTotalCount(queryResult.getTotalHitCount());
            resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
        }

        // include any errors
        if (queryResult.getError() != null) {
            final Set<String> errors = new HashSet<>();
            errors.add(queryResult.getError());
            resultsDto.setErrors(errors);
        }

        // set the generated timestamp
        final Date now = new Date();
        resultsDto.setGenerated(now);
        resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime()));

        // get the oldest available event time
        final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1);
        if (!firstEvent.isEmpty()) {
            resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime()));
        }

        provenanceDto.setResults(resultsDto);
        return provenanceDto;
    } catch (final IOException ioe) {
        throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe);
    }
}
 
Example #14
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 50000)
public void testQuerySpecificField() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP);

    // add 2 events, one of which we will query for.
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L));

    // Create a query that searches for the event with the FlowFile UUID equal to the first event's.
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid()));

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example #15
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 50000)
public void testQuerySpecificField() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP);

    // add 2 events, one of which we will query for.
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L));

    // Create a query that searches for the event with the FlowFile UUID equal to the first event's.
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid()));

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(4000, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example #16
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example #17
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 10000)
public void testNotAuthorizedQuery() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
        @Override
        public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) {
            return event.getEventId() > 2;
        }
    };

    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query("1234");
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234"));
    final QuerySubmission submission = repo.submitQuery(query, createUser());

    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        Thread.sleep(100L);
    }

    // Ensure that we gets events with ID's 3 through 10.
    final List<ProvenanceEventRecord> events = result.getMatchingEvents();
    assertEquals(7, events.size());
    final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList());
    for (int i = 0; i < 7; i++) {
        Assert.assertEquals(i + 3, eventIds.get(i).intValue());
    }
}
 
Example #18
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
@Ignore("This test relies too much on timing of background events by using Thread.sleep().")
public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    Thread.sleep(2000L);

    final FileFilter indexFileFilter = file -> file.getName().startsWith("index");
    final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length;
    assertEquals(1, numIndexDirs);

    // add more records so that we will create a new index
    final long secondBatchStartTime = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(System.currentTimeMillis());
        repo.registerEvent(builder.build());
    }

    // wait for indexing to happen
    repo.waitForRollover();

    // verify we get the results expected
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(20, result.getMatchingEvents().size());

    // Ensure index directories exists
    File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(2, indexDirs.length);

    // expire old events and indexes
    final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime;
    config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertEquals(10, newRecordSet.getMatchingEvents().size());

    // Ensure that one index directory is gone
    indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(1, indexDirs.length);
}
 
Example #19
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexOnRolloverWithImmenseAttribute() throws IOException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test
    StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
    for (int i=0; i < immenseAttrSize; i++) {
        immenseBldr.append('0');
    }
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    attributes.put("immense", immenseBldr.toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #20
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example #21
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    // Give time for rollover to happen
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();

    Thread.sleep(1000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
 
Example #22
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
@Ignore("This test relies too much on timing of background events by using Thread.sleep().")
public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    Thread.sleep(2000L);

    final FileFilter indexFileFilter = file -> file.getName().startsWith("index");
    final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length;
    assertEquals(1, numIndexDirs);

    // add more records so that we will create a new index
    final long secondBatchStartTime = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(System.currentTimeMillis());
        repo.registerEvent(builder.build());
    }

    // wait for indexing to happen
    repo.waitForRollover();

    // verify we get the results expected
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(20, result.getMatchingEvents().size());

    // Ensure index directories exists
    File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(2, indexDirs.length);

    // expire old events and indexes
    final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime;
    config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertEquals(10, newRecordSet.getMatchingEvents().size());

    // Ensure that one index directory is gone
    indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(1, indexDirs.length);
}
 
Example #23
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example #24
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexOnRolloverWithImmenseAttribute() throws IOException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test
    StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
    for (int i = 0; i < immenseAttrSize; i++) {
        immenseBldr.append('0');
    }
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    attributes.put("immense", immenseBldr.toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #25
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example #26
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L * 10);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L * 10);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "10000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
 
Example #27
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    // Give time for rollover to happen
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();

    Thread.sleep(1000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
 
Example #28
Source File: ControllerFacade.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Retrieves the results of a provenance query.
 *
 * @param provenanceId id
 * @return the results of a provenance query
 */
public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) {
    try {
        // get the query to the provenance repository
        final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
        final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser());

        // ensure the query results could be found
        if (querySubmission == null) {
            throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged.");
        }

        // get the original query and the results
        final Query query = querySubmission.getQuery();
        final QueryResult queryResult = querySubmission.getResult();

        // build the response
        final ProvenanceDTO provenanceDto = new ProvenanceDTO();
        final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
        final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO();

        // include the original request and results
        provenanceDto.setRequest(requestDto);
        provenanceDto.setResults(resultsDto);

        // convert the original request
        requestDto.setStartDate(query.getStartDate());
        requestDto.setEndDate(query.getEndDate());
        requestDto.setMinimumFileSize(query.getMinFileSize());
        requestDto.setMaximumFileSize(query.getMaxFileSize());
        requestDto.setMaxResults(query.getMaxResults());
        if (query.getSearchTerms() != null) {
            final Map<String, String> searchTerms = new HashMap<>();
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
            }
            requestDto.setSearchTerms(searchTerms);
        }

        // convert the provenance
        provenanceDto.setId(query.getIdentifier());
        provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime());
        provenanceDto.setExpiration(queryResult.getExpiration());
        provenanceDto.setFinished(queryResult.isFinished());
        provenanceDto.setPercentCompleted(queryResult.getPercentComplete());

        // convert each event
        final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults);
        if (includeResults || queryResult.isFinished()) {
            final List<ProvenanceEventDTO> events = new ArrayList<>();
            for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) {
                events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize)));
            }
            resultsDto.setProvenanceEvents(events);
        }

        if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
            resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
            resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
        } else {
            resultsDto.setTotalCount(queryResult.getTotalHitCount());
            resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
        }

        // include any errors
        if (queryResult.getError() != null) {
            final Set<String> errors = new HashSet<>();
            errors.add(queryResult.getError());
            resultsDto.setErrors(errors);
        }

        // set the generated timestamp
        final Date now = new Date();
        resultsDto.setGenerated(now);
        resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime()));

        // get the oldest available event time
        final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1);
        if (!firstEvent.isEmpty()) {
            resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime()));
        }

        provenanceDto.setResults(resultsDto);
        return provenanceDto;
    } catch (final IOException ioe) {
        throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe);
    }
}
 
Example #29
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 10000)
public void testNotAuthorizedQuery() throws IOException, InterruptedException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
        @Override
        public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) {
            return event.getEventId() > 2;
        }
    };

    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query("1234");
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234"));
    final QuerySubmission submission = repo.submitQuery(query, createUser());

    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        Thread.sleep(100L);
    }

    // Ensure that we gets events with ID's 3 through 10.
    final List<ProvenanceEventRecord> events = result.getMatchingEvents();
    assertEquals(7, events.size());
    final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList());
    for (int i = 0; i < 7; i++) {
        Assert.assertEquals(i + 3, eventIds.get(i).intValue());
    }
}