org.apache.nifi.provenance.search.Query Java Examples

The following examples show how to use org.apache.nifi.provenance.search.Query. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 7 votes vote down vote up
private List<Document> runQuery(final File indexDirectory, final List<File> storageDirs, final String query) throws IOException, ParseException {
    try (final DirectoryReader directoryReader = DirectoryReader.open(FSDirectory.open(indexDirectory))) {
        final IndexSearcher searcher = new IndexSearcher(directoryReader);

        final Analyzer analyzer = new SimpleAnalyzer();
        final org.apache.lucene.search.Query luceneQuery = new QueryParser("uuid", analyzer).parse(query);

        final Query q = new Query("");
        q.setMaxResults(1000);
        final TopDocs topDocs = searcher.search(luceneQuery, 1000);

        final List<Document> docs = new ArrayList<>();
        for (final ScoreDoc scoreDoc : topDocs.scoreDocs) {
            final int docId = scoreDoc.doc;
            final Document d = directoryReader.document(docId);
            docs.add(d);
        }

        return docs;
    }
}
 
Example #2
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
private List<Document> runQuery(final File indexDirectory, final List<File> storageDirs, final String query) throws IOException, ParseException {
    assumeFalse(isWindowsEnvironment());
    try (final DirectoryReader directoryReader = DirectoryReader.open(FSDirectory.open(indexDirectory.toPath()))) {
        final IndexSearcher searcher = new IndexSearcher(directoryReader);

        final Analyzer analyzer = new SimpleAnalyzer();
        final org.apache.lucene.search.Query luceneQuery = new QueryParser("uuid", analyzer).parse(query);

        final Query q = new Query("");
        q.setMaxResults(1000);
        final TopDocs topDocs = searcher.search(luceneQuery, 1000);

        final List<Document> docs = new ArrayList<>();
        for (final ScoreDoc scoreDoc : topDocs.scoreDocs) {
            final int docId = scoreDoc.doc;
            final Document d = directoryReader.document(docId);
            docs.add(d);
        }

        return docs;
    }
}
 
Example #3
Source File: VolatileProvenanceRepository.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }

    return result;
}
 
Example #4
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of not having any event
 * records while having correct headers
 */
@Test
public void testWithWithEventFileMissingRecord() throws Exception {
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);

    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.writeUTF("BlahBlah");
    in.writeInt(4);
    in.close();
    assertTrue(eventFile.exists());
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #5
Source File: LatestEventsQuery.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public Optional<List<Long>> evaluate(final Query query) {
    if (latestRecords.getSize() < query.getMaxResults()) {
        return Optional.empty();
    }

    if (query.getSearchTerms().isEmpty() && query.getStartDate() == null && query.getEndDate() == null) {
        final List<Long> eventList = latestRecords.asList();
        if (eventList.size() > query.getMaxResults()) {
            return Optional.of(eventList.subList(0, query.getMaxResults()));
        } else {
            return Optional.of(eventList);
        }
    } else {
        return Optional.empty();
    }
}
 
Example #6
Source File: MiNiFiPersistentProvenanceRepository.java    From nifi-minifi with Apache License 2.0 6 votes vote down vote up
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    logger.info("{} got {} hits", query, result.getTotalHitCount());
    return result;
}
 
Example #7
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of not having any event
 * records while having correct headers
 */
@Test
public void testWithWithEventFileMissingRecord() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);

    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.writeUTF("BlahBlah");
    in.writeInt(4);
    in.close();
    assertTrue(eventFile.exists());
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #8
Source File: LatestEventsQuery.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public Optional<List<Long>> evaluate(final Query query) {
    if (latestRecords.getSize() < query.getMaxResults()) {
        return Optional.empty();
    }

    if (query.getSearchTerms().isEmpty() && query.getStartDate() == null && query.getEndDate() == null) {
        final List<Long> eventList = latestRecords.asList();
        if (eventList.size() > query.getMaxResults()) {
            return Optional.of(eventList.subList(0, query.getMaxResults()));
        } else {
            return Optional.of(eventList);
        }
    } else {
        return Optional.empty();
    }
}
 
Example #9
Source File: PersistentProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    logger.info("{} got {} hits", query, result.getTotalHitCount());
    return result;
}
 
Example #10
Source File: PersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    logger.info("{} got {} hits", query, result.getTotalHitCount());
    return result;
}
 
Example #11
Source File: VolatileProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
        }
    }

    if (result.getError() != null) {
        throw new IOException(result.getError());
    }

    return result;
}
 
Example #12
Source File: TestLuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new StandardIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final ProvenanceEventRecord event = createEvent();

    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));

    final Query query = new Query(UUID.randomUUID().toString());

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(4000, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example #13
Source File: StandardQueryResult.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public StandardQueryResult(final Query query, final int numSteps) {
    this.query = query;
    this.numSteps = numSteps;
    this.creationNanos = System.nanoTime();

    updateExpiration();
}
 
Example #14
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #15
Source File: StandardQueryResult.java    From nifi with Apache License 2.0 5 votes vote down vote up
public StandardQueryResult(final Query query, final int numSteps) {
    this.query = query;
    this.numSteps = numSteps;
    this.creationNanos = System.nanoTime();

    updateExpiration();
}
 
Example #16
Source File: LuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
private void validate(final Query query) {
    final int numQueries = querySubmissionMap.size();
    if (numQueries > MAX_UNDELETED_QUERY_RESULTS) {
        purgeObsoleteQueries();
        if (querySubmissionMap.size() > MAX_UNDELETED_QUERY_RESULTS) {
            throw new IllegalStateException("Cannot process query because there are currently " + numQueries + " queries whose results have not "
                + "been deleted due to poorly behaving clients not issuing DELETE requests. Please try again later.");
        }
    }

    if (query.getEndDate() != null && query.getStartDate() != null && query.getStartDate().getTime() > query.getEndDate().getTime()) {
        throw new IllegalArgumentException("Query End Time cannot be before Query Start Time");
    }
}
 
Example #17
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example #18
Source File: LatestEventsPerProcessorQuery.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<List<Long>> evaluate(final Query query) {
    if (query.getMaxResults() > 1000) {
        // If query max results > 1000 then we know we don't have enough results. So just return empty.
        return Optional.empty();
    }

    final List<SearchTerm> terms = query.getSearchTerms();
    if (terms.size() != 1) {
        return Optional.empty();
    }

    final SearchTerm term = terms.get(0);
    if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) {
        return Optional.empty();
    }

    if (query.getEndDate() != null || query.getStartDate() != null) {
        return Optional.empty();
    }

    final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue());
    if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) {
        return Optional.empty();
    }

    List<Long> eventIds = ringBuffer.asList();
    if (eventIds.size() > query.getMaxResults()) {
        eventIds = eventIds.subList(0, query.getMaxResults());
    }

    return Optional.of(eventIds);
}
 
Example #19
Source File: PersistentProvenanceRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
public QueryRunnable(final Query query, final AsyncQuerySubmission submission, final NiFiUser user, final File indexDir, final AtomicInteger retrievalCount) {
    this.query = query;
    this.submission = submission;
    this.user = user;
    this.indexDir = indexDir;
    this.retrievalCount = retrievalCount;
}
 
Example #20
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);

    final ProvenanceEventRecord event = createEvent();

    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));

    final Query query = new Query(UUID.randomUUID().toString());

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example #21
Source File: PersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public QueryRunnable(final Query query, final AsyncQuerySubmission submission, final NiFiUser user, final File indexDir, final AtomicInteger retrievalCount) {
    this.query = query;
    this.submission = submission;
    this.user = user;
    this.indexDir = indexDir;
    this.retrievalCount = retrievalCount;
}
 
Example #22
Source File: LuceneEventIndex.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
private void validate(final Query query) {
    final int numQueries = querySubmissionMap.size();
    if (numQueries > MAX_UNDELETED_QUERY_RESULTS) {
        throw new IllegalStateException("Cannot process query because there are currently " + numQueries + " queries whose results have not "
            + "been deleted due to poorly behaving clients not issuing DELETE requests. Please try again later.");
    }

    if (query.getEndDate() != null && query.getStartDate() != null && query.getStartDate().getTime() > query.getEndDate().getTime()) {
        throw new IllegalArgumentException("Query End Time cannot be before Query Start Time");
    }
}
 
Example #23
Source File: LatestEventsPerProcessorQuery.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<List<Long>> evaluate(final Query query) {
    if (query.getMaxResults() > 1000) {
        // If query max results > 1000 then we know we don't have enough results. So just return empty.
        return Optional.empty();
    }

    final List<SearchTerm> terms = query.getSearchTerms();
    if (terms.size() != 1) {
        return Optional.empty();
    }

    final SearchTerm term = terms.get(0);
    if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) {
        return Optional.empty();
    }

    if (query.getEndDate() != null || query.getStartDate() != null) {
        return Optional.empty();
    }

    final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue());
    if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) {
        return Optional.empty();
    }

    List<Long> eventIds = ringBuffer.asList();
    if (eventIds.size() > query.getMaxResults()) {
        eventIds = eventIds.subList(0, query.getMaxResults());
    }

    return Optional.of(eventIds);
}
 
Example #24
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 10000)
public void testNotAuthorizedQuery() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
        @Override
        public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) {
            return event.getEventId() > 2;
        }
    };

    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query("1234");
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234"));
    final QuerySubmission submission = repo.submitQuery(query, createUser());

    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        Thread.sleep(100L);
    }

    // Ensure that we gets events with ID's 3 through 10.
    final List<ProvenanceEventRecord> events = result.getMatchingEvents();
    assertEquals(7, events.size());
    final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList());
    for (int i = 0; i < 7; i++) {
        Assert.assertEquals(i + 3, eventIds.get(i).intValue());
    }
}
 
Example #25
Source File: TestVolatileProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
    repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(10, submission.getResult().getMatchingEvents().size());
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example #26
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    // Give time for rollover to happen
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();

    Thread.sleep(1000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
 
Example #27
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example #28
Source File: WriteAheadProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public QuerySubmission submitQuery(final Query query, final NiFiUser user) {
    return eventIndex.submitQuery(query, createEventAuthorizer(user), user == null ? null : user.getIdentity());
}
 
Example #29
Source File: PersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
public GetMostRecentRunnable(final Query query, final AsyncQuerySubmission submission, final NiFiUser user) {
    this.query = query;
    this.submission = submission;
    this.user = user;
}
 
Example #30
Source File: AsyncQuerySubmission.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public Query getQuery() {
    return query;
}