org.apache.nifi.provenance.search.QueryResult Java Examples
The following examples show how to use
org.apache.nifi.provenance.search.QueryResult.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 6 votes |
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException { final QuerySubmission submission = submitQuery(query, user); final QueryResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { Thread.currentThread().interrupt(); } } if (result.getError() != null) { throw new IOException(result.getError()); } logger.info("{} got {} hits", query, result.getTotalHitCount()); return result; }
Example #2
Source File: VolatileProvenanceRepository.java From nifi with Apache License 2.0 | 6 votes |
public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException { final QuerySubmission submission = submitQuery(query, user); final QueryResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { } } if (result.getError() != null) { throw new IOException(result.getError()); } return result; }
Example #3
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 6 votes |
/** * Here the event file is simply corrupted by virtue of not having any event * records while having correct headers */ @Test public void testWithWithEventFileMissingRecord() throws Exception { assumeFalse(isWindowsEnvironment()); File eventFile = this.prepCorruptedEventFileTests(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*")); query.setMaxResults(100); DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile))); in.writeUTF("BlahBlah"); in.writeInt(4); in.close(); assertTrue(eventFile.exists()); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); }
Example #4
Source File: PersistentProvenanceRepository.java From nifi with Apache License 2.0 | 6 votes |
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException { final QuerySubmission submission = submitQuery(query, user); final QueryResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { Thread.currentThread().interrupt(); } } if (result.getError() != null) { throw new IOException(result.getError()); } logger.info("{} got {} hits", query, result.getTotalHitCount()); return result; }
Example #5
Source File: MiNiFiPersistentProvenanceRepository.java From nifi-minifi with Apache License 2.0 | 6 votes |
QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException { final QuerySubmission submission = submitQuery(query, user); final QueryResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { Thread.currentThread().interrupt(); } } if (result.getError() != null) { throw new IOException(result.getError()); } logger.info("{} got {} hits", query, result.getTotalHitCount()); return result; }
Example #6
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 6 votes |
/** * Here the event file is simply corrupted by virtue of not having any event * records while having correct headers */ @Test public void testWithWithEventFileMissingRecord() throws Exception { File eventFile = this.prepCorruptedEventFileTests(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*")); query.setMaxResults(100); DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile))); in.writeUTF("BlahBlah"); in.writeInt(4); in.close(); assertTrue(eventFile.exists()); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); }
Example #7
Source File: VolatileProvenanceRepository.java From localization_nifi with Apache License 2.0 | 6 votes |
public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException { final QuerySubmission submission = submitQuery(query, user); final QueryResult result = submission.getResult(); while (!result.isFinished()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { } } if (result.getError() != null) { throw new IOException(result.getError()); } return result; }
Example #8
Source File: TestLuceneEventIndex.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void addThenQueryWithEmptyQuery() throws InterruptedException { final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new SimpleIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP); final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); final Query query = new Query(UUID.randomUUID().toString()); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(100, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example #9
Source File: TestLuceneEventIndex.java From nifi with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void addThenQueryWithEmptyQuery() throws InterruptedException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new StandardIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP); final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); final Query query = new Query(UUID.randomUUID().toString()); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(4000, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example #10
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 5 votes |
/** * Here the event file is simply corrupted by virtue of being empty (0 * bytes) */ @Test public void testWithWithEventFileCorrupted() throws Exception { assumeFalse(isWindowsEnvironment()); File eventFile = this.prepCorruptedEventFileTests(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*")); query.setMaxResults(100); DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile))); in.close(); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); }
Example #11
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 5 votes |
/** * Here the event file is simply corrupted by virtue of being empty (0 * bytes) */ @Test public void testWithWithEventFileCorrupted() throws Exception { File eventFile = this.prepCorruptedEventFileTests(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*")); query.setMaxResults(100); DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile))); in.close(); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); }
Example #12
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L * 10); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L * 10); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "10000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(2000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example #13
Source File: ControllerFacade.java From nifi with Apache License 2.0 | 4 votes |
/** * Retrieves the results of a provenance query. * * @param provenanceId id * @return the results of a provenance query */ public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) { try { // get the query to the provenance repository final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository(); final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser()); // ensure the query results could be found if (querySubmission == null) { throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged."); } // get the original query and the results final Query query = querySubmission.getQuery(); final QueryResult queryResult = querySubmission.getResult(); // build the response final ProvenanceDTO provenanceDto = new ProvenanceDTO(); final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO(); final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO(); // include the original request and results provenanceDto.setRequest(requestDto); provenanceDto.setResults(resultsDto); // convert the original request requestDto.setStartDate(query.getStartDate()); requestDto.setEndDate(query.getEndDate()); requestDto.setMinimumFileSize(query.getMinFileSize()); requestDto.setMaximumFileSize(query.getMaxFileSize()); requestDto.setMaxResults(query.getMaxResults()); if (query.getSearchTerms() != null) { final Map<String, String> searchTerms = new HashMap<>(); for (final SearchTerm searchTerm : query.getSearchTerms()) { searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue()); } requestDto.setSearchTerms(searchTerms); } // convert the provenance provenanceDto.setId(query.getIdentifier()); provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime()); provenanceDto.setExpiration(queryResult.getExpiration()); provenanceDto.setFinished(queryResult.isFinished()); provenanceDto.setPercentCompleted(queryResult.getPercentComplete()); // convert each event final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults); if (includeResults || queryResult.isFinished()) { final List<ProvenanceEventDTO> events = new ArrayList<>(); for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) { events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize))); } resultsDto.setProvenanceEvents(events); } if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) { resultsDto.setTotalCount(requestDto.getMaxResults().longValue()); resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+"); } else { resultsDto.setTotalCount(queryResult.getTotalHitCount()); resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount())); } // include any errors if (queryResult.getError() != null) { final Set<String> errors = new HashSet<>(); errors.add(queryResult.getError()); resultsDto.setErrors(errors); } // set the generated timestamp final Date now = new Date(); resultsDto.setGenerated(now); resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime())); // get the oldest available event time final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1); if (!firstEvent.isEmpty()) { resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime())); } provenanceDto.setResults(resultsDto); return provenanceDto; } catch (final IOException ioe) { throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe); } }
Example #14
Source File: TestLuceneEventIndex.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 50000) public void testQuerySpecificField() throws InterruptedException { final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new SimpleIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP); // add 2 events, one of which we will query for. final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L)); // Create a query that searches for the event with the FlowFile UUID equal to the first event's. final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid())); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(100, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example #15
Source File: TestLuceneEventIndex.java From nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 50000) public void testQuerySpecificField() throws InterruptedException { final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new StandardIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP); // add 2 events, one of which we will query for. final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L)); // Create a query that searches for the event with the FlowFile UUID equal to the first event's. final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid())); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(4000, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example #16
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException { final List<SearchableField> searchableFields = new ArrayList<>(); searchableFields.add(SearchableFields.ComponentID); final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileLife(2, TimeUnit.SECONDS); config.setSearchableFields(searchableFields); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("uuid", UUID.randomUUID().toString()); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); final ProvenanceEventRecord record = builder.build(); for (int i = 0; i < 10; i++) { repo.registerEvent(record); } builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record. attributes.put("uuid", "11111111-1111-1111-1111-111111111111"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.) // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different // log file than the previous one. attributes.put("uuid", "22222222-2222-2222-2222-222222222222"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(2, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example #17
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 10000) public void testNotAuthorizedQuery() throws IOException, InterruptedException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(5, TimeUnit.MINUTES); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setDesiredIndexSize(10); // force new index to be created for each rollover repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) { @Override public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) { return event.getEventId() > 2; } }; repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(10L); // make sure the events are destroyed when we call purge repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query("1234"); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234")); final QuerySubmission submission = repo.submitQuery(query, createUser()); final QueryResult result = submission.getResult(); while (!result.isFinished()) { Thread.sleep(100L); } // Ensure that we gets events with ID's 3 through 10. final List<ProvenanceEventRecord> events = result.getMatchingEvents(); assertEquals(7, events.size()); final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList()); for (int i = 0; i < 7; i++) { Assert.assertEquals(i + 3, eventIds.get(i).intValue()); } }
Example #18
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test @Ignore("This test relies too much on timing of background events by using Thread.sleep().") public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(5, TimeUnit.MINUTES); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setDesiredIndexSize(10); // force new index to be created for each rollover repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(10L); // make sure the events are destroyed when we call purge repo.registerEvent(builder.build()); } repo.waitForRollover(); Thread.sleep(2000L); final FileFilter indexFileFilter = file -> file.getName().startsWith("index"); final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length; assertEquals(1, numIndexDirs); // add more records so that we will create a new index final long secondBatchStartTime = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(System.currentTimeMillis()); repo.registerEvent(builder.build()); } // wait for indexing to happen repo.waitForRollover(); // verify we get the results expected final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(20, result.getMatchingEvents().size()); // Ensure index directories exists File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter); assertEquals(2, indexDirs.length); // expire old events and indexes final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime; config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(2000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertEquals(10, newRecordSet.getMatchingEvents().size()); // Ensure that one index directory is gone indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter); assertEquals(1, indexDirs.length); }
Example #19
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexOnRolloverWithImmenseAttribute() throws IOException { final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false)); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test StringBuilder immenseBldr = new StringBuilder(immenseAttrSize); for (int i=0; i < immenseAttrSize; i++) { immenseBldr.append('0'); } final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); attributes.put("immense", immenseBldr.toString()); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); }
Example #20
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example #21
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } // Give time for rollover to happen repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(1000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example #22
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test @Ignore("This test relies too much on timing of background events by using Thread.sleep().") public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(5, TimeUnit.MINUTES); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setDesiredIndexSize(10); // force new index to be created for each rollover repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(10L); // make sure the events are destroyed when we call purge repo.registerEvent(builder.build()); } repo.waitForRollover(); Thread.sleep(2000L); final FileFilter indexFileFilter = file -> file.getName().startsWith("index"); final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length; assertEquals(1, numIndexDirs); // add more records so that we will create a new index final long secondBatchStartTime = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(System.currentTimeMillis()); repo.registerEvent(builder.build()); } // wait for indexing to happen repo.waitForRollover(); // verify we get the results expected final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(20, result.getMatchingEvents().size()); // Ensure index directories exists File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter); assertEquals(2, indexDirs.length); // expire old events and indexes final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime; config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(2000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertEquals(10, newRecordSet.getMatchingEvents().size()); // Ensure that one index directory is gone indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter); assertEquals(1, indexDirs.length); }
Example #23
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example #24
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexOnRolloverWithImmenseAttribute() throws IOException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false)); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test StringBuilder immenseBldr = new StringBuilder(immenseAttrSize); for (int i = 0; i < immenseAttrSize; i++) { immenseBldr.append('0'); } final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); attributes.put("immense", immenseBldr.toString()); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); }
Example #25
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException { assumeFalse(isWindowsEnvironment()); final List<SearchableField> searchableFields = new ArrayList<>(); searchableFields.add(SearchableFields.ComponentID); final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileLife(2, TimeUnit.SECONDS); config.setSearchableFields(searchableFields); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("uuid", UUID.randomUUID().toString()); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); final ProvenanceEventRecord record = builder.build(); for (int i = 0; i < 10; i++) { repo.registerEvent(record); } builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record. attributes.put("uuid", "11111111-1111-1111-1111-111111111111"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.) // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different // log file than the previous one. attributes.put("uuid", "22222222-2222-2222-2222-222222222222"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(2, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example #26
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L * 10); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L * 10); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "10000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(2000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example #27
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } // Give time for rollover to happen repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(1000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example #28
Source File: ControllerFacade.java From localization_nifi with Apache License 2.0 | 4 votes |
/** * Retrieves the results of a provenance query. * * @param provenanceId id * @return the results of a provenance query */ public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) { try { // get the query to the provenance repository final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository(); final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser()); // ensure the query results could be found if (querySubmission == null) { throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged."); } // get the original query and the results final Query query = querySubmission.getQuery(); final QueryResult queryResult = querySubmission.getResult(); // build the response final ProvenanceDTO provenanceDto = new ProvenanceDTO(); final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO(); final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO(); // include the original request and results provenanceDto.setRequest(requestDto); provenanceDto.setResults(resultsDto); // convert the original request requestDto.setStartDate(query.getStartDate()); requestDto.setEndDate(query.getEndDate()); requestDto.setMinimumFileSize(query.getMinFileSize()); requestDto.setMaximumFileSize(query.getMaxFileSize()); requestDto.setMaxResults(query.getMaxResults()); if (query.getSearchTerms() != null) { final Map<String, String> searchTerms = new HashMap<>(); for (final SearchTerm searchTerm : query.getSearchTerms()) { searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue()); } requestDto.setSearchTerms(searchTerms); } // convert the provenance provenanceDto.setId(query.getIdentifier()); provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime()); provenanceDto.setExpiration(queryResult.getExpiration()); provenanceDto.setFinished(queryResult.isFinished()); provenanceDto.setPercentCompleted(queryResult.getPercentComplete()); // convert each event final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults); if (includeResults || queryResult.isFinished()) { final List<ProvenanceEventDTO> events = new ArrayList<>(); for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) { events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize))); } resultsDto.setProvenanceEvents(events); } if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) { resultsDto.setTotalCount(requestDto.getMaxResults().longValue()); resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+"); } else { resultsDto.setTotalCount(queryResult.getTotalHitCount()); resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount())); } // include any errors if (queryResult.getError() != null) { final Set<String> errors = new HashSet<>(); errors.add(queryResult.getError()); resultsDto.setErrors(errors); } // set the generated timestamp final Date now = new Date(); resultsDto.setGenerated(now); resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime())); // get the oldest available event time final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1); if (!firstEvent.isEmpty()) { resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime())); } provenanceDto.setResults(resultsDto); return provenanceDto; } catch (final IOException ioe) { throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe); } }
Example #29
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 10000) public void testNotAuthorizedQuery() throws IOException, InterruptedException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(5, TimeUnit.MINUTES); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setDesiredIndexSize(10); // force new index to be created for each rollover repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) { @Override public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) { return event.getEventId() > 2; } }; repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(10L); // make sure the events are destroyed when we call purge repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query("1234"); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234")); final QuerySubmission submission = repo.submitQuery(query, createUser()); final QueryResult result = submission.getResult(); while (!result.isFinished()) { Thread.sleep(100L); } // Ensure that we gets events with ID's 3 through 10. final List<ProvenanceEventRecord> events = result.getMatchingEvents(); assertEquals(7, events.size()); final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList()); for (int i = 0; i < 7; i++) { Assert.assertEquals(i + 3, eventIds.get(i).intValue()); } }