Java Code Examples for org.apache.nifi.provenance.search.Query#addSearchTerm()

The following examples show how to use org.apache.nifi.provenance.search.Query#addSearchTerm() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of not having any event
 * records while having correct headers
 */
@Test
public void testWithWithEventFileMissingRecord() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);

    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.writeUTF("BlahBlah");
    in.writeInt(4);
    in.close();
    assertTrue(eventFile.exists());
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example 2
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example 3
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example 4
Source File: TestVolatileProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
    repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null, null));

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(10, submission.getResult().getMatchingEvents().size());
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example 5
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
@Ignore("This test relies too much on timing of background events by using Thread.sleep().")
public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    Thread.sleep(2000L);

    final FileFilter indexFileFilter = file -> file.getName().startsWith("index");
    final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length;
    assertEquals(1, numIndexDirs);

    // add more records so that we will create a new index
    final long secondBatchStartTime = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(System.currentTimeMillis());
        repo.registerEvent(builder.build());
    }

    // wait for indexing to happen
    repo.waitForRollover();

    // verify we get the results expected
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(20, result.getMatchingEvents().size());

    // Ensure index directories exists
    File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(2, indexDirs.length);

    // expire old events and indexes
    final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime;
    config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertEquals(10, newRecordSet.getMatchingEvents().size());

    // Ensure that one index directory is gone
    indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(1, indexDirs.length);
}
 
Example 6
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.addStorageDirectory("2", new File("target/storage/" + UUID.randomUUID().toString()));
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(1, TimeUnit.SECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    for (int j = 0; j < 3; j++) {
        attributes.put("iteration", String.valueOf(j));

        builder.setEventTime(System.currentTimeMillis());
        builder.setEventType(ProvenanceEventType.RECEIVE);
        builder.setTransitUri("nifi://unit-test");
        builder.setComponentId("1234");
        builder.setComponentType("dummy processor");
        builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));

        for (int i = 0; i < 10; i++) {
            String uuidSuffix = String.valueOf(i + j * 10);
            if (uuidSuffix.length() < 2) {
                uuidSuffix = "0" + uuidSuffix;
            }

            attributes.put("uuid", "00000000-0000-0000-0000-0000000000" + uuidSuffix);
            builder.fromFlowFile(createFlowFile(i + j * 10, 3000L, attributes));
            repo.registerEvent(builder.build());
        }

        repo.waitForRollover();
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(30, submission.getResult().getMatchingEvents().size());
    final Map<String, Integer> counts = new HashMap<>();
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);

        final String index = match.getAttributes().get("iteration");
        Integer count = counts.get(index);
        if (count == null) {
            count = 0;
        }
        counts.put(index, count + 1);
    }

    assertEquals(3, counts.size());
    assertEquals(10, counts.get("0").intValue());
    assertEquals(10, counts.get("1").intValue());
    assertEquals(10, counts.get("2").intValue());

    config.setMaxRecordLife(1, TimeUnit.MILLISECONDS);

    repo.purgeOldEvents();

    Thread.sleep(2000L); // purge is async. Give it time to do its job.

    query.setMaxResults(100);
    final QuerySubmission noResultSubmission = repo.submitQuery(query, createUser());
    while (!noResultSubmission.getResult().isFinished()) {
        Thread.sleep(10L);
    }

    assertEquals(0, noResultSubmission.getResult().getTotalHitCount());
}
 
Example 7
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L * 10);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L * 10);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "10000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
 
Example 8
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 10000)
public void testNotAuthorizedQuery() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
        @Override
        public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) {
            return event.getEventId() > 2;
        }
    };

    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query("1234");
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234"));
    final QuerySubmission submission = repo.submitQuery(query, createUser());

    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        Thread.sleep(100L);
    }

    // Ensure that we gets events with ID's 3 through 10.
    final List<ProvenanceEventRecord> events = result.getMatchingEvents();
    assertEquals(7, events.size());
    final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList());
    for (int i = 0; i < 7; i++) {
        Assert.assertEquals(i + 3, eventIds.get(i).intValue());
    }
}
 
Example 9
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexOnRolloverWithImmenseAttribute() throws IOException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test
    StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
    for (int i = 0; i < immenseAttrSize; i++) {
        immenseBldr.append('0');
    }
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    attributes.put("immense", immenseBldr.toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example 10
Source File: ITestPersistentProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example 11
Source File: ControllerFacade.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Submits a provenance query.
 *
 * @param provenanceDto dto
 * @return provenance info
 */
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
    final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();

    // create the query
    final Query query = new Query(provenanceDto.getId());

    // if the request was specified
    if (requestDto != null) {
        // add each search term specified
        final Map<String, String> searchTerms = requestDto.getSearchTerms();
        if (searchTerms != null) {
            for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
                SearchableField field;

                field = SearchableFields.getSearchableField(searchTerm.getKey());
                if (field == null) {
                    field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
                }
                query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
            }
        }

        // specify the start date if specified
        if (requestDto.getStartDate() != null) {
            query.setStartDate(requestDto.getStartDate());
        }

        // ensure an end date is populated
        if (requestDto.getEndDate() != null) {
            query.setEndDate(requestDto.getEndDate());
        }

        // set the min/max file size
        query.setMinFileSize(requestDto.getMinimumFileSize());
        query.setMaxFileSize(requestDto.getMaximumFileSize());

        // set the max results desired
        query.setMaxResults(requestDto.getMaxResults());
    }

    // submit the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());

    // return the query with the results populated at this point
    return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}
 
Example 12
Source File: TestLuceneEventIndex.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 50000)
public void testQuerySpecificField() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);

    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP);

    // add 2 events, one of which we will query for.
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L));

    // Create a query that searches for the event with the FlowFile UUID equal to the first event's.
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid()));

    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);

    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);

        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);

        assertTrue(result.isFinished());
        assertNull(result.getError());

        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        Thread.sleep(100L); // avoid crushing the CPU
    }

    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}
 
Example 13
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 10000)
public void testNotAuthorizedQuery() throws IOException, InterruptedException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
        @Override
        public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) {
            return event.getEventId() > 2;
        }
    };

    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query("1234");
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234"));
    final QuerySubmission submission = repo.submitQuery(query, createUser());

    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        Thread.sleep(100L);
    }

    // Ensure that we gets events with ID's 3 through 10.
    final List<ProvenanceEventRecord> events = result.getMatchingEvents();
    assertEquals(7, events.size());
    final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList());
    for (int i = 0; i < 7; i++) {
        Assert.assertEquals(i + 3, eventIds.get(i).intValue());
    }
}
 
Example 14
Source File: ControllerFacade.java    From nifi with Apache License 2.0 4 votes vote down vote up
/**
 * Submits a provenance query.
 *
 * @param provenanceDto dto
 * @return provenance info
 */
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
    final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();

    // create the query
    final Query query = new Query(provenanceDto.getId());

    // if the request was specified
    if (requestDto != null) {
        // add each search term specified
        final Map<String, String> searchTerms = requestDto.getSearchTerms();
        if (searchTerms != null) {
            for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
                SearchableField field;

                field = SearchableFields.getSearchableField(searchTerm.getKey());
                if (field == null) {
                    field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
                }
                query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
            }
        }

        // specify the start date if specified
        if (requestDto.getStartDate() != null) {
            query.setStartDate(requestDto.getStartDate());
        }

        // ensure an end date is populated
        if (requestDto.getEndDate() != null) {
            query.setEndDate(requestDto.getEndDate());
        }

        // set the min/max file size
        query.setMinFileSize(requestDto.getMinimumFileSize());
        query.setMaxFileSize(requestDto.getMaximumFileSize());

        // set the max results desired
        query.setMaxResults(requestDto.getMaxResults());
    }

    // submit the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());

    // return the query with the results populated at this point
    return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}
 
Example 15
Source File: TestVolatileProvenanceRepository.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
    repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(10, submission.getResult().getMatchingEvents().size());
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example 16
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.addStorageDirectory("2", new File("target/storage/" + UUID.randomUUID().toString()));
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(1, TimeUnit.SECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    for (int j = 0; j < 3; j++) {
        attributes.put("iteration", String.valueOf(j));

        builder.setEventTime(System.currentTimeMillis());
        builder.setEventType(ProvenanceEventType.RECEIVE);
        builder.setTransitUri("nifi://unit-test");
        builder.setComponentId("1234");
        builder.setComponentType("dummy processor");
        builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));

        for (int i = 0; i < 10; i++) {
            String uuidSuffix = String.valueOf(i + j * 10);
            if (uuidSuffix.length() < 2) {
                uuidSuffix = "0" + uuidSuffix;
            }

            attributes.put("uuid", "00000000-0000-0000-0000-0000000000" + uuidSuffix);
            builder.fromFlowFile(createFlowFile(i + j * 10, 3000L, attributes));
            repo.registerEvent(builder.build());
        }

        repo.waitForRollover();
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(30, submission.getResult().getMatchingEvents().size());
    final Map<String, Integer> counts = new HashMap<>();
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);

        final String index = match.getAttributes().get("iteration");
        Integer count = counts.get(index);
        if (count == null) {
            count = 0;
        }
        counts.put(index, count + 1);
    }

    assertEquals(3, counts.size());
    assertEquals(10, counts.get("0").intValue());
    assertEquals(10, counts.get("1").intValue());
    assertEquals(10, counts.get("2").intValue());

    config.setMaxRecordLife(1, TimeUnit.MILLISECONDS);

    repo.purgeOldEvents();

    Thread.sleep(2000L); // purge is async. Give it time to do its job.

    query.setMaxResults(100);
    final QuerySubmission noResultSubmission = repo.submitQuery(query, createUser());
    while (!noResultSubmission.getResult().isFinished()) {
        Thread.sleep(10L);
    }

    assertEquals(0, noResultSubmission.getResult().getTotalHitCount());
}
 
Example 17
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L * 10);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L * 10);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "10000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
 
Example 18
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 
Example 19
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testIndexOnRolloverWithImmenseAttribute() throws IOException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test
    StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
    for (int i=0; i < immenseAttrSize; i++) {
        immenseBldr.append('0');
    }
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    attributes.put("immense", immenseBldr.toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
 
Example 20
Source File: TestPersistentProvenanceRepository.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}