Java Code Examples for org.apache.nifi.provenance.search.Query#setMaxResults()

The following examples show how to use org.apache.nifi.provenance.search.Query#setMaxResults() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

6 votes

/**
 * Here the event file is simply corrupted by virtue of not having any event
 * records while having correct headers
 */
@Test
public void testWithWithEventFileMissingRecord() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);

    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.writeUTF("BlahBlah");
    in.writeInt(4);
    in.close();
    assertTrue(eventFile.exists());
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}

Example 2

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

6 votes

/**
 * Here the event file is simply corrupted by virtue of not having any event
 * records while having correct headers
 */
@Test
public void testWithWithEventFileMissingRecord() throws Exception {
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);

    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.writeUTF("BlahBlah");
    in.writeInt(4);
    in.close();
    assertTrue(eventFile.exists());
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}

Example 3

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

6 votes

private List<Document> runQuery(final File indexDirectory, final List<File> storageDirs, final String query) throws IOException, ParseException {
    assumeFalse(isWindowsEnvironment());
    try (final DirectoryReader directoryReader = DirectoryReader.open(FSDirectory.open(indexDirectory.toPath()))) {
        final IndexSearcher searcher = new IndexSearcher(directoryReader);

        final Analyzer analyzer = new SimpleAnalyzer();
        final org.apache.lucene.search.Query luceneQuery = new QueryParser("uuid", analyzer).parse(query);

        final Query q = new Query("");
        q.setMaxResults(1000);
        final TopDocs topDocs = searcher.search(luceneQuery, 1000);

        final List<Document> docs = new ArrayList<>();
        for (final ScoreDoc scoreDoc : topDocs.scoreDocs) {
            final int docId = scoreDoc.doc;
            final Document d = directoryReader.document(docId);
            docs.add(d);
        }

        return docs;
    }
}

Example 4

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

5 votes

/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    assumeFalse(isWindowsEnvironment());
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}

Example 5

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

5 votes

/**
 * Here the event file is simply corrupted by virtue of being empty (0
 * bytes)
 */
@Test
public void testWithWithEventFileCorrupted() throws Exception {
    File eventFile = this.prepCorruptedEventFileTests();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "foo-*"));
    query.setMaxResults(100);
    DataOutputStream in = new DataOutputStream(new GZIPOutputStream(new FileOutputStream(eventFile)));
    in.close();
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}

Example 6

Source File: TestVolatileProvenanceRepository.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
    repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null, null));

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(10, submission.getResult().getMatchingEvents().size());
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
    }
}

Example 7

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

4 votes

@Test
public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    // Give time for rollover to happen
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();

    Thread.sleep(1000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}

Example 8

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

4 votes

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.addStorageDirectory("2", new File("target/storage/" + UUID.randomUUID().toString()));
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(1, TimeUnit.SECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    for (int j = 0; j < 3; j++) {
        attributes.put("iteration", String.valueOf(j));

        builder.setEventTime(System.currentTimeMillis());
        builder.setEventType(ProvenanceEventType.RECEIVE);
        builder.setTransitUri("nifi://unit-test");
        builder.setComponentId("1234");
        builder.setComponentType("dummy processor");
        builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));

        for (int i = 0; i < 10; i++) {
            String uuidSuffix = String.valueOf(i + j * 10);
            if (uuidSuffix.length() < 2) {
                uuidSuffix = "0" + uuidSuffix;
            }

            attributes.put("uuid", "00000000-0000-0000-0000-0000000000" + uuidSuffix);
            builder.fromFlowFile(createFlowFile(i + j * 10, 3000L, attributes));
            repo.registerEvent(builder.build());
        }

        repo.waitForRollover();
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(30, submission.getResult().getMatchingEvents().size());
    final Map<String, Integer> counts = new HashMap<>();
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);

        final String index = match.getAttributes().get("iteration");
        Integer count = counts.get(index);
        if (count == null) {
            count = 0;
        }
        counts.put(index, count + 1);
    }

    assertEquals(3, counts.size());
    assertEquals(10, counts.get("0").intValue());
    assertEquals(10, counts.get("1").intValue());
    assertEquals(10, counts.get("2").intValue());

    config.setMaxRecordLife(1, TimeUnit.MILLISECONDS);

    repo.purgeOldEvents();

    Thread.sleep(2000L); // purge is async. Give it time to do its job.

    query.setMaxResults(100);
    final QuerySubmission noResultSubmission = repo.submitQuery(query, createUser());
    while (!noResultSubmission.getResult().isFinished()) {
        Thread.sleep(10L);
    }

    assertEquals(0, noResultSubmission.getResult().getTotalHitCount());
}

Example 9

Source File: TestVolatileProvenanceRepository.java From nifi with Apache License 2.0

4 votes

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
    repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null));

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(10, submission.getResult().getMatchingEvents().size());
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
    }
}

Example 10

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

4 votes

@Test
public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}

Example 11

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

4 votes

@Test
public void testIndexOnRolloverWithImmenseAttribute() throws IOException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test
    StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
    for (int i = 0; i < immenseAttrSize; i++) {
        immenseBldr.append('0');
    }
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    attributes.put("immense", immenseBldr.toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}

Example 12

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

4 votes

@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}

Example 13

Source File: ControllerFacade.java From localization_nifi with Apache License 2.0

4 votes

/**
 * Submits a provenance query.
 *
 * @param provenanceDto dto
 * @return provenance info
 */
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
    final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();

    // create the query
    final Query query = new Query(provenanceDto.getId());

    // if the request was specified
    if (requestDto != null) {
        // add each search term specified
        final Map<String, String> searchTerms = requestDto.getSearchTerms();
        if (searchTerms != null) {
            for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
                SearchableField field;

                field = SearchableFields.getSearchableField(searchTerm.getKey());
                if (field == null) {
                    field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
                }
                query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
            }
        }

        // specify the start date if specified
        if (requestDto.getStartDate() != null) {
            query.setStartDate(requestDto.getStartDate());
        }

        // ensure an end date is populated
        if (requestDto.getEndDate() != null) {
            query.setEndDate(requestDto.getEndDate());
        }

        // set the min/max file size
        query.setMinFileSize(requestDto.getMinimumFileSize());
        query.setMaxFileSize(requestDto.getMaximumFileSize());

        // set the max results desired
        query.setMaxResults(requestDto.getMaxResults());
    }

    // submit the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());

    // return the query with the results populated at this point
    return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}

Example 14

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}

Example 15

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

4 votes

@Test
@Ignore("This test relies too much on timing of background events by using Thread.sleep().")
public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    Thread.sleep(2000L);

    final FileFilter indexFileFilter = file -> file.getName().startsWith("index");
    final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length;
    assertEquals(1, numIndexDirs);

    // add more records so that we will create a new index
    final long secondBatchStartTime = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(System.currentTimeMillis());
        repo.registerEvent(builder.build());
    }

    // wait for indexing to happen
    repo.waitForRollover();

    // verify we get the results expected
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(20, result.getMatchingEvents().size());

    // Ensure index directories exists
    File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(2, indexDirs.length);

    // expire old events and indexes
    final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime;
    config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertEquals(10, newRecordSet.getMatchingEvents().size());

    // Ensure that one index directory is gone
    indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(1, indexDirs.length);
}

Example 16

Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0

4 votes

@Test
@Ignore("This test relies too much on timing of background events by using Thread.sleep().")
public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setDesiredIndexSize(10); // force new index to be created for each rollover

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(10L); // make sure the events are destroyed when we call purge
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    Thread.sleep(2000L);

    final FileFilter indexFileFilter = file -> file.getName().startsWith("index");
    final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length;
    assertEquals(1, numIndexDirs);

    // add more records so that we will create a new index
    final long secondBatchStartTime = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(System.currentTimeMillis());
        repo.registerEvent(builder.build());
    }

    // wait for indexing to happen
    repo.waitForRollover();

    // verify we get the results expected
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(20, result.getMatchingEvents().size());

    // Ensure index directories exists
    File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(2, indexDirs.length);

    // expire old events and indexes
    final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime;
    config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertEquals(10, newRecordSet.getMatchingEvents().size());

    // Ensure that one index directory is gone
    indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(1, indexDirs.length);
}

Example 17

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.addStorageDirectory("2", new File("target/storage/" + UUID.randomUUID().toString()));
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(1, TimeUnit.SECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    for (int j = 0; j < 3; j++) {
        attributes.put("iteration", String.valueOf(j));

        builder.setEventTime(System.currentTimeMillis());
        builder.setEventType(ProvenanceEventType.RECEIVE);
        builder.setTransitUri("nifi://unit-test");
        builder.setComponentId("1234");
        builder.setComponentType("dummy processor");
        builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));

        for (int i = 0; i < 10; i++) {
            String uuidSuffix = String.valueOf(i + j * 10);
            if (uuidSuffix.length() < 2) {
                uuidSuffix = "0" + uuidSuffix;
            }

            attributes.put("uuid", "00000000-0000-0000-0000-0000000000" + uuidSuffix);
            builder.fromFlowFile(createFlowFile(i + j * 10, 3000L, attributes));
            repo.registerEvent(builder.build());
        }

        repo.waitForRollover();
    }

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }

    assertEquals(30, submission.getResult().getMatchingEvents().size());
    final Map<String, Integer> counts = new HashMap<>();
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);

        final String index = match.getAttributes().get("iteration");
        Integer count = counts.get(index);
        if (count == null) {
            count = 0;
        }
        counts.put(index, count + 1);
    }

    assertEquals(3, counts.size());
    assertEquals(10, counts.get("0").intValue());
    assertEquals(10, counts.get("1").intValue());
    assertEquals(10, counts.get("2").intValue());

    config.setMaxRecordLife(1, TimeUnit.MILLISECONDS);

    repo.purgeOldEvents();

    Thread.sleep(2000L); // purge is async. Give it time to do its job.

    query.setMaxResults(100);
    final QuerySubmission noResultSubmission = repo.submitQuery(query, createUser());
    while (!noResultSubmission.getResult().isFinished()) {
        Thread.sleep(10L);
    }

    assertEquals(0, noResultSubmission.getResult().getTotalHitCount());
}

Example 18

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L * 10);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L * 10);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));

    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "10000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }

    Thread.sleep(2000L);

    config.setMaxStorageCapacity(100L);
    config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);

    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}

Example 19

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}

Example 20

Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testIndexOnRolloverWithImmenseAttribute() throws IOException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    int immenseAttrSize = 33000; // must be greater than 32766 for a meaningful test
    StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
    for (int i=0; i < immenseAttrSize; i++) {
        immenseBldr.append('0');
    }
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    attributes.put("immense", immenseBldr.toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }

    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}