org.apache.nifi.provenance.util.NamedThreadFactory Java Examples

@Override
public void initialize(final EventStore eventStore) {
    this.eventStore = eventStore;
    directoryManager.initialize();

    maintenanceExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Provenance Repository Maintenance"));
    maintenanceExecutor.scheduleWithFixedDelay(() -> performMaintenance(), 1, 1, TimeUnit.MINUTES);
    maintenanceExecutor.scheduleWithFixedDelay(new RemoveExpiredQueryResults(), 30, 30, TimeUnit.SECONDS);

    cachedQueries.add(new LatestEventsQuery());
    cachedQueries.add(new LatestEventsPerProcessorQuery());

    final Optional<Integer> warmCacheMinutesOption = config.getWarmCacheFrequencyMinutes();
    if (warmCacheMinutesOption.isPresent() && warmCacheMinutesOption.get() > 0) {
        for (final File storageDir : config.getStorageDirectories().values()) {
            final int minutes = warmCacheMinutesOption.get();
            cacheWarmerExecutor.scheduleWithFixedDelay(new LuceneCacheWarmer(storageDir, indexManager), 1, minutes, TimeUnit.MINUTES);
        }
    }
}
 

public PersistentProvenanceRepository(final RepositoryConfiguration configuration, final int rolloverCheckMillis) throws IOException {
    if (configuration.getStorageDirectories().isEmpty()) {
        throw new IllegalArgumentException("Must specify at least one storage directory");
    }

    this.configuration = configuration;
    this.maxAttributeChars = configuration.getMaxAttributeChars();

    for (final File file : configuration.getStorageDirectories().values()) {
        final Path storageDirectory = file.toPath();
        final Path journalDirectory = storageDirectory.resolve("journals");

        if (!Files.exists(journalDirectory)) {
            Files.createDirectories(journalDirectory);
        } else if (!Files.isDirectory(journalDirectory)) {
            throw new IllegalArgumentException("Storage Location " + journalDirectory + " is not a directory");
        }
    }

    this.maxPartitionMillis = configuration.getMaxEventFileLife(TimeUnit.MILLISECONDS);
    this.maxPartitionBytes = configuration.getMaxEventFileCapacity();
    this.indexConfig = new IndexConfiguration(configuration);
    this.indexManager = new SimpleIndexManager(configuration);
    this.alwaysSync = configuration.isAlwaysSync();
    this.rolloverCheckMillis = rolloverCheckMillis;

    scheduledExecService = Executors.newScheduledThreadPool(3, new NamedThreadFactory("Provenance Maintenance Thread"));
    queryExecService = Executors.newFixedThreadPool(configuration.getQueryThreadPoolSize(), new NamedThreadFactory("Provenance Query Thread"));

    // The number of rollover threads is a little bit arbitrary but comes from the idea that multiple storage directories generally
    // live on separate physical partitions. As a result, we want to use at least one thread per partition in order to utilize the
    // disks efficiently. However, the rollover actions can be somewhat CPU intensive, so we double the number of threads in order
    // to account for that.
    final int numRolloverThreads = configuration.getStorageDirectories().size() * 2;
    rolloverExecutor = Executors.newScheduledThreadPool(numRolloverThreads, new NamedThreadFactory("Provenance Repository Rollover Thread"));
}
 

public LuceneEventIndex(final RepositoryConfiguration config, final IndexManager indexManager, final int maxEventsPerCommit, final EventReporter eventReporter) {
    this.eventReporter = eventReporter;
    queryExecutor = Executors.newFixedThreadPool(config.getQueryThreadPoolSize(), new NamedThreadFactory("Provenance Query"));
    indexExecutor = Executors.newFixedThreadPool(config.getIndexThreadPoolSize(), new NamedThreadFactory("Index Provenance Events"));
    cacheWarmerExecutor = Executors.newScheduledThreadPool(config.getStorageDirectories().size(), new NamedThreadFactory("Warm Lucene Index", true));
    directoryManager = new IndexDirectoryManager(config);

    // Limit number of indexing threads to 100. When we restore the repository on restart,
    // we have to re-index up to MAX_THREADS * MAX_DOCUMENTS_PER_THREADS events prior to
    // the last event that the index holds. This is done because we could have that many
    // events 'in flight', waiting to be indexed when the last index writer was committed,
    // so even though the index says the largest event ID is 1,000,000 for instance, Event
    // with ID 999,999 may still not have been indexed because another thread was in the
    // process of writing the event to the index.
    final int configuredIndexPoolSize = config.getIndexThreadPoolSize();
    final int numIndexThreads;
    if (configuredIndexPoolSize > MAX_INDEX_THREADS) {
        logger.warn("The Provenance Repository is configured to perform indexing of events using {} threads. This number exceeds the maximum allowable number of threads, which is {}. "
            + "Will proceed using {} threads. This value is limited because the performance of indexing will decrease and startup times will increase when setting this value too high.",
            configuredIndexPoolSize, MAX_INDEX_THREADS, MAX_INDEX_THREADS);
        numIndexThreads = MAX_INDEX_THREADS;
    } else {
        numIndexThreads = configuredIndexPoolSize;
    }

    for (int i = 0; i < numIndexThreads; i++) {
        final EventIndexTask task = new EventIndexTask(documentQueue, config, indexManager, directoryManager, maxEventsPerCommit, eventReporter);
        indexTasks.add(task);
        indexExecutor.submit(task);
    }
    this.config = config;
    this.indexManager = indexManager;
    this.eventConverter = new ConvertEventToLuceneDocument(config.getSearchableFields(), config.getSearchableAttributes());
}
 

public MiNiFiPersistentProvenanceRepository(final RepositoryConfiguration configuration, final int rolloverCheckMillis) throws IOException {
    if (configuration.getStorageDirectories().isEmpty()) {
        throw new IllegalArgumentException("Must specify at least one storage directory");
    }

    this.configuration = configuration;
    this.maxAttributeChars = configuration.getMaxAttributeChars();

    for (final File file : configuration.getStorageDirectories().values()) {
        final Path storageDirectory = file.toPath();
        final Path journalDirectory = storageDirectory.resolve("journals");

        if (!Files.exists(journalDirectory)) {
            Files.createDirectories(journalDirectory);
        } else if (!Files.isDirectory(journalDirectory)) {
            throw new IllegalArgumentException("Storage Location " + journalDirectory + " is not a directory");
        }
    }

    this.maxPartitionMillis = configuration.getMaxEventFileLife(TimeUnit.MILLISECONDS);
    this.maxPartitionBytes = configuration.getMaxEventFileCapacity();
    this.indexConfig = new IndexConfiguration(configuration);
    this.indexManager = new SimpleIndexManager(configuration);
    this.alwaysSync = configuration.isAlwaysSync();
    this.rolloverCheckMillis = rolloverCheckMillis;

    scheduledExecService = Executors.newScheduledThreadPool(3, new NamedThreadFactory("Provenance Maintenance Thread"));
    queryExecService = Executors.newFixedThreadPool(configuration.getQueryThreadPoolSize(), new NamedThreadFactory("Provenance Query Thread"));

    // The number of rollover threads is a little bit arbitrary but comes from the idea that multiple storage directories generally
    // live on separate physical partitions. As a result, we want to use at least one thread per partition in order to utilize the
    // disks efficiently. However, the rollover actions can be somewhat CPU intensive, so we double the number of threads in order
    // to account for that.
    final int numRolloverThreads = configuration.getStorageDirectories().size() * 2;
    rolloverExecutor = Executors.newScheduledThreadPool(numRolloverThreads, new NamedThreadFactory("Provenance Repository Rollover Thread"));
}
 

public PersistentProvenanceRepository(final RepositoryConfiguration configuration, final int rolloverCheckMillis) throws IOException {
    if (configuration.getStorageDirectories().isEmpty()) {
        throw new IllegalArgumentException("Must specify at least one storage directory");
    }

    this.configuration = configuration;
    this.maxAttributeChars = configuration.getMaxAttributeChars();

    for (final File file : configuration.getStorageDirectories().values()) {
        final Path storageDirectory = file.toPath();
        final Path journalDirectory = storageDirectory.resolve("journals");

        if (!Files.exists(journalDirectory)) {
            Files.createDirectories(journalDirectory);
        } else if (!Files.isDirectory(journalDirectory)) {
            throw new IllegalArgumentException("Storage Location " + journalDirectory + " is not a directory");
        }
    }

    this.maxPartitionMillis = configuration.getMaxEventFileLife(TimeUnit.MILLISECONDS);
    this.maxPartitionBytes = configuration.getMaxEventFileCapacity();
    this.indexConfig = new IndexConfiguration(configuration);
    this.indexManager = new StandardIndexManager(configuration);
    this.alwaysSync = configuration.isAlwaysSync();
    this.rolloverCheckMillis = rolloverCheckMillis;

    scheduledExecService = Executors.newScheduledThreadPool(3, new NamedThreadFactory("Provenance Maintenance Thread"));
    queryExecService = Executors.newFixedThreadPool(configuration.getQueryThreadPoolSize(), new NamedThreadFactory("Provenance Query Thread"));

    // The number of rollover threads is a little bit arbitrary but comes from the idea that multiple storage directories generally
    // live on separate physical partitions. As a result, we want to use at least one thread per partition in order to utilize the
    // disks efficiently. However, the rollover actions can be somewhat CPU intensive, so we double the number of threads in order
    // to account for that.
    final int numRolloverThreads = configuration.getStorageDirectories().size() * 2;
    rolloverExecutor = Executors.newScheduledThreadPool(numRolloverThreads, new NamedThreadFactory("Provenance Repository Rollover Thread"));
}
 

public LuceneEventIndex(final RepositoryConfiguration config, final IndexManager indexManager, final int maxEventsPerCommit, final EventReporter eventReporter) {
    this.eventReporter = eventReporter;
    queryExecutor = Executors.newFixedThreadPool(config.getQueryThreadPoolSize(), new NamedThreadFactory("Provenance Query"));
    indexExecutor = Executors.newFixedThreadPool(config.getIndexThreadPoolSize(), new NamedThreadFactory("Index Provenance Events"));
    cacheWarmerExecutor = Executors.newScheduledThreadPool(config.getStorageDirectories().size(), new NamedThreadFactory("Warm Lucene Index", true));
    directoryManager = new IndexDirectoryManager(config);

    // Limit number of indexing threads to 100. When we restore the repository on restart,
    // we have to re-index up to MAX_THREADS * MAX_DOCUMENTS_PER_THREADS events prior to
    // the last event that the index holds. This is done because we could have that many
    // events 'in flight', waiting to be indexed when the last index writer was committed,
    // so even though the index says the largest event ID is 1,000,000 for instance, Event
    // with ID 999,999 may still not have been indexed because another thread was in the
    // process of writing the event to the index.
    final int configuredIndexPoolSize = config.getIndexThreadPoolSize();
    final int numIndexThreads;
    if (configuredIndexPoolSize > MAX_INDEX_THREADS) {
        logger.warn("The Provenance Repository is configured to perform indexing of events using {} threads. This number exceeds the maximum allowable number of threads, which is {}. "
            + "Will proceed using {} threads. This value is limited because the performance of indexing will decrease and startup times will increase when setting this value too high.",
            configuredIndexPoolSize, MAX_INDEX_THREADS, MAX_INDEX_THREADS);
        numIndexThreads = MAX_INDEX_THREADS;
    } else {
        numIndexThreads = configuredIndexPoolSize;
    }

    for (int i = 0; i < numIndexThreads; i++) {
        final EventIndexTask task = new EventIndexTask(documentQueue, indexManager, directoryManager, maxEventsPerCommit, eventReporter);
        indexTasks.add(task);
        indexExecutor.submit(task);
    }

    this.config = config;
    this.indexManager = indexManager;
    this.eventConverter = new ConvertEventToLuceneDocument(config.getSearchableFields(), config.getSearchableAttributes());
}
 

@Override
public void initialize(final EventStore eventStore) {
    this.eventStore = eventStore;
    directoryManager.initialize();

    maintenanceExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Provenance Repository Maintenance"));
    maintenanceExecutor.scheduleWithFixedDelay(this::performMaintenance, 1, 1, TimeUnit.MINUTES);
    maintenanceExecutor.scheduleWithFixedDelay(this::purgeObsoleteQueries, 30, 30, TimeUnit.SECONDS);

    cachedQueries.add(new LatestEventsQuery());
    cachedQueries.add(new LatestEventsPerProcessorQuery());

    triggerReindexOfDefunctIndices();
    triggerCacheWarming();
}
 

private void triggerReindexOfDefunctIndices() {
    final ExecutorService rebuildIndexExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Rebuild Defunct Provenance Indices", true));
    final List<File> allIndexDirectories = directoryManager.getAllIndexDirectories(true, true);
    allIndexDirectories.sort(DirectoryUtils.OLDEST_INDEX_FIRST);
    final List<File> defunctIndices = detectDefunctIndices(allIndexDirectories);

    final AtomicInteger rebuildCount = new AtomicInteger(0);
    final int totalCount = defunctIndices.size();

    for (final File defunctIndex : defunctIndices) {
        try {
            if (isLucene4IndexPresent(defunctIndex)) {
                logger.info("Encountered Lucene 8 index {} and also the corresponding Lucene 4 index; will only trigger rebuilding of one directory.", defunctIndex);
                rebuildCount.incrementAndGet();
                continue;
            }

            logger.info("Determined that Lucene Index Directory {} is defunct. Will destroy and rebuild index", defunctIndex);

            final Tuple<Long, Long> timeRange = getTimeRange(defunctIndex, allIndexDirectories);
            rebuildIndexExecutor.submit(new MigrateDefunctIndex(defunctIndex, indexManager, directoryManager, timeRange.getKey(), timeRange.getValue(),
                eventStore, eventReporter, eventConverter, rebuildCount, totalCount));
        } catch (final Exception e) {
            logger.error("Detected defunct index {} but failed to rebuild index", defunctIndex, e);
        }
    }

    rebuildIndexExecutor.shutdown();

    if (!allIndexDirectories.isEmpty()) {
        final File newestIndexDirectory = allIndexDirectories.get(allIndexDirectories.size() - 1);
        if (defunctIndices.contains(newestIndexDirectory)) {
            newestIndexDefunct = true;
        }
    }
}
 

public SimpleIndexManager(final RepositoryConfiguration repoConfig) {
    this.repoConfig = repoConfig;
    this.searchExecutor = Executors.newFixedThreadPool(repoConfig.getQueryThreadPoolSize(), new NamedThreadFactory("Search Lucene Index"));
}
 

public StandardIndexManager(final RepositoryConfiguration repoConfig) {
    this.repoConfig = repoConfig;
    this.searchExecutor = Executors.newFixedThreadPool(repoConfig.getQueryThreadPoolSize(), new NamedThreadFactory("Search Lucene Index", true));
}