Java Code Examples for org.apache.nifi.provenance.index.EventIndexWriter#commit()

The following examples show how to use org.apache.nifi.provenance.index.EventIndexWriter#commit() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MigrateDefunctIndex.java    From nifi with Apache License 2.0 6 votes vote down vote up 
private void rebuildIndex(final File tempIndexDir, final File migratedIndexDir) throws IOException {
    final EventIndexWriter writer = indexManager.borrowIndexWriter(tempIndexDir);

    try {
        final EventIterator eventIterator = eventStore.getEventsByTimestamp(minTimestamp, maxTimestamp);

        final StopWatch stopWatch = new StopWatch(true);

        Optional<ProvenanceEventRecord> optionalEvent;
        while ((optionalEvent = eventIterator.nextEvent()).isPresent()) {
            final ProvenanceEventRecord event = optionalEvent.get();

            final Document document = eventConverter.convert(event, event.getEventId());
            writer.index(document, Integer.MAX_VALUE);
            successCount++;
        }

        writer.commit();
        stopWatch.stop();
        logger.info("Successfully indexed {} events to {} in {}", successCount, tempIndexDir, stopWatch.getDuration());
    } finally {
        indexManager.returnIndexWriter(writer, true, true);
    }

    Files.move(tempIndexDir.toPath(), migratedIndexDir.toPath());
}
Example 2
Source File: LuceneEventIndex.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Override
public void commitChanges(final String partitionName) throws IOException {
    final Optional<File> indexDir = directoryManager.getActiveIndexDirectory(partitionName);
    if (indexDir.isPresent()) {
        final EventIndexWriter eventIndexWriter = indexManager.borrowIndexWriter(indexDir.get());
        try {
            eventIndexWriter.commit();
        } finally {
            indexManager.returnIndexWriter(eventIndexWriter, false, false);
        }
    }
}
Example 3
Source File: LuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Override
public void commitChanges(final String partitionName) throws IOException {
    final Optional<File> indexDir = directoryManager.getActiveIndexDirectory(partitionName);
    if (indexDir.isPresent()) {
        final EventIndexWriter eventIndexWriter = indexManager.borrowIndexWriter(indexDir.get());
        try {
            eventIndexWriter.commit();
        } finally {
            indexManager.returnIndexWriter(eventIndexWriter, false, false);
        }
    }
}
Example 4
Source File: EventIndexTask.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up 
protected void commit(final EventIndexWriter indexWriter) throws IOException {
    final long start = System.nanoTime();
    final long approximateCommitCount = indexWriter.commit();
    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    logger.debug("Successfully committed approximately {} Events to {} in {} millis", approximateCommitCount, indexWriter, millis);
}
Example 5
Source File: CachingIndexManager.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up 
@Override
public void returnIndexWriter(final EventIndexWriter writer, final boolean commit, final boolean isCloseable) {
    final File indexDirectory = writer.getDirectory();
    final File absoluteFile = indexDirectory.getAbsoluteFile();
    logger.trace("Returning Index Writer for {} to IndexManager", indexDirectory);

    lock.lock();
    try {
        final IndexWriterCount count = writerCounts.get(absoluteFile);

        try {
            if (count == null) {
                logger.warn("Index Writer {} was returned to IndexManager for {}, but this writer is not known. "
                    + "This could potentially lead to a resource leak", writer, indexDirectory);
                writer.close();
            } else if (count.getCount() <= 1) {
                // we are finished with this writer.
                logger.info("Decrementing count for Index Writer for {} to {}. Now finished writing to this Index Directory",
                    indexDirectory, count.getCount() - 1);

                try {
                    if (commit) {
                        writer.commit();
                    }
                } finally {
                    if (isCloseable) {
                        try {
                            count.close();
                        } finally {
                            writerCounts.remove(absoluteFile);
                        }
                    }
                }
            } else {
                // decrement the count.
                logger.debug("Decrementing count for Index Writer for {} to {}", indexDirectory, count.getCount() - 1);
                writerCounts.put(absoluteFile, new IndexWriterCount(count.getWriter(), count.getAnalyzer(), count.getDirectory(), count.getCount() - 1));
            }
        } catch (final IOException ioe) {
            logger.warn("Failed to close Index Writer {} due to {}", writer, ioe);
            if (logger.isDebugEnabled()) {
                logger.warn("", ioe);
            }
        }
    } finally {
        lock.unlock();
    }
}
Example 6
Source File: SimpleIndexManager.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up 
@Override
public void returnIndexWriter(final EventIndexWriter writer, final boolean commit, final boolean isCloseable) {
    final File indexDirectory = writer.getDirectory();
    final File absoluteFile = indexDirectory.getAbsoluteFile();
    logger.trace("Returning Index Writer for {} to IndexManager", indexDirectory);

    boolean unused = false;
    IndexWriterCount count = null;
    boolean close = isCloseable;
    try {
        synchronized (writerCounts) {
            count = writerCounts.get(absoluteFile);
            if (count != null && count.isCloseableWhenUnused()) {
                close = true;
            }

            if (count == null) {
                logger.warn("Index Writer {} was returned to IndexManager for {}, but this writer is not known. "
                    + "This could potentially lead to a resource leak", writer, indexDirectory);
                writer.close();
            } else if (count.getCount() <= 1) {
                // we are finished with this writer.
                unused = true;
                if (close) {
                    logger.debug("Decrementing count for Index Writer for {} to {}; closing writer", indexDirectory, count.getCount() - 1);
                    writerCounts.remove(absoluteFile);
                } else {
                    logger.trace("Decrementing count for Index Writer for {} to {}", indexDirectory, count.getCount() - 1);

                    // If writer is not closeable, then we need to decrement its count.
                    writerCounts.put(absoluteFile, new IndexWriterCount(count.getWriter(), count.getAnalyzer(), count.getDirectory(),
                        count.getCount() - 1, close));
                }
            } else {
                // decrement the count.
                if (close) {
                    logger.debug("Decrementing count for Index Writer for {} to {} and marking as closeable when no longer in use", indexDirectory, count.getCount() - 1);
                } else {
                    logger.trace("Decrementing count for Index Writer for {} to {}", indexDirectory, count.getCount() - 1);
                }

                writerCounts.put(absoluteFile, new IndexWriterCount(count.getWriter(), count.getAnalyzer(),
                    count.getDirectory(), count.getCount() - 1, close));
            }

            if (writerCounts.size() > repoConfig.getStorageDirectories().size() * 2) {
                logger.debug("Index Writer returned; writer count map now has size {}; writer = {}, commit = {}, isCloseable = {}, writerCount = {}; full writerCounts Map = {}",
                    writerCounts.size(), writer, commit, isCloseable, count, writerCounts);
            }
        }

        // Committing and closing are very expensive, so we want to do those outside of the synchronized block.
        // So we use an 'unused' variable to tell us whether or not we should actually do so.
        if (unused) {
            try {
                if (commit) {
                    writer.commit();
                }
            } finally {
                if (close) {
                    logger.info("Index Writer for {} has been returned to Index Manager and is no longer in use. Closing Index Writer", indexDirectory);
                    close(count);
                }
            }
        }
    } catch (final Exception e) {
        logger.warn("Failed to close Index Writer {} due to {}", writer, e.toString(), e);
    }
}
Example 7
Source File: EventIndexTask.java    From nifi with Apache License 2.0 4 votes vote down vote up 
protected void commit(final EventIndexWriter indexWriter) throws IOException {
    final long start = System.nanoTime();
    final long approximateCommitCount = indexWriter.commit();
    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    logger.debug("Successfully committed approximately {} Events to {} in {} millis", approximateCommitCount, indexWriter, millis);
}
Example 8
Source File: StandardIndexManager.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@Override
public void returnIndexWriter(final EventIndexWriter writer, final boolean commit, final boolean isCloseable) {
    final File indexDirectory = writer.getDirectory();
    final File absoluteFile = indexDirectory.getAbsoluteFile();
    logger.trace("Returning Index Writer for {} to IndexManager", indexDirectory);

    boolean unused = false;
    IndexWriterCount count;
    boolean close = isCloseable;
    try {
        synchronized (writerCounts) {
            count = writerCounts.get(absoluteFile);
            if (count != null && count.isCloseableWhenUnused()) {
                close = true;
            }

            if (count == null) {
                logger.warn("Index Writer {} was returned to IndexManager for {}, but this writer is not known. "
                    + "This could potentially lead to a resource leak", writer, indexDirectory);
                writer.close();
            } else if (count.getCount() <= 1) {
                // we are finished with this writer.
                unused = true;
                if (close) {
                    logger.debug("Decrementing count for Index Writer for {} to {}; closing writer", indexDirectory, count.getCount() - 1);
                    writerCounts.remove(absoluteFile);
                } else {
                    logger.trace("Decrementing count for Index Writer for {} to {}", indexDirectory, count.getCount() - 1);

                    // If writer is not closeable, then we need to decrement its count.
                    writerCounts.put(absoluteFile, new IndexWriterCount(count.getWriter(), count.getAnalyzer(), count.getDirectory(),
                        count.getCount() - 1, close));
                }
            } else {
                // decrement the count.
                if (close) {
                    logger.debug("Decrementing count for Index Writer for {} to {} and marking as closeable when no longer in use", indexDirectory, count.getCount() - 1);
                } else {
                    logger.trace("Decrementing count for Index Writer for {} to {}", indexDirectory, count.getCount() - 1);
                }

                writerCounts.put(absoluteFile, new IndexWriterCount(count.getWriter(), count.getAnalyzer(),
                    count.getDirectory(), count.getCount() - 1, close));
            }

            if (writerCounts.size() > repoConfig.getStorageDirectories().size() * 2) {
                logger.debug("Index Writer returned; writer count map now has size {}; writer = {}, commit = {}, isCloseable = {}, writerCount = {}; full writerCounts Map = {}",
                    writerCounts.size(), writer, commit, isCloseable, count, writerCounts);
            }
        }

        // Committing and closing are very expensive, so we want to do those outside of the synchronized block.
        // So we use an 'unused' variable to tell us whether or not we should actually do so.
        if (unused) {
            try {
                if (commit) {
                    writer.commit();
                }
            } finally {
                if (close) {
                    logger.info("Index Writer for {} has been returned to Index Manager and is no longer in use. Closing Index Writer", indexDirectory);
                    close(count);
                }
            }
        }
    } catch (final Exception e) {
        logger.warn("Failed to close Index Writer {} due to {}", writer, e.toString(), e);
    }
}