com.indeed.util.core.io.Closeables2 Java Examples

The following examples show how to use com.indeed.util.core.io.Closeables2. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StableGeneration.java    From lsmtree with Apache License 2.0 6 votes vote down vote up
public BlockCompressedStableGeneration(
        BloomFilter.MemoryManager memoryManager, File file, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final CompressionCodec codec, final boolean mlockBTree
) throws IOException {
    this.file = file;
    reader = new ImmutableBTreeIndex.Reader(file, comparator, keySerializer, new LongSerializer(), mlockBTree);
    final File valuesFile = new File(file, "values.bin");
    recordFile =
            new BlockCompressedRecordFile.Builder(valuesFile, valueSerializer, codec).setMlockFiles(mlockBTree).build();
    final File bloomFilterFile = new File(file, "bloomfilter.bin");
    if (bloomFilterFile.exists()) {
        bloomFilter = new BloomFilter.Reader(memoryManager, bloomFilterFile, keySerializer);
    } else {
        bloomFilter = null;
    }
    sizeInBytes = reader.sizeInBytes()+valuesFile.length()+(bloomFilter == null ? 0 : bloomFilter.sizeInBytes());
    stuffToClose = SharedReference.create((Closeable)new Closeable() {
        public void close() throws IOException {
            Closeables2.closeQuietly(reader, log);
            if (bloomFilter != null) Closeables2.closeQuietly(bloomFilter, log);
            Closeables2.closeQuietly(recordFile, log);
        }
    });
}
 
Example #2
Source File: Store.java    From lsmtree with Apache License 2.0 6 votes vote down vote up
public void checkpoint(File checkpointDir) throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        checkpointDir.mkdirs();
        localState.get().volatileGeneration.checkpoint(checkpointDir);
        for (Generation<K, V> generation : localState.get().stableGenerations) {
            generation.checkpoint(checkpointDir);
        }
        PosixFileOperations.cplr(new File(localState.get().path, "state"), checkpointDir);
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 
Example #3
Source File: FlamdexFTGSIterator.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Override
public final void close() {
    synchronized (session) {
        if (intTermDocIterator != null) {
            Closeables2.closeQuietly(intTermDocIterator, ImhotepLocalSession.log);
            intTermDocIterator = null;
        }
        if (stringTermDocIterator != null) {
            Closeables2.closeQuietly(stringTermDocIterator, ImhotepLocalSession.log);
            stringTermDocIterator = null;
        }
        if (flamdexReader != null) {
            Closeables2.closeQuietly(flamdexReader, ImhotepLocalSession.log);
            flamdexReader = null;
        }
    }
}
 
Example #4
Source File: FlamdexSubsetFTGSIterator.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Override
public final void close() {
    synchronized (session) {
        if (docIdStream != null) {
            Closeables2.closeQuietly(docIdStream, ImhotepLocalSession.log);
        }
        if (intTermIterator != null) {
            Closeables2.closeQuietly(intTermIterator, ImhotepLocalSession.log);
            intTermIterator = null;
        }
        if (stringTermIterator != null) {
            Closeables2.closeQuietly(stringTermIterator, ImhotepLocalSession.log);
            stringTermIterator = null;
        }
        if (flamdexReader != null) {
            Closeables2.closeQuietly(flamdexReader, ImhotepLocalSession.log);
            flamdexReader = null;
        }
    }
}
 
Example #5
Source File: Store.java    From lsmtree with Apache License 2.0 6 votes vote down vote up
/**
 * Flushes volatile generation to disk.
 *
 * @throws IOException  if an I/O error occurs
 */
public void sync() throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        try {
            localState.get().volatileGeneration.sync();
        } catch (IOException e) {
            compactor.compact();
            throw e;
        }
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 
Example #6
Source File: AbstractImhotepMultiSession.java    From imhotep with Apache License 2.0 6 votes vote down vote up
public RawFTGSIterator mergeFTGSSplit(final String[] intFields, final String[] stringFields, final String sessionId, final InetSocketAddress[] nodes, final int splitIndex) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[nodes.length];
    try {
        execute(splits, nodes, new ThrowingFunction<InetSocketAddress, RawFTGSIterator>() {
            public RawFTGSIterator apply(final InetSocketAddress node) throws Exception {
                final ImhotepRemoteSession remoteSession = new ImhotepRemoteSession(node.getHostName(), node.getPort(), sessionId, tempFileSizeBytesLeft);
                remoteSession.setNumStats(numStats);
                return remoteSession.getFTGSIteratorSplit(intFields, stringFields, splitIndex, nodes.length);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return mergeFTGSSplits(splits);
}
 
Example #7
Source File: RecordLogDirectory.java    From lsmtree with Apache License 2.0 6 votes vote down vote up
@Override
public E get(long address) throws IOException {
    final int segmentNum = (int)(address>>> segmentShift);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get(segmentNum);
    if (option.isNone()) {
        throw new IOException("address is invalid: "+address);
    }
    final SharedReference<BlockCompressedRecordFile<E>> reference = option.some();
    final E ret;
    try {
        ret = reference.get().get(address & segmentMask);
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
    return ret;
}
 
Example #8
Source File: BlockCompressedRecordFile.java    From lsmtree with Apache License 2.0 6 votes vote down vote up
public BlockCompressedRecordFile(final Supplier<? extends Either<IOException, ? extends RandomAccessDataInput>> inputSupplier, final Closeable closeable, String file, Serializer<E> serializer, CompressionCodec codec, BlockingQueue<Decompressor> decompressorPool, int blockSize, int recordIndexBits, int padBits, int maxChunkSize) throws IOException {
    this.inputSupplier = inputSupplier;
    this.file = file;
    this.serializer = serializer;
    this.codec = codec;
    this.blockSize = blockSize;
    this.padBits = padBits;
    this.maxChunkSize = maxChunkSize;
    pad = 1<<padBits;
    padMask = ~(long)(pad-1);
    shift = Math.max(recordIndexBits - padBits, 0);
    mask = (1L<<recordIndexBits)-1;
    closeableRef = SharedReference.create(closeable);
    try {
        blockCache = new BlockCache(decompressorPool);
    } catch (Throwable t) {
        Closeables2.closeQuietly(closeableRef, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 
Example #9
Source File: AbstractImhotepMultiSession.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Override
public RawFTGSIterator mergeSubsetFTGSSplit(final Map<String, long[]> intFields, final Map<String, String[]> stringFields, final String sessionId, final InetSocketAddress[] nodes, final int splitIndex) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[nodes.length];
    try {
        execute(splits, nodes, new ThrowingFunction<InetSocketAddress, RawFTGSIterator>() {
            public RawFTGSIterator apply(final InetSocketAddress node) throws Exception {
                final ImhotepRemoteSession remoteSession = new ImhotepRemoteSession(node.getHostName(), node.getPort(), sessionId, tempFileSizeBytesLeft);
                remoteSession.setNumStats(numStats);
                return remoteSession.getSubsetFTGSIteratorSplit(intFields, stringFields, splitIndex, nodes.length);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return mergeFTGSSplits(splits);
}
 
Example #10
Source File: Store.java    From lsmtree with Apache License 2.0 6 votes vote down vote up
public void compact() throws IOException {
    lock.lock();
    try {
        if (!closed) {
            final SharedReference<GenerationState<K, V>> localStateReference = generationState.getCopy();
            try {
                if (localStateReference == null) return;
                final GenerationState<K, V> localState = localStateReference.get();
                //this is double checked locking but in this case it doesn't really matter since it's just a heuristic
                if (localState.volatileGeneration.sizeInBytes() > maxVolatileGenerationSize) {
                    final GenerationState<K, V> nextState = startNewLog(localState);
                    startCompaction(nextState);
                }
            } finally {
                Closeables2.closeQuietly(localStateReference, log);
            }
        }
    } finally {
        lock.unlock();
    }
}
 
Example #11
Source File: CachedFlamdexReader.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Override
public void close() {
    try {
        if (readLockRef == null) {
            Closeables2.closeAll(log, metricCache, wrapped);
        } else {
            Closeables2.closeAll(log, metricCache, wrapped, readLockRef);                
        }
    } finally {
        if (memory == null) {
            return;
        }
        if (memory.usedMemory() > 0) {
            log.error("CachedFlamdexReader is leaking! memory reserved after all memory has been freed: "+memory.usedMemory());
        }
        Closeables2.closeQuietly(memory, log);
    }
}
 
Example #12
Source File: Store.java    From lsmtree with Apache License 2.0 6 votes vote down vote up
private GenerationState<K, V> startNewLog(final GenerationState<K, V> localState) throws IOException {
    //create new volatile generation and checkpoint
    final File newLog = getNextLogFile();
    final VolatileGeneration<K,V> nextVolatileGeneration = new VolatileGeneration<K, V>(newLog, keySerializer, valueSerializer, comparator);
    final List<SharedReference<? extends Generation<K,V>>> nextStableGenerations = Lists.newArrayList();
    nextStableGenerations.add(localState.volatileGenerationReference.copy());
    for (SharedReference<? extends Generation<K, V>> reference : localState.stableGenerationReferences) {
        nextStableGenerations.add(reference.copy());
    }
    final File checkpointDir = getNextCheckpointDir();
    checkpointDir.mkdirs();
    final GenerationState<K,V> nextState = new GenerationState<K, V>(nextStableGenerations, SharedReference.create(nextVolatileGeneration), checkpointDir);
    checkpointGenerationState(nextState, checkpointDir);
    //there will be a brief period of time where there is no writable generation, put and delete will block during this time
    localState.volatileGeneration.closeWriter();
    PosixFileOperations.atomicLink(checkpointDir, new File(root, "latest"));
    final SharedReference<GenerationState<K, V>> oldState = Preconditions.checkNotNull(generationState.getAndSet(nextState));
    oldState.get().delete();
    Closeables2.closeQuietly(oldState, log);
    return nextState;
}
 
Example #13
Source File: MetricCacheImpl.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Override
public void close() {
    synchronized (loadedMetrics) {
        if (!closed) {
            closed = true;
            try {
                Closeables2.closeAll(log, Collections2.transform(loadedMetrics.entrySet(), new Function<Map.Entry<String, IntValueLookup>, Closeable>() {
                    public Closeable apply(final Map.Entry<String, IntValueLookup> metric) {
                        return closeMetric.asCloseable(metric);
                    }
                }));
            } finally {
                loadedMetrics.clear();
            }
        }
    }
}
 
Example #14
Source File: RecordLogDirectory.java    From lsmtree with Apache License 2.0 5 votes vote down vote up
private boolean getSegmentReader(int segmentNum) throws IOException {
    Closeables2.closeQuietly(currentReader, log);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get(segmentNum);
    for (SharedReference<BlockCompressedRecordFile<E>> reference : option) {
        try {
            currentReader = reference.get().reader();
        } finally {
            Closeables2.closeQuietly(reference, log);
        }
        return true;
    }
    return false;
}
 
Example #15
Source File: BloomFilter.java    From lsmtree with Apache License 2.0 5 votes vote down vote up
/**
 * writing to or reading from referenced pages after this call will result in undefined behaviour
 * @throws IOException in exceptional circumstances
 */
@Override
public void close() throws IOException {
    try {
        sync();
    } finally {
        synchronized (addressSpaces) {
            addressSpaces.remove(this);
        }
        for (PageTableEntry pte : pageTable) {
            final Memory freePage;
            synchronized (pte) {
                if (pte.memory != null) {
                    final long address = (long)pte.index << PAGE_BITS;
                    if (log.isDebugEnabled()) log.debug("evicting page in file "+file.getPath()+" at address "+ address);
                    if (pte.dirty) {
                        log.error("page in file "+file.getPath()+" at address "+address+" is dirty");
                    }
                    if (pte.refCount > 0) {
                        log.error("page in file "+file.getPath()+" at address "+address+" is still in use, refcount: "+pte.refCount);
                    }
                }
                freePage = pte.memory;
                pte.memory = null;
            }
            if (freePage != null) {
                synchronized (freePages) {
                    freePages.add(freePage);
                }
            }
        }
        Closeables2.closeQuietly(raf, log);
    }
}
 
Example #16
Source File: SafeFiles.java    From util with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void close() throws IOException {
    if (! closed) {
        Closeables2.closeQuietly(fileChannel, LOG);
        deleteIfExistsQuietly(tempFile);

        closed = true;
    }
}
 
Example #17
Source File: AtomicSharedReference.java    From util with Apache License 2.0 5 votes vote down vote up
/**
 * Just like mapWithCopy() except that this silently swallows any exception
 * that calling close() on the copy might throw.
 *
 * @param function The function to apply to the value of the local reference.
 * @param <Z> The type of object produced by the function.
 * @return The value that was produced by the supplied {@code function}.
 */
public @Nullable <Z> Z mapWithCopyQuietly(Function<T, Z> function) {
    final @Nullable SharedReference<T> localRef = getCopy();
    try {
        if (localRef == null) {
            return function.apply(null);
        } else {
            return function.apply(localRef.get());
        }
    } finally {
        if (localRef != null) Closeables2.closeQuietly(localRef, log);
    }
}
 
Example #18
Source File: AbstractFTGSMerger.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void close() {
    if (!done) {
        done = true;
        Closeables2.closeAll(log, Closeables2.forArray(log, iterators), doneCallback);
    }
}
 
Example #19
Source File: RecordLogDirectory.java    From lsmtree with Apache License 2.0 5 votes vote down vote up
public Reader(long address) throws IOException {
    currentSegmentNum = (int)(address>>> segmentShift);
    final Option<SharedReference<BlockCompressedRecordFile<E>>> recordFile = fileCache.get(currentSegmentNum);
    if (recordFile.isNone()) {
        throw new IOException("address is invalid: "+address);
    }
    final SharedReference<BlockCompressedRecordFile<E>> reference = recordFile.some();
    try {
        currentReader = reference.get().reader(address & segmentMask);
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
}
 
Example #20
Source File: RecordLogDirectory.java    From lsmtree with Apache License 2.0 5 votes vote down vote up
public Option<RecordFile.Reader<E>> getFileReader(final long segmentNum) throws IOException {
    final Option<SharedReference<BlockCompressedRecordFile<E>>> option = fileCache.get((int) segmentNum);
    if (option.isNone()) return Option.none();
    final SharedReference<BlockCompressedRecordFile<E>> reference = option.some();
    final RecordFile.Reader<E> segmentReader;
    try {
        segmentReader = reference.get().reader();
    } finally {
        Closeables2.closeQuietly(reference, log);
    }
    return Option.<RecordFile.Reader<E>>some(new RecordFile.Reader<E>() {

        final long segmentShift = 64-fileIndexBits;
        final long maxSegmentPosition = (1L << segmentShift)-1;

        @Override
        public boolean next() throws IOException {
            return segmentReader.next();
        }

        @Override
        public long getPosition() {
            final long segmentPosition = segmentReader.getPosition();
            if (segmentPosition > maxSegmentPosition) {
                throw new IllegalStateException("position in segment file"+segmentNum+" is too high to be addressable in record log directory with "+fileIndexBits+" file index bits");
            }
            return (segmentNum<<segmentShift)+segmentPosition;
        }

        @Override
        public E get() {
            return segmentReader.get();
        }

        @Override
        public void close() throws IOException {
            Closeables2.closeQuietly(segmentReader, log);
        }
    });
}
 
Example #21
Source File: Store.java    From lsmtree with Apache License 2.0 5 votes vote down vote up
private void finishClose() throws IOException {
    try {
        final SharedReference<GenerationState<K, V>> state = generationState.getAndUnset();
        try {
            if (state != null) {
                final VolatileGeneration<K, V> volatileGeneration = state.get().volatileGeneration;
                if (volatileGeneration != null) volatileGeneration.closeWriter();
            }
        } finally {
            Closeables2.closeQuietly(state, log);
        }
    } finally {
        threadPool.shutdown();
    }
}
 
Example #22
Source File: Store.java    From lsmtree with Apache License 2.0 5 votes vote down vote up
/**
 * Close the store, clean up lock files.
 *
 * @throws IOException  if an I/O error occurs
 */
@Override
public void close() throws IOException {
    if (closed) return;
    closed = true;
    if (compactor != null) {
        Closeables2.closeQuietly(compactor, log);
    } else {
        Closeables2.closeQuietly(generationState.getAndUnset(), log);
    }
    if (lockFile != null) {
        lockFile.delete();
    }
}
 
Example #23
Source File: ImmutableBTreeIndex.java    From lsmtree with Apache License 2.0 5 votes vote down vote up
public Reader(Path path, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final boolean mlockFiles) throws IOException {
    this.comparator = comparator;
    indexFile = path.resolve("index.bin");
    sizeInBytes = Files.size(indexFile);
    buffer = new MMapBuffer(indexFile, FileChannel.MapMode.READ_ONLY, ByteOrder.LITTLE_ENDIAN);
    try {
        stuffToClose = SharedReference.create((Closeable)buffer);
        final MemoryDataInput in = new MemoryDataInput(buffer.memory());
        if (sizeInBytes < Header.length()) {
            throw new IOException("file is less than header length bytes");
        }
        final byte[] headerBytes = new byte[Header.length()];
        in.seek(sizeInBytes - Header.length());
        in.readFully(headerBytes);
        final LittleEndianDataInputStream headerStream = new LittleEndianDataInputStream(new ByteArrayInputStream(headerBytes));
        final Header header = new HeaderSerializer().read(headerStream);
        hasDeletions = header.hasDeletions;
        size = header.size;
        if (header.fileLength != sizeInBytes) {
            log.error(header.fileLength);
            throw new IOException("file length written to last 8 bytes of file does not match file length, file is inconsistent");
        }
        rootLevel = Level.build(buffer.memory(), keySerializer, valueSerializer, comparator, header.hasDeletions, header.indexLevels);
        rootLevelStartAddress = header.rootLevelStartAddress;
        if (mlockFiles) buffer.mlock(0, buffer.memory().length());
    } catch (Throwable t) {
        Closeables2.closeQuietly(buffer, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 
Example #24
Source File: DocIteratorMerger.java    From imhotep with Apache License 2.0 5 votes vote down vote up
public synchronized void close() {
    try {
        Closeables2.closeAll(log, circularBuffer.getInputStream(), circularBuffer.getOutputStream(), iterator);
    } finally {
        try {
            executorService.shutdownNow();
            executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            throw Throwables.propagate(e);
        } finally {
            Closeables2.closeAll(log, iterators);
        }
    }
}
 
Example #25
Source File: FTGSSplitter.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
public void close() {
    if (done.compareAndSet(false, true)) {
        try {
            if (Thread.currentThread() != runThread) {
                while (true) {
                    try {
                        runThread.interrupt();
                        runThread.join(10);
                        if (!runThread.isAlive()) break;
                    } catch (InterruptedException e) {
                        //ignore
                    }
                }
            }
        } finally {
            Closeables2.closeAll(log, iterator, Closeables2.forIterable(log, Iterables.transform(Arrays.asList(files), new Function<File, Closeable>() {
                public Closeable apply(final File input) {
                    return new Closeable() {
                        public void close() throws IOException {
                            input.delete();
                        }
                    };
                }
            })), Closeables2.forArray(log, outputs), Closeables2.forArray(log, ftgsIterators), Closeables2.forArray(log, outputStreams));
        }
    }
}
 
Example #26
Source File: AbstractImhotepMultiSession.java    From imhotep with Apache License 2.0 5 votes vote down vote up
protected void preClose() {
    try {
        if (lastIterator != null) {
            Closeables2.closeQuietly(lastIterator, log);
            lastIterator = null;
        }
    } finally {
        getSplitBufferThreads.shutdown();
        mergeSplitBufferThreads.shutdown();
    }
}
 
Example #27
Source File: AbstractImhotepMultiSession.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
public RawFTGSIterator getSubsetFTGSIteratorSplit(final Map<String, long[]> intFields, final Map<String, String[]> stringFields, final int splitIndex, final int numSplits) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[sessions.length];
    try {
        executeSessions(splits, new ThrowingFunction<ImhotepSession, RawFTGSIterator>() {
            public RawFTGSIterator apply(final ImhotepSession imhotepSession) throws Exception {
                return imhotepSession.getSubsetFTGSIteratorSplit(intFields, stringFields, splitIndex, numSplits);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return new RawFTGSMerger(Arrays.asList(splits), numStats, null);
}
 
Example #28
Source File: AbstractImhotepMultiSession.java    From imhotep with Apache License 2.0 5 votes vote down vote up
public RawFTGSIterator getFTGSIteratorSplit(final String[] intFields, final String[] stringFields, final int splitIndex, final int numSplits) {
    final RawFTGSIterator[] splits = new RawFTGSIterator[sessions.length];
    try {
        executeSessions(splits, new ThrowingFunction<ImhotepSession, RawFTGSIterator>() {
            public RawFTGSIterator apply(final ImhotepSession imhotepSession) throws Exception {
                return imhotepSession.getFTGSIteratorSplit(intFields, stringFields, splitIndex, numSplits);
            }
        });
    } catch (Throwable t) {
        Closeables2.closeAll(log, splits);
        throw Throwables.propagate(t);
    }
    return new RawFTGSMerger(Arrays.asList(splits), numStats, null);
}
 
Example #29
Source File: AbstractImhotepMultiSession.java    From imhotep with Apache License 2.0 5 votes vote down vote up
public final DocIterator getDocIterator(String[] intFields, String[] stringFields) throws ImhotepOutOfMemoryException {
    final Closer closer = Closer.create();
    try {
        final List<DocIterator> docIterators = Lists.newArrayList();
        for (ImhotepSession session : sessions) {
            docIterators.add(closer.register(session.getDocIterator(intFields, stringFields)));
        }
        return new DocIteratorMerger(docIterators, intFields.length, stringFields.length);
    } catch (Throwable t) {
        Closeables2.closeQuietly(closer, log);
        throw Throwables2.propagate(t, ImhotepOutOfMemoryException.class);
    }
}
 
Example #30
Source File: FastIntFTGSMerger.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void close() {
    if (!done) {
        done = true;
        Closeables2.closeAll(log, Closeables2.forArray(log, iterators), doneCallback);
    }
}