org.elasticsearch.common.util.concurrent.ReleasableLock Java Examples

public Translog.Operation readFromLocal(Location location) {
    try (ReleasableLock lock = readLock.acquire()) {
        ensureOpen();
        long fileGeneration = location.generation;
        FileChannel readChannel = readChannels.get(fileGeneration);
        if (readChannel == null) {
            readChannel = openReader(fileGeneration);
        }
        if (readChannel == null) {
            // throw exception since could not find reader
            throw new ElasticsearchException("could not open reader for file generation {}", fileGeneration);
        }
        ByteBuffer buffer = ByteBuffer.allocate(location.size);
        Channels.readFromFileChannelWithEofException(readChannel, location.translogLocation, buffer);
        buffer.flip();
        ByteBufferStreamInput in = new ByteBufferStreamInput(buffer);
        Translog.Operation.Type type = Translog.Operation.Type.fromId(in.readByte());
        Translog.Operation operation = DistributedTranslog.newOperationFromType(type);
        operation.readFrom(in);
        in.close();
        return operation;
    } catch (IOException e) {
        logger.error("errors while read from local", e);
        throw new ElasticsearchException("failed to read source from translog location " + location, e);
    }
}
 

@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 

/**
 * 
 * @param operation
 * @return
 * @throws IOException 
 */
public Tuple<Future<DLSN>, Tuple<BytesReference, Long>> writeOperation(Translog.Operation operation, AtomicLong txid) throws IOException {
    BytesStreamOutput out = new BytesStreamOutput();
    try (ReleasableLock lock = writeLock.acquire()) {
        Future<DLSN> writeResult = null;
        out.writeByte(operation.opType().id());
        operation.writeTo(out);
        BytesReference bytes = out.bytes();
        LogRecord logRecord = new LogRecord(txid.incrementAndGet(), bytes.toBytes());
        writeResult = logWriter.write(logRecord);
        sizeInBytes += (20 + logRecord.getPayload().length);
        ++ numOperations;
        return new Tuple<Future<DLSN>, Tuple<BytesReference, Long>>(writeResult, new Tuple<BytesReference, Long>(bytes, txid.get()));
    } catch (TransactionIdOutOfOrderException e) {
        throw e;
    } finally {
        out.close();
    }
}
 

@Override
public SnapshotIndexCommit snapshotIndex(final boolean flushFirst) throws EngineException {
    // we have to flush outside of the readlock otherwise we might have a problem upgrading
    // the to a write lock when we fail the engine in this operation
    if (flushFirst) {
        logger.trace("start flush for snapshot");
        flush(false, true);
        logger.trace("finish flush for snapshot");
    }
    try (ReleasableLock lock = readLock.acquire()) {
        ensureOpen();
        logger.trace("pulling snapshot");
        return deletionPolicy.snapshot();
    } catch (IOException e) {
        throw new SnapshotFailedEngineException(shardId, e);
    }
}
 

@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 

public Location writeToLocal(BytesReference data) throws IOException {
    final long position;
    final long generation;
    try (ReleasableLock lock = writeLock.acquire()) {
        ensureOpen();
        if (writtenOffset > TRANSLOG_ROLLING_SIZE_BYTES) {
            IOUtils.close(writeChannel);
            tmpTranslogGeneration.incrementAndGet();
            writeChannel = FileChannel.open(this.translogPath.resolve(getFileNameFromId(tmpTranslogGeneration.get())), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
            writtenOffset = 0;
        }
        generation = tmpTranslogGeneration.get();
        position = writtenOffset;
        try {
            data.writeTo(writeChannel);
        } catch (Throwable e) {
            throw e;
        }
        writtenOffset = writtenOffset + data.length();
    }
    return new Translog.Location(generation, position, data.length());
}
 

/**
 * Gets the minimum generation that could contain any sequence number after the specified sequence number, or the current generation if
 * there is no generation that could any such sequence number.
 *
 * @param seqNo the sequence number
 * @return the minimum generation for the sequence number
 */
public TranslogGeneration getMinGenerationForSeqNo(final long seqNo) {
    try (ReleasableLock ignored = readLock.acquire()) {
        /*
         * When flushing, the engine will ask the translog for the minimum generation that could contain any sequence number after the
         * local checkpoint. Immediately after flushing, there will be no such generation, so this minimum generation in this case will
         * be the current translog generation as we do not need any prior generations to have a complete history up to the current local
         * checkpoint.
         */
        long minTranslogFileGeneration = this.currentFileGeneration();
        for (final TranslogReader reader : readers) {
            if (seqNo <= reader.getCheckpoint().maxSeqNo) {
                minTranslogFileGeneration = Math.min(minTranslogFileGeneration, reader.getGeneration());
            }
        }
        return new TranslogGeneration(translogUUID, minTranslogFileGeneration);
    }
}
 

/**
 * Returns a view into the current translog that is guaranteed to retain all current operations
 * while receiving future ones as well
 */
public Translog.View newView() {
    // we need to acquire the read lock to make sure no new translog is created
    // and will be missed by the view we're making
    try (ReleasableLock lock = readLock.acquire()) {
        ensureOpen();
        ArrayList<TranslogReader> translogs = new ArrayList<>();
        try {
            if (currentCommittingTranslog != null) {
                translogs.add(currentCommittingTranslog.clone());
            }
            translogs.add(current.newReaderFromWriter());
            View view = new View(translogs, onViewClose);
            // this is safe as we know that no new translog is being made at the moment
            // (we hold a read lock) and the view will be notified of any future one
            outstandingViews.add(view);
            translogs.clear();
            return view;
        } finally {
            // close if anything happend and we didn't reach the clear
            IOUtils.closeWhileHandlingException(translogs);
        }
    }
}
 

@Override
public void commit() throws IOException {
    ImmutableTranslogReader toClose = null;
    try (ReleasableLock lock = writeLock.acquire()) {
        ensureOpen();
        if (currentCommittingTranslog == null) {
            prepareCommit();
        }
        lastCommittedTranslogFileGeneration = current.getGeneration(); // this is important - otherwise old files will not be cleaned up
        if (recoveredTranslogs.isEmpty() == false) {
            IOUtils.close(recoveredTranslogs);
            recoveredTranslogs.clear();
        }
        toClose = this.currentCommittingTranslog;
        this.currentCommittingTranslog = null;
    } finally {
        IOUtils.close(toClose);
    }
}
 

/**
 * Read the Operation object from the given location. This method will try to read the given location from
 * the current or from the currently committing translog file. If the location is in a file that has already
 * been closed or even removed the method will return <code>null</code> instead.
 */
public Translog.Operation read(Location location) {
    try (ReleasableLock lock = readLock.acquire()) {
        final TranslogReader reader;
        final long currentGeneration = current.getGeneration();
        if (currentGeneration == location.generation) {
            reader = current;
        } else if (currentCommittingTranslog != null && currentCommittingTranslog.getGeneration() == location.generation) {
            reader = currentCommittingTranslog;
        } else if (currentGeneration < location.generation) {
            throw new IllegalStateException("location generation [" + location.generation + "] is greater than the current generation [" + currentGeneration + "]");
        } else {
            return null;
        }
        return reader.read(location);
    } catch (IOException e) {
        throw new ElasticsearchException("failed to read source from translog location " + location, e);
    }
}
 

final boolean tryRenewSyncCommit() {
    boolean renewed = false;
    try (ReleasableLock lock = writeLock.acquire()) {
        ensureOpen();
        ensureCanFlush();
        String syncId = lastCommittedSegmentInfos.getUserData().get(SYNC_COMMIT_ID);
        long translogGenOfLastCommit = Long.parseLong(lastCommittedSegmentInfos.userData.get(Translog.TRANSLOG_GENERATION_KEY));
        if (syncId != null && indexWriter.hasUncommittedChanges() && translog.totalOperationsByMinGen(translogGenOfLastCommit) == 0) {
            logger.trace("start renewing sync commit [{}]", syncId);
            commitIndexWriter(indexWriter, translog, syncId);
            logger.debug("successfully sync committed. sync id [{}].", syncId);
            lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
            renewed = true;
        }
    } catch (IOException ex) {
        maybeFailEngine("renew sync commit", ex);
        throw new EngineException(shardId, "failed to renew sync commit", ex);
    }
    if (renewed) {
        // refresh outside of the write lock
        // we have to refresh internal searcher here to ensure we release unreferenced segments.
        refresh("renew sync commit", SearcherScope.INTERNAL);
    }
    return renewed;
}
 

public LocalTranslog(TranslogConfig config) throws IOException {
    super(config.getShardId(), config.getIndexSettings());
    ReadWriteLock rwl = new ReentrantReadWriteLock();
    readLock = new ReleasableLock(rwl.readLock());
    writeLock = new ReleasableLock(rwl.writeLock());
    this.translogPath = config.getTranslogPath();
    // clean all files
    Files.createDirectories(this.translogPath);
    Files.walkFileTree(this.translogPath, new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult visitFile(Path file,
                BasicFileAttributes attrs) throws IOException {
            Files.delete(file);
            return FileVisitResult.CONTINUE;
        }
    });
    
    // create a new directory
    writeChannel = FileChannel.open(this.translogPath.resolve(getFileNameFromId(tmpTranslogGeneration.get())), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
    writtenOffset = 0;
}
 

/**
 * Flush the engine (committing segments to disk and truncating the
 * translog) and close it.
 */
public void flushAndClose() throws IOException {
    if (isClosed.get() == false) {
        logger.trace("flushAndClose now acquire writeLock");
        try (ReleasableLock lock = writeLock.acquire()) {
            logger.trace("flushAndClose now acquired writeLock");
            try {
                logger.debug("flushing shard on close - this might take some time to sync files to disk");
                try {
                    flush(); // TODO we might force a flush in the future since we have the write lock already even though recoveries are running.
                } catch (AlreadyClosedException ex) {
                    logger.debug("engine already closed - skipping flushAndClose");
                }
            } finally {
                close(); // double close is not a problem
            }
        }
    }
    awaitPendingClose();
}
 

/**
 * Reads and returns the operation from the given location if the generation it references is still available. Otherwise
 * this method will return <code>null</code>.
 */
public Operation readOperation(Location location) throws IOException {
    try (ReleasableLock ignored = readLock.acquire()) {
        ensureOpen();
        if (location.generation < getMinFileGeneration()) {
            return null;
        }
        if (current.generation == location.generation) {
            // no need to fsync here the read operation will ensure that buffers are written to disk
            // if they are still in RAM and we are reading onto that position
            return current.read(location);
        } else {
            // read backwards - it's likely we need to read on that is recent
            for (int i = readers.size() - 1; i >= 0; i--) {
                TranslogReader translogReader = readers.get(i);
                if (translogReader.generation == location.generation) {
                    return translogReader.read(location);
                }
            }
        }
    } catch (final Exception ex) {
        closeOnTragicEvent(ex);
        throw ex;
    }
    return null;
}
 

public Snapshot newSnapshotFromGen(TranslogGeneration fromGeneration, long upToSeqNo) throws IOException {
    try (ReleasableLock ignored = readLock.acquire()) {
        ensureOpen();
        final long fromFileGen = fromGeneration.translogFileGeneration;
        if (fromFileGen < getMinFileGeneration()) {
            throw new IllegalArgumentException("requested snapshot generation [" + fromFileGen + "] is not available. " +
                "Min referenced generation is [" + getMinFileGeneration() + "]");
        }
        TranslogSnapshot[] snapshots = Stream.concat(readers.stream(), Stream.of(current))
            .filter(reader -> reader.getGeneration() >= fromFileGen && reader.getCheckpoint().minSeqNo <= upToSeqNo)
            .map(BaseTranslogReader::newSnapshot).toArray(TranslogSnapshot[]::new);
        final Snapshot snapshot = newMultiSnapshot(snapshots);
        if (upToSeqNo == Long.MAX_VALUE) {
            return snapshot;
        } else {
            return new SeqNoFilterSnapshot(snapshot, Long.MIN_VALUE, upToSeqNo);
        }
    }
}
 

@Override
public SnapshotIndexCommit snapshotIndex(final boolean flushFirst) throws EngineException {
    // we have to flush outside of the readlock otherwise we might have a problem upgrading
    // the to a write lock when we fail the engine in this operation
    if (flushFirst) {
        logger.trace("start flush for snapshot");
        flush(false, true);
        logger.trace("finish flush for snapshot");
    }
    try (ReleasableLock lock = readLock.acquire()) {
        ensureOpen();
        logger.trace("pulling snapshot");
        return deletionPolicy.snapshot();
    } catch (IOException e) {
        throw new SnapshotFailedEngineException(shardId, e);
    }
}
 

@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 

final boolean tryRenewSyncCommit() {
    boolean renewed = false;
    try (ReleasableLock lock = writeLock.acquire()) {
        ensureOpen();
        String syncId = lastCommittedSegmentInfos.getUserData().get(SYNC_COMMIT_ID);
        if (syncId != null && translog.totalOperations() == 0 && indexWriter.hasUncommittedChanges()) {
            logger.trace("start renewing sync commit [{}]", syncId);
            commitIndexWriter(indexWriter, translog, syncId);
            logger.debug("successfully sync committed. sync id [{}].", syncId);
            lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
            renewed = true;
        }
    } catch (IOException ex) {
        maybeFailEngine("renew sync commit", ex);
        throw new EngineException(shardId, "failed to renew sync commit", ex);
    }
    if (renewed) { // refresh outside of the write lock
        refresh("renew sync commit");
    }

    return renewed;
}
 

@Override
public void close() throws IOException {
    if (closed.compareAndSet(false, true)) {
        try (ReleasableLock lock = writeLock.acquire()) {
            try {
                current.sync();
            } finally {
                try {
                    IOUtils.close(current, currentCommittingTranslog);
                } finally {
                    IOUtils.close(recoveredTranslogs);
                    recoveredTranslogs.clear();
                }
            }
        } finally {
            FutureUtils.cancel(syncScheduler);
            logger.debug("translog closed");
        }
    }
}
 

@Override
public synchronized void sync() throws IOException {
    if (syncNeeded()) {
        ensureOpen(); // this call gives a better exception that the incRef if we are closed by a tragic event
        channelReference.incRef();
        try {
            final long offsetToSync;
            final int opsCounter;
            try (ReleasableLock lock = writeLock.acquire()) {
                flush();
                offsetToSync = totalOffset;
                opsCounter = operationCounter;
            }
            // we can do this outside of the write lock but we have to protect from
            // concurrent syncs
            ensureOpen(); // just for kicks - the checkpoint happens or not either way
            checkpoint(offsetToSync, opsCounter, channelReference);
            lastSyncedOffset = offsetToSync;
        } finally {
            channelReference.decRef();
        }
    }
}
 

public Translog(TranslogConfig config, String nodeId) {
    super(config.getShardId(), config.getIndexSettings());
    this.config = null;
    recoveredTranslogs = null;
    syncScheduler = null;
    bigArrays = null;
    ReadWriteLock rwl = new ReentrantReadWriteLock();
    readLock = new ReleasableLock(rwl.readLock());
    writeLock = new ReleasableLock(rwl.writeLock());
    location = null;
    current = null;
    currentCommittingTranslog = null;
    lastCommittedTranslogFileGeneration = -1; 
    config = null;
    translogUUID = null;
}
 

@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 

final boolean tryRenewSyncCommit() {
    boolean renewed = false;
    try (ReleasableLock lock = writeLock.acquire()) {
        ensureOpen();
        ensureCanFlush();
        String syncId = lastCommittedSegmentInfos.getUserData().get(SYNC_COMMIT_ID);
        long translogGenOfLastCommit = Long.parseLong(lastCommittedSegmentInfos.userData.get(Translog.TRANSLOG_GENERATION_KEY));
        if (syncId != null && indexWriter.hasUncommittedChanges() && translog.totalOperationsByMinGen(translogGenOfLastCommit) == 0) {
            logger.trace("start renewing sync commit [{}]", syncId);
            commitIndexWriter(indexWriter, translog, syncId);
            logger.debug("successfully sync committed. sync id [{}].", syncId);
            lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
            renewed = true;
        }
    } catch (IOException ex) {
        maybeFailEngine("renew sync commit", ex);
        throw new EngineException(shardId, "failed to renew sync commit", ex);
    }
    if (renewed) {
        // refresh outside of the write lock
        // we have to refresh internal searcher here to ensure we release unreferenced segments.
        refresh("renew sync commit", SearcherScope.INTERNAL);
    }
    return renewed;
}
 

/**
 * add the given bytes to the translog and return the location they were written at
 */
public Translog.Location add(BytesReference data) throws IOException {
    final long position;
    try (ReleasableLock lock = writeLock.acquire()) {
        ensureOpen();
        position = writtenOffset;
        try {
            data.writeTo(channel);
        } catch (Throwable e) {
            closeWithTragicEvent(e);
            throw e;
        }
        writtenOffset = writtenOffset + data.length();
        operationCounter++;;
    }
    return new Translog.Location(generation, position, data.length());
}
 

@Override
public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecoveryRunner, long recoverUpToSeqNo) throws IOException {
    flushLock.lock();
    try (ReleasableLock lock = readLock.acquire()) {
        ensureOpen();
        if (pendingTranslogRecovery.get() == false) {
            throw new IllegalStateException("Engine has already been recovered");
        }
        try {
            recoverFromTranslogInternal(translogRecoveryRunner, recoverUpToSeqNo);
        } catch (Exception e) {
            try {
                pendingTranslogRecovery.set(true); // just play safe and never allow commits on this see #ensureCanFlush
                failEngine("failed to recover from translog", e);
            } catch (Exception inner) {
                e.addSuppressed(inner);
            }
            throw e;
        }
    } finally {
        flushLock.unlock();
    }
    return this;
}
 

/**
 * returns a new immutable reader which only exposes the current written operation *
 */
public ImmutableTranslogReader immutableReader() throws TranslogException {
    if (channelReference.tryIncRef()) {
        try (ReleasableLock lock = writeLock.acquire()) {
            ensureOpen();
            flush();
            ImmutableTranslogReader reader = new ImmutableTranslogReader(this.generation, channelReference, firstOperationOffset, writtenOffset, operationCounter);
            channelReference.incRef(); // for new reader
            return reader;
        } catch (Exception e) {
            throw new TranslogException(shardId, "exception while creating an immutable reader", e);
        } finally {
            channelReference.decRef();
        }
    } else {
        throw new TranslogException(shardId, "can't increment channel [" + channelReference + "] ref count");
    }
}
 

/**
 * The a {@linkplain Location} that will sort after the {@linkplain Location} returned by the last write but before any locations which
 * can be returned by the next write.
 */
public Location getLastWriteLocation() {
    try (ReleasableLock lock = readLock.acquire()) {
        /*
         * We use position = current - 1 and size = Integer.MAX_VALUE here instead of position current and size = 0 for two reasons:
         * 1. Translog.Location's compareTo doesn't actually pay attention to size even though it's equals method does.
         * 2. It feels more right to return a *position* that is before the next write's position rather than rely on the size.
         */
        return new Location(current.generation, current.sizeInBytes() - 1, Integer.MAX_VALUE);
    }
}
 

/**
 * Returns <code>true</code> iff the given generation is the current gbeneration of this translog
 */
public boolean isCurrent(TranslogGeneration generation) {
    try (ReleasableLock lock = writeLock.acquire()) {
        if (generation != null) {
            if (generation.translogUUID.equals(translogUUID) == false) {
                throw new IllegalArgumentException("commit belongs to a different translog: " + generation.translogUUID + " vs. " + translogUUID);
            }
            return generation.translogFileGeneration == currentFileGeneration();
        }
    }
    return false;
}
 

/**
 * Ensures that the given location has be synced / written to the underlying storage.
 *
 * @return Returns <code>true</code> iff this call caused an actual sync operation otherwise <code>false</code>
 */
public boolean ensureSynced(Location location) throws IOException {
    try (ReleasableLock lock = readLock.acquire()) {
        if (location.generation == current.generation) { // if we have a new one it's already synced
            ensureOpen();
            return current.syncUpTo(location.translogLocation + location.size);
        }
    } catch (Throwable ex) {
        closeOnTragicEvent(ex);
        throw ex;
    }
    return false;
}
 

@Override
public void close() throws IOException {
    if (isClosed.get() == false) { // don't acquire the write lock if we are already closed
        logger.debug("close now acquiring writeLock");
        try (ReleasableLock lock = writeLock.acquire()) {
            logger.debug("close acquired writeLock");
            closeNoLock("api", closedLatch);
        }
    }
    awaitPendingClose();
}