org.apache.ratis.proto.RaftProtos.LogEntryProto Java Examples
The following examples show how to use
org.apache.ratis.proto.RaftProtos.LogEntryProto.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SegmentedRaftLog.java From ratis with Apache License 2.0 | 6 votes |
@Override public LogEntryProto get(long index) throws RaftLogIOException { checkLogState(); LogSegment segment; LogRecordWithEntry recordAndEntry; try (AutoCloseableLock readLock = readLock()) { segment = cache.getSegment(index); if (segment == null) { return null; } recordAndEntry = segment.getEntryWithoutLoading(index); if (recordAndEntry == null) { return null; } if (recordAndEntry.hasEntry()) { return recordAndEntry.getEntry(); } } // the entry is not in the segment's cache. Load the cache without holding // RaftLog's lock. checkAndEvictCache(); return segment.loadCache(recordAndEntry.getRecord()); }
Example #2
Source File: TestSegmentedRaftLog.java From ratis with Apache License 2.0 | 6 votes |
@Test public void testTruncate() throws Exception { // prepare the log for truncation List<SegmentRange> ranges = prepareRanges(0, 5, 200, 0); List<LogEntryProto> entries = prepareLogEntries(ranges, null); try (SegmentedRaftLog raftLog = new SegmentedRaftLog(peerId, null, storage, -1, properties)) { raftLog.open(RaftServerConstants.INVALID_LOG_INDEX, null); // append entries to the raftlog entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join); } for (long fromIndex = 900; fromIndex >= 0; fromIndex -= 150) { testTruncate(entries, fromIndex); } }
Example #3
Source File: ServerState.java From ratis with Apache License 2.0 | 6 votes |
/** * note we do not apply log entries to the state machine here since we do not * know whether they have been committed. */ private RaftLog initLog(RaftPeerId id, RaftProperties prop, long lastIndexInSnapshot, Consumer<LogEntryProto> logConsumer) throws IOException { final RaftLog log; if (RaftServerConfigKeys.Log.useMemory(prop)) { final int maxBufferSize = RaftServerConfigKeys.Log.Appender.bufferByteLimit(prop).getSizeInt(); log = new MemoryRaftLog(id, lastIndexInSnapshot, maxBufferSize); } else { log = new SegmentedRaftLog(id, server, this.storage, lastIndexInSnapshot, prop); } log.open(lastIndexInSnapshot, logConsumer); return log; }
Example #4
Source File: RaftLog.java From incubator-ratis with Apache License 2.0 | 6 votes |
/** * Validate the term and index of entry w.r.t RaftLog */ protected void validateLogEntry(LogEntryProto entry) { if (entry.hasMetadataEntry()) { return; } long latestSnapshotIndex = getSnapshotIndex(); TermIndex lastTermIndex = getLastEntryTermIndex(); if (lastTermIndex != null) { long lastIndex = lastTermIndex.getIndex() > latestSnapshotIndex ? lastTermIndex.getIndex() : latestSnapshotIndex; Preconditions.assertTrue(entry.getTerm() >= lastTermIndex.getTerm(), "Entry term less than RaftLog's last term: %d, entry: %s", lastTermIndex.getTerm(), entry); Preconditions.assertTrue(entry.getIndex() == lastIndex + 1, "Difference between entry index and RaftLog's last index %d (or snapshot index %d) " + "is greater than 1, entry: %s", lastTermIndex.getIndex(), latestSnapshotIndex, entry); } else { Preconditions.assertTrue(entry.getIndex() == latestSnapshotIndex + 1, "Difference between entry index and RaftLog's latest snapshot index %d is greater than 1 " + "and in between log entries are not present, entry: %s", latestSnapshotIndex, entry); } }
Example #5
Source File: TestSegmentedRaftLogCache.java From incubator-ratis with Apache License 2.0 | 6 votes |
private void checkCache(long start, long end, int segmentSize) { Assert.assertEquals(start, cache.getStartIndex()); Assert.assertEquals(end, cache.getEndIndex()); for (long index = start; index <= end; index++) { final LogSegment segment = cache.getSegment(index); final LogRecord record = segment.getLogRecord(index); final LogEntryProto entry = segment.getEntryFromCache(record.getTermIndex()); Assert.assertEquals(index, entry.getIndex()); } long[] offsets = new long[]{start, start + 1, start + (end - start) / 2, end - 1, end}; for (long offset : offsets) { checkCacheEntries(offset, (int) (end - offset + 1), end); checkCacheEntries(offset, 1, end); checkCacheEntries(offset, 20, end); checkCacheEntries(offset, segmentSize, end); checkCacheEntries(offset, segmentSize - 1, end); } }
Example #6
Source File: RaftLog.java From incubator-ratis with Apache License 2.0 | 6 votes |
public final void open(long lastIndexInSnapshot, Consumer<LogEntryProto> consumer) throws IOException { openImpl(lastIndexInSnapshot, e -> { if (e.hasMetadataEntry()) { lastMetadataEntry = e; } else if (consumer != null) { consumer.accept(e); } }); Optional.ofNullable(lastMetadataEntry).ifPresent( e -> commitIndex.updateToMax(e.getMetadataEntry().getCommitIndex(), infoIndexChange)); state.open(); final long startIndex = getStartIndex(); if (startIndex > LEAST_VALID_LOG_INDEX) { purgeIndex.updateIncreasingly(startIndex - 1, infoIndexChange); } }
Example #7
Source File: TestRaftLogSegment.java From ratis with Apache License 2.0 | 6 votes |
static void checkLogSegment(LogSegment segment, long start, long end, boolean isOpen, long totalSize, long term) throws Exception { Assert.assertEquals(start, segment.getStartIndex()); Assert.assertEquals(end, segment.getEndIndex()); Assert.assertEquals(isOpen, segment.isOpen()); Assert.assertEquals(totalSize, segment.getTotalSize()); long offset = SegmentedRaftLogFormat.getHeaderLength(); for (long i = start; i <= end; i++) { LogSegment.LogRecord record = segment.getLogRecord(i); LogRecordWithEntry lre = segment.getEntryWithoutLoading(i); Assert.assertEquals(i, lre.getRecord().getTermIndex().getIndex()); Assert.assertEquals(term, lre.getRecord().getTermIndex().getTerm()); Assert.assertEquals(offset, record.getOffset()); LogEntryProto entry = lre.hasEntry() ? lre.getEntry() : segment.loadCache(lre.getRecord()); offset += getEntrySize(entry); } }
Example #8
Source File: SegmentedRaftLogWorker.java From incubator-ratis with Apache License 2.0 | 6 votes |
WriteLog(LogEntryProto entry) { this.entry = ServerProtoUtils.removeStateMachineData(entry); if (this.entry == entry || stateMachine == null) { this.stateMachineFuture = null; } else { try { // this.entry != entry iff the entry has state machine data this.stateMachineFuture = stateMachine.data().write(entry); } catch (Throwable e) { LOG.error(name + ": writeStateMachineData failed for index " + entry.getIndex() + ", entry=" + ServerProtoUtils.toLogEntryString(entry, stateMachine::toStateMachineLogEntryString), e); throw e; } } this.combined = stateMachineFuture == null? super.getFuture() : super.getFuture().thenCombine(stateMachineFuture, (index, stateMachineResult) -> index); }
Example #9
Source File: RaftLog.java From incubator-ratis with Apache License 2.0 | 6 votes |
private long appendMetadataImpl(long term, long newCommitIndex) { checkLogState(); if (!shouldAppendMetadata(newCommitIndex)) { return INVALID_LOG_INDEX; } final LogEntryProto entry; final long nextIndex; try(AutoCloseableLock writeLock = writeLock()) { nextIndex = getNextIndex(); entry = ServerProtoUtils.toLogEntryProto(newCommitIndex, term, nextIndex); appendEntry(entry); } lastMetadataEntry = entry; return nextIndex; }
Example #10
Source File: TestSegmentedRaftLog.java From incubator-ratis with Apache License 2.0 | 6 votes |
/** * Keep appending entries, make sure the rolling is correct. */ @Test public void testAppendAndRoll() throws Exception { RaftServerConfigKeys.Log.setPreallocatedSize(properties, SizeInBytes.valueOf("16KB")); RaftServerConfigKeys.Log.setSegmentSizeMax(properties, SizeInBytes.valueOf("128KB")); List<SegmentRange> ranges = prepareRanges(0, 1, 1024, 0); final byte[] content = new byte[1024]; List<LogEntryProto> entries = prepareLogEntries(ranges, () -> new String(content)); try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); // append entries to the raftlog entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join); } try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); // check if the raft log is correct checkEntries(raftLog, entries, 0, entries.size()); Assert.assertEquals(9, raftLog.getRaftLogCache().getNumOfSegments()); } }
Example #11
Source File: TestSegmentedRaftLog.java From incubator-ratis with Apache License 2.0 | 6 votes |
@Test public void testTruncate() throws Exception { // prepare the log for truncation List<SegmentRange> ranges = prepareRanges(0, 5, 200, 0); List<LogEntryProto> entries = prepareLogEntries(ranges, null); try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); // append entries to the raftlog entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join); } for (long fromIndex = 900; fromIndex >= 0; fromIndex -= 150) { testTruncate(entries, fromIndex); } }
Example #12
Source File: RaftTestUtil.java From ratis with Apache License 2.0 | 6 votes |
static void checkLogEntries(RaftLog log, SimpleMessage[] expectedMessages, Predicate<LogEntryProto> predicate) { TermIndex[] termIndices = log.getEntries(0, Long.MAX_VALUE); for (int i = 0; i < termIndices.length; i++) { for (int j = 0; j < expectedMessages.length; j++) { final LogEntryProto e; try { e = log.get(termIndices[i].getIndex()); if (Arrays.equals(expectedMessages[j].getContent().toByteArray(), e.getStateMachineLogEntry().getLogData().toByteArray())) { Assert.assertTrue(predicate.test(e)); } } catch (IOException exception) { exception.printStackTrace(); } } } }
Example #13
Source File: FileStoreStateMachine.java From ratis with Apache License 2.0 | 6 votes |
@Override public CompletableFuture<ByteString> readStateMachineData(LogEntryProto entry) { final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry(); final ByteString data = smLog.getLogData(); final FileStoreRequestProto proto; try { proto = FileStoreRequestProto.parseFrom(data); } catch (InvalidProtocolBufferException e) { return FileStoreCommon.completeExceptionally( entry.getIndex(), "Failed to parse data, entry=" + entry, e); } if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) { return null; } final WriteRequestHeaderProto h = proto.getWriteHeader(); CompletableFuture<ExamplesProtos.ReadReplyProto> reply = files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength()); return reply.thenApply(ExamplesProtos.ReadReplyProto::getData); }
Example #14
Source File: SegmentedRaftLog.java From ratis with Apache License 2.0 | 6 votes |
@Override public List<CompletableFuture<Long>> appendImpl(LogEntryProto... entries) { checkLogState(); if (entries == null || entries.length == 0) { return Collections.emptyList(); } try(AutoCloseableLock writeLock = writeLock()) { final TruncateIndices ti = cache.computeTruncateIndices(this::failClientRequest, entries); final long truncateIndex = ti.getTruncateIndex(); final int index = ti.getArrayIndex(); LOG.debug("truncateIndex={}, arrayIndex={}", truncateIndex, index); final List<CompletableFuture<Long>> futures; if (truncateIndex != -1) { futures = new ArrayList<>(entries.length - index + 1); futures.add(truncate(truncateIndex)); } else { futures = new ArrayList<>(entries.length - index); } for (int i = index; i < entries.length; i++) { futures.add(appendEntry(entries[i])); } return futures; } }
Example #15
Source File: ArithmeticStateMachine.java From incubator-ratis with Apache License 2.0 | 6 votes |
@Override public CompletableFuture<Message> applyTransaction(TransactionContext trx) { final LogEntryProto entry = trx.getLogEntry(); final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData()); final long index = entry.getIndex(); final Double result; try(AutoCloseableLock writeLock = writeLock()) { result = assignment.evaluate(variables); updateLastAppliedTermIndex(entry.getTerm(), index); } final Expression r = Expression.Utils.double2Expression(result); final CompletableFuture<Message> f = CompletableFuture.completedFuture(Expression.Utils.toMessage(r)); final RaftPeerRole role = trx.getServerRole(); if (role == RaftPeerRole.LEADER) { LOG.info("{}:{}-{}: {} = {}", role, getId(), index, assignment, r); } else { LOG.debug("{}:{}-{}: {} = {}", role, getId(), index, assignment, r); } if (LOG.isTraceEnabled()) { LOG.trace("{}-{}: variables={}", getId(), index, variables); } return f; }
Example #16
Source File: TestRaftLogReadWrite.java From incubator-ratis with Apache License 2.0 | 6 votes |
/** * Test basic functionality: write several log entries, then read */ @Test public void testReadWriteLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLogFormat.getHeaderLength(); final LogEntryProto[] entries = new LogEntryProto[100]; try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize))) { size += writeMessages(entries, out); } finally { storage.close(); } Assert.assertEquals(size, openSegment.length()); LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); }
Example #17
Source File: TestRaftLogReadWrite.java From incubator-ratis with Apache License 2.0 | 6 votes |
/** * Simulate the scenario that the peer is shutdown without truncating * log segment file padding. Make sure the reader can correctly handle this. */ @Test public void testReadWithPadding() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLogFormat.getHeaderLength(); LogEntryProto[] entries = new LogEntryProto[100]; final SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize)); size += writeMessages(entries, out); out.flush(); // make sure the file contains padding Assert.assertEquals( RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length()); // check if the reader can correctly read the log file LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); out.close(); Assert.assertEquals(size, openSegment.length()); }
Example #18
Source File: ArithmeticStateMachine.java From ratis with Apache License 2.0 | 6 votes |
@Override public CompletableFuture<Message> applyTransaction(TransactionContext trx) { final LogEntryProto entry = trx.getLogEntry(); final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData()); final long index = entry.getIndex(); final Double result; try(final AutoCloseableLock writeLock = writeLock()) { result = assignment.evaluate(variables); updateLastAppliedTermIndex(entry.getTerm(), index); } final Expression r = Expression.Utils.double2Expression(result); final CompletableFuture<Message> f = CompletableFuture.completedFuture(Expression.Utils.toMessage(r)); final RaftPeerRole role = trx.getServerRole(); if (role == RaftPeerRole.LEADER) { LOG.info("{}:{}-{}: {} = {}", role, getId(), index, assignment, r); } else { LOG.debug("{}:{}-{}: {} = {}", role, getId(), index, assignment, r); } if (LOG.isTraceEnabled()) { LOG.trace("{}-{}: variables={}", getId(), index, variables); } return f; }
Example #19
Source File: FileStoreStateMachine.java From incubator-ratis with Apache License 2.0 | 6 votes |
@Override public CompletableFuture<ByteString> read(LogEntryProto entry) { final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry(); final ByteString data = smLog.getLogData(); final FileStoreRequestProto proto; try { proto = FileStoreRequestProto.parseFrom(data); } catch (InvalidProtocolBufferException e) { return FileStoreCommon.completeExceptionally( entry.getIndex(), "Failed to parse data, entry=" + entry, e); } if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) { return null; } final WriteRequestHeaderProto h = proto.getWriteHeader(); CompletableFuture<ExamplesProtos.ReadReplyProto> reply = files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength()); return reply.thenApply(ExamplesProtos.ReadReplyProto::getData); }
Example #20
Source File: TestRaftLogReadWrite.java From ratis with Apache License 2.0 | 6 votes |
/** * Simulate the scenario that the peer is shutdown without truncating * log segment file padding. Make sure the reader can correctly handle this. */ @Test public void testReadWithPadding() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLogFormat.getHeaderLength(); LogEntryProto[] entries = new LogEntryProto[100]; LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize); size += writeMessages(entries, out); out.flush(); // make sure the file contains padding Assert.assertEquals( RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length()); // check if the reader can correctly read the log file LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); out.close(); Assert.assertEquals(size, openSegment.length()); }
Example #21
Source File: LogOutputStream.java From ratis with Apache License 2.0 | 6 votes |
/** * Write the given entry to this output stream. * * Format: * (1) The serialized size of the entry. * (2) The entry. * (3) 4-byte checksum of the entry. * * Size in bytes to be written: * (size to encode n) + n + (checksum size), * where n is the entry serialized size and the checksum size is 4. */ public void write(LogEntryProto entry) throws IOException { final int serialized = entry.getSerializedSize(); final int bufferSize = CodedOutputStream.computeUInt32SizeNoTag(serialized) + serialized; preallocateIfNecessary(bufferSize + 4); byte[] buf = new byte[bufferSize]; CodedOutputStream cout = CodedOutputStream.newInstance(buf); cout.writeUInt32NoTag(serialized); entry.writeTo(cout); checksum.reset(); checksum.update(buf, 0, buf.length); final int sum = (int) checksum.getValue(); out.write(buf); writeInt(sum); }
Example #22
Source File: RaftLog.java From ratis with Apache License 2.0 | 5 votes |
/** * Validate the term and index of entry w.r.t RaftLog */ void validateLogEntry(LogEntryProto entry) { if (entry.hasMetadataEntry()) { return; } TermIndex lastTermIndex = getLastEntryTermIndex(); if (lastTermIndex != null) { Preconditions.assertTrue(entry.getTerm() >= lastTermIndex.getTerm(), "Entry term less than RaftLog's last term: %d, entry: %s", lastTermIndex.getTerm(), entry); Preconditions.assertTrue(entry.getIndex() == lastTermIndex.getIndex() + 1, "Difference between entry index and RaftLog's last index %d greater than 1, entry: %s", lastTermIndex.getIndex(), entry); } }
Example #23
Source File: RaftLog.java From incubator-ratis with Apache License 2.0 | 5 votes |
private long appendImpl(long term, RaftConfiguration newConf) { checkLogState(); try(AutoCloseableLock writeLock = writeLock()) { final long nextIndex = getNextIndex(); final LogEntryProto e = ServerProtoUtils.toLogEntryProto(newConf, term, nextIndex); appendEntry(e); return nextIndex; } }
Example #24
Source File: RaftLog.java From ratis with Apache License 2.0 | 5 votes |
private long appendImpl(long term, RaftConfiguration newConf) { checkLogState(); try(AutoCloseableLock writeLock = writeLock()) { final long nextIndex = getNextIndex(); final LogEntryProto e = ServerProtoUtils.toLogEntryProto(newConf, term, nextIndex); appendEntry(e); return nextIndex; } }
Example #25
Source File: SegmentedRaftLog.java From incubator-ratis with Apache License 2.0 | 5 votes |
/** * When the server is null, return the dummy instance of {@link ServerLogMethods}. * Otherwise, the server is non-null, return the implementation using the given server. */ private ServerLogMethods newServerLogMethods(RaftServerImpl impl) { if (impl == null) { return ServerLogMethods.DUMMY; } return new ServerLogMethods() { @Override public boolean shouldEvictCache() { return cache.shouldEvict(); } @Override public long[] getFollowerNextIndices() { return impl.getFollowerNextIndices(); } @Override public long getLastAppliedIndex() { return impl.getState().getLastAppliedIndex(); } @Override public void notifyTruncatedLogEntry(TermIndex ti) { try { final LogEntryProto entry = get(ti.getIndex()); impl.notifyTruncatedLogEntry(entry); } catch (RaftLogIOException e) { LOG.error("{}: Failed to read log {}", getName(), ti, e); } } }; }
Example #26
Source File: LogStateMachine.java From incubator-ratis with Apache License 2.0 | 5 votes |
@Override public CompletableFuture<Message> applyTransaction(TransactionContext trx) { try { checkInitialization(); final LogEntryProto entry = trx.getLogEntry(); LogServiceRequestProto logServiceRequestProto = LogServiceRequestProto.parseFrom(entry.getStateMachineLogEntry().getLogData()); switch (logServiceRequestProto.getRequestCase()) { case CHANGESTATE: return recordTime(getCloseLogTimer, new Task(){ @Override public CompletableFuture<Message> run() { return processChangeState(logServiceRequestProto); }}); case APPENDREQUEST: return recordTime(appendRequestTimer, new Task(){ @Override public CompletableFuture<Message> run() { return processAppendRequest(trx, logServiceRequestProto); }}); case SYNCREQUEST: return recordTime(syncRequesTimer, new Task(){ @Override public CompletableFuture<Message> run() { return processSyncRequest(trx, logServiceRequestProto); }}); case ARCHIVELOG: return updateArchiveLogInfo(logServiceRequestProto); default: //TODO return null; } } catch (IOException e) { // TODO exception handling throw new RuntimeException(e); } }
Example #27
Source File: FileStoreStateMachine.java From incubator-ratis with Apache License 2.0 | 5 votes |
@Override public CompletableFuture<Message> applyTransaction(TransactionContext trx) { final LogEntryProto entry = trx.getLogEntry(); final long index = entry.getIndex(); updateLastAppliedTermIndex(entry.getTerm(), index); final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry(); final FileStoreRequestProto request; try { request = FileStoreRequestProto.parseFrom(smLog.getLogData()); } catch (InvalidProtocolBufferException e) { return FileStoreCommon.completeExceptionally(index, "Failed to parse logData in" + smLog, e); } switch(request.getRequestCase()) { case DELETE: return delete(index, request.getDelete()); case WRITEHEADER: return writeCommit(index, request.getWriteHeader(), smLog.getStateMachineEntry().getStateMachineData().size()); case WRITE: // WRITE should not happen here since // startTransaction converts WRITE requests to WRITEHEADER requests. default: LOG.error(getId() + ": Unexpected request case " + request.getRequestCase()); return FileStoreCommon.completeExceptionally(index, "Unexpected request case " + request.getRequestCase()); } }
Example #28
Source File: SegmentedRaftLog.java From ratis with Apache License 2.0 | 5 votes |
private boolean isSegmentFull(LogSegment segment, LogEntryProto entry) { if (segment.getTotalSize() >= segmentMaxSize) { return true; } else { final long entrySize = LogSegment.getEntrySize(entry); // if entry size is greater than the max segment size, write it directly // into the current segment return entrySize <= segmentMaxSize && segment.getTotalSize() + entrySize > segmentMaxSize; } }
Example #29
Source File: LogSegment.java From incubator-ratis with Apache License 2.0 | 5 votes |
public static int readSegmentFile(File file, long start, long end, boolean isOpen, CorruptionPolicy corruptionPolicy, RaftLogMetrics raftLogMetrics, Consumer<LogEntryProto> entryConsumer) throws IOException { int count = 0; try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file, start, end, isOpen, raftLogMetrics)) { for(LogEntryProto prev = null, next; (next = in.nextEntry()) != null; prev = next) { if (prev != null) { Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1, "gap between entry %s and entry %s", prev, next); } if (entryConsumer != null) { entryConsumer.accept(next); } count++; } } catch (IOException ioe) { switch (corruptionPolicy) { case EXCEPTION: throw ioe; case WARN_AND_RETURN: LOG.warn("Failed to read segment file {} (start={}, end={}, isOpen? {}): only {} entries read successfully", file, start, end, isOpen, count, ioe); break; default: throw new IllegalStateException("Unexpected enum value: " + corruptionPolicy + ", class=" + CorruptionPolicy.class); } } return count; }
Example #30
Source File: LogSegment.java From incubator-ratis with Apache License 2.0 | 5 votes |
@Override public LogEntryProto load(LogRecord key) throws IOException { final File file = getSegmentFile(); // note the loading should not exceed the endIndex: it is possible that // the on-disk log file should be truncated but has not been done yet. readSegmentFile(file, startIndex, endIndex, isOpen, getLogCorruptionPolicy(), raftLogMetrics, entry -> entryCache.put(ServerProtoUtils.toTermIndex(entry), entry)); loadingTimes.incrementAndGet(); return Objects.requireNonNull(entryCache.get(key.getTermIndex())); }