org.apache.ratis.proto.RaftProtos.LogEntryProto Java Examples

The following examples show how to use org.apache.ratis.proto.RaftProtos.LogEntryProto. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SegmentedRaftLog.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Override
public LogEntryProto get(long index) throws RaftLogIOException {
  checkLogState();
  LogSegment segment;
  LogRecordWithEntry recordAndEntry;
  try (AutoCloseableLock readLock = readLock()) {
    segment = cache.getSegment(index);
    if (segment == null) {
      return null;
    }
    recordAndEntry = segment.getEntryWithoutLoading(index);
    if (recordAndEntry == null) {
      return null;
    }
    if (recordAndEntry.hasEntry()) {
      return recordAndEntry.getEntry();
    }
  }

  // the entry is not in the segment's cache. Load the cache without holding
  // RaftLog's lock.
  checkAndEvictCache();
  return segment.loadCache(recordAndEntry.getRecord());
}
 
Example #2
Source File: TestSegmentedRaftLog.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Test
public void testTruncate() throws Exception {
  // prepare the log for truncation
  List<SegmentRange> ranges = prepareRanges(0, 5, 200, 0);
  List<LogEntryProto> entries = prepareLogEntries(ranges, null);

  try (SegmentedRaftLog raftLog =
           new SegmentedRaftLog(peerId, null, storage, -1, properties)) {
    raftLog.open(RaftServerConstants.INVALID_LOG_INDEX, null);
    // append entries to the raftlog
    entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join);
  }

  for (long fromIndex = 900; fromIndex >= 0; fromIndex -= 150) {
    testTruncate(entries, fromIndex);
  }
}
 
Example #3
Source File: ServerState.java    From ratis with Apache License 2.0 6 votes vote down vote up
/**
 * note we do not apply log entries to the state machine here since we do not
 * know whether they have been committed.
 */
private RaftLog initLog(RaftPeerId id, RaftProperties prop,
    long lastIndexInSnapshot, Consumer<LogEntryProto> logConsumer)
    throws IOException {
  final RaftLog log;
  if (RaftServerConfigKeys.Log.useMemory(prop)) {
    final int maxBufferSize =
        RaftServerConfigKeys.Log.Appender.bufferByteLimit(prop).getSizeInt();
    log = new MemoryRaftLog(id, lastIndexInSnapshot, maxBufferSize);
  } else {
    log = new SegmentedRaftLog(id, server, this.storage,
        lastIndexInSnapshot, prop);
  }
  log.open(lastIndexInSnapshot, logConsumer);
  return log;
}
 
Example #4
Source File: RaftLog.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Validate the term and index of entry w.r.t RaftLog
 */
protected void validateLogEntry(LogEntryProto entry) {
  if (entry.hasMetadataEntry()) {
    return;
  }
  long latestSnapshotIndex = getSnapshotIndex();
  TermIndex lastTermIndex = getLastEntryTermIndex();
  if (lastTermIndex != null) {
    long lastIndex = lastTermIndex.getIndex() > latestSnapshotIndex ?
        lastTermIndex.getIndex() : latestSnapshotIndex;
    Preconditions.assertTrue(entry.getTerm() >= lastTermIndex.getTerm(),
        "Entry term less than RaftLog's last term: %d, entry: %s", lastTermIndex.getTerm(), entry);
    Preconditions.assertTrue(entry.getIndex() == lastIndex + 1,
        "Difference between entry index and RaftLog's last index %d (or snapshot index %d) " +
            "is greater than 1, entry: %s",
        lastTermIndex.getIndex(), latestSnapshotIndex, entry);
  } else {
    Preconditions.assertTrue(entry.getIndex() == latestSnapshotIndex + 1,
        "Difference between entry index and RaftLog's latest snapshot index %d is greater than 1 " +
            "and in between log entries are not present, entry: %s",
        latestSnapshotIndex, entry);
  }
}
 
Example #5
Source File: TestSegmentedRaftLogCache.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
private void checkCache(long start, long end, int segmentSize) {
  Assert.assertEquals(start, cache.getStartIndex());
  Assert.assertEquals(end, cache.getEndIndex());

  for (long index = start; index <= end; index++) {
    final LogSegment segment = cache.getSegment(index);
    final LogRecord record = segment.getLogRecord(index);
    final LogEntryProto entry = segment.getEntryFromCache(record.getTermIndex());
    Assert.assertEquals(index, entry.getIndex());
  }

  long[] offsets = new long[]{start, start + 1, start + (end - start) / 2,
      end - 1, end};
  for (long offset : offsets) {
    checkCacheEntries(offset, (int) (end - offset + 1), end);
    checkCacheEntries(offset, 1, end);
    checkCacheEntries(offset, 20, end);
    checkCacheEntries(offset, segmentSize, end);
    checkCacheEntries(offset, segmentSize - 1, end);
  }
}
 
Example #6
Source File: RaftLog.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
public final void open(long lastIndexInSnapshot, Consumer<LogEntryProto> consumer) throws IOException {
  openImpl(lastIndexInSnapshot, e -> {
    if (e.hasMetadataEntry()) {
      lastMetadataEntry = e;
    } else if (consumer != null) {
      consumer.accept(e);
    }
  });
  Optional.ofNullable(lastMetadataEntry).ifPresent(
      e -> commitIndex.updateToMax(e.getMetadataEntry().getCommitIndex(), infoIndexChange));
  state.open();

  final long startIndex = getStartIndex();
  if (startIndex > LEAST_VALID_LOG_INDEX) {
    purgeIndex.updateIncreasingly(startIndex - 1, infoIndexChange);
  }
}
 
Example #7
Source File: TestRaftLogSegment.java    From ratis with Apache License 2.0 6 votes vote down vote up
static void checkLogSegment(LogSegment segment, long start, long end,
    boolean isOpen, long totalSize, long term) throws Exception {
  Assert.assertEquals(start, segment.getStartIndex());
  Assert.assertEquals(end, segment.getEndIndex());
  Assert.assertEquals(isOpen, segment.isOpen());
  Assert.assertEquals(totalSize, segment.getTotalSize());

  long offset = SegmentedRaftLogFormat.getHeaderLength();
  for (long i = start; i <= end; i++) {
    LogSegment.LogRecord record = segment.getLogRecord(i);
    LogRecordWithEntry lre = segment.getEntryWithoutLoading(i);
    Assert.assertEquals(i, lre.getRecord().getTermIndex().getIndex());
    Assert.assertEquals(term, lre.getRecord().getTermIndex().getTerm());
    Assert.assertEquals(offset, record.getOffset());

    LogEntryProto entry = lre.hasEntry() ?
        lre.getEntry() : segment.loadCache(lre.getRecord());
    offset += getEntrySize(entry);
  }
}
 
Example #8
Source File: SegmentedRaftLogWorker.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
WriteLog(LogEntryProto entry) {
  this.entry = ServerProtoUtils.removeStateMachineData(entry);
  if (this.entry == entry || stateMachine == null) {
    this.stateMachineFuture = null;
  } else {
    try {
      // this.entry != entry iff the entry has state machine data
      this.stateMachineFuture = stateMachine.data().write(entry);
    } catch (Throwable e) {
      LOG.error(name + ": writeStateMachineData failed for index " + entry.getIndex()
          + ", entry=" + ServerProtoUtils.toLogEntryString(entry, stateMachine::toStateMachineLogEntryString), e);
      throw e;
    }
  }
  this.combined = stateMachineFuture == null? super.getFuture()
      : super.getFuture().thenCombine(stateMachineFuture, (index, stateMachineResult) -> index);
}
 
Example #9
Source File: RaftLog.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
private long appendMetadataImpl(long term, long newCommitIndex) {
  checkLogState();
  if (!shouldAppendMetadata(newCommitIndex)) {
    return INVALID_LOG_INDEX;
  }

  final LogEntryProto entry;
  final long nextIndex;
  try(AutoCloseableLock writeLock = writeLock()) {
    nextIndex = getNextIndex();
    entry = ServerProtoUtils.toLogEntryProto(newCommitIndex, term, nextIndex);
    appendEntry(entry);
  }
  lastMetadataEntry = entry;
  return nextIndex;
}
 
Example #10
Source File: TestSegmentedRaftLog.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Keep appending entries, make sure the rolling is correct.
 */
@Test
public void testAppendAndRoll() throws Exception {
  RaftServerConfigKeys.Log.setPreallocatedSize(properties, SizeInBytes.valueOf("16KB"));
  RaftServerConfigKeys.Log.setSegmentSizeMax(properties, SizeInBytes.valueOf("128KB"));

  List<SegmentRange> ranges = prepareRanges(0, 1, 1024, 0);
  final byte[] content = new byte[1024];
  List<LogEntryProto> entries = prepareLogEntries(ranges,
      () -> new String(content));

  try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) {
    raftLog.open(RaftLog.INVALID_LOG_INDEX, null);
    // append entries to the raftlog
    entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join);
  }

  try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) {
    raftLog.open(RaftLog.INVALID_LOG_INDEX, null);
    // check if the raft log is correct
    checkEntries(raftLog, entries, 0, entries.size());
    Assert.assertEquals(9, raftLog.getRaftLogCache().getNumOfSegments());
  }
}
 
Example #11
Source File: TestSegmentedRaftLog.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
@Test
public void testTruncate() throws Exception {
  // prepare the log for truncation
  List<SegmentRange> ranges = prepareRanges(0, 5, 200, 0);
  List<LogEntryProto> entries = prepareLogEntries(ranges, null);

  try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) {
    raftLog.open(RaftLog.INVALID_LOG_INDEX, null);
    // append entries to the raftlog
    entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join);
  }

  for (long fromIndex = 900; fromIndex >= 0; fromIndex -= 150) {
    testTruncate(entries, fromIndex);
  }
}
 
Example #12
Source File: RaftTestUtil.java    From ratis with Apache License 2.0 6 votes vote down vote up
static void checkLogEntries(RaftLog log, SimpleMessage[] expectedMessages,
    Predicate<LogEntryProto> predicate) {
  TermIndex[] termIndices = log.getEntries(0, Long.MAX_VALUE);
  for (int i = 0; i < termIndices.length; i++) {
    for (int j = 0; j < expectedMessages.length; j++) {
      final LogEntryProto e;
      try {
        e = log.get(termIndices[i].getIndex());
        if (Arrays.equals(expectedMessages[j].getContent().toByteArray(),
            e.getStateMachineLogEntry().getLogData().toByteArray())) {
          Assert.assertTrue(predicate.test(e));
        }
      } catch (IOException exception) {
        exception.printStackTrace();
      }
    }
  }
}
 
Example #13
Source File: FileStoreStateMachine.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<ByteString> readStateMachineData(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  CompletableFuture<ExamplesProtos.ReadReplyProto> reply =
      files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength());

  return reply.thenApply(ExamplesProtos.ReadReplyProto::getData);
}
 
Example #14
Source File: SegmentedRaftLog.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Override
public List<CompletableFuture<Long>> appendImpl(LogEntryProto... entries) {
  checkLogState();
  if (entries == null || entries.length == 0) {
    return Collections.emptyList();
  }
  try(AutoCloseableLock writeLock = writeLock()) {
    final TruncateIndices ti = cache.computeTruncateIndices(this::failClientRequest, entries);
    final long truncateIndex = ti.getTruncateIndex();
    final int index = ti.getArrayIndex();
    LOG.debug("truncateIndex={}, arrayIndex={}", truncateIndex, index);

    final List<CompletableFuture<Long>> futures;
    if (truncateIndex != -1) {
      futures = new ArrayList<>(entries.length - index + 1);
      futures.add(truncate(truncateIndex));
    } else {
      futures = new ArrayList<>(entries.length - index);
    }
    for (int i = index; i < entries.length; i++) {
      futures.add(appendEntry(entries[i]));
    }
    return futures;
  }
}
 
Example #15
Source File: ArithmeticStateMachine.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();
  final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData());

  final long index = entry.getIndex();
  final Double result;
  try(AutoCloseableLock writeLock = writeLock()) {
    result = assignment.evaluate(variables);
    updateLastAppliedTermIndex(entry.getTerm(), index);
  }
  final Expression r = Expression.Utils.double2Expression(result);
  final CompletableFuture<Message> f = CompletableFuture.completedFuture(Expression.Utils.toMessage(r));

  final RaftPeerRole role = trx.getServerRole();
  if (role == RaftPeerRole.LEADER) {
    LOG.info("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  } else {
    LOG.debug("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}-{}: variables={}", getId(), index, variables);
  }
  return f;
}
 
Example #16
Source File: TestRaftLogReadWrite.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Test basic functionality: write several log entries, then read
 */
@Test
public void testReadWriteLog() throws IOException {
  final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  File openSegment = storage.getStorageDir().getOpenLogFile(0);
  long size = SegmentedRaftLogFormat.getHeaderLength();

  final LogEntryProto[] entries = new LogEntryProto[100];
  try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false,
      segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize))) {
    size += writeMessages(entries, out);
  } finally {
    storage.close();
  }

  Assert.assertEquals(size, openSegment.length());

  LogEntryProto[] readEntries = readLog(openSegment, 0,
      RaftServerConstants.INVALID_LOG_INDEX, true);
  Assert.assertArrayEquals(entries, readEntries);
}
 
Example #17
Source File: TestRaftLogReadWrite.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Simulate the scenario that the peer is shutdown without truncating
 * log segment file padding. Make sure the reader can correctly handle this.
 */
@Test
public void testReadWithPadding() throws IOException {
  final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  File openSegment = storage.getStorageDir().getOpenLogFile(0);
  long size = SegmentedRaftLogFormat.getHeaderLength();

  LogEntryProto[] entries = new LogEntryProto[100];
  final SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false,
      segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize));
  size += writeMessages(entries, out);
  out.flush();

  // make sure the file contains padding
  Assert.assertEquals(
      RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(),
      openSegment.length());

  // check if the reader can correctly read the log file
  LogEntryProto[] readEntries = readLog(openSegment, 0,
      RaftServerConstants.INVALID_LOG_INDEX, true);
  Assert.assertArrayEquals(entries, readEntries);

  out.close();
  Assert.assertEquals(size, openSegment.length());
}
 
Example #18
Source File: ArithmeticStateMachine.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();
  final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData());

  final long index = entry.getIndex();
  final Double result;
  try(final AutoCloseableLock writeLock = writeLock()) {
    result = assignment.evaluate(variables);
    updateLastAppliedTermIndex(entry.getTerm(), index);
  }
  final Expression r = Expression.Utils.double2Expression(result);
  final CompletableFuture<Message> f = CompletableFuture.completedFuture(Expression.Utils.toMessage(r));

  final RaftPeerRole role = trx.getServerRole();
  if (role == RaftPeerRole.LEADER) {
    LOG.info("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  } else {
    LOG.debug("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}-{}: variables={}", getId(), index, variables);
  }
  return f;
}
 
Example #19
Source File: FileStoreStateMachine.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<ByteString> read(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  CompletableFuture<ExamplesProtos.ReadReplyProto> reply =
      files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength());

  return reply.thenApply(ExamplesProtos.ReadReplyProto::getData);
}
 
Example #20
Source File: TestRaftLogReadWrite.java    From ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Simulate the scenario that the peer is shutdown without truncating
 * log segment file padding. Make sure the reader can correctly handle this.
 */
@Test
public void testReadWithPadding() throws IOException {
  final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  File openSegment = storage.getStorageDir().getOpenLogFile(0);
  long size = SegmentedRaftLogFormat.getHeaderLength();

  LogEntryProto[] entries = new LogEntryProto[100];
  LogOutputStream out = new LogOutputStream(openSegment, false,
      segmentMaxSize, preallocatedSize, bufferSize);
  size += writeMessages(entries, out);
  out.flush();

  // make sure the file contains padding
  Assert.assertEquals(
      RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(),
      openSegment.length());

  // check if the reader can correctly read the log file
  LogEntryProto[] readEntries = readLog(openSegment, 0,
      RaftServerConstants.INVALID_LOG_INDEX, true);
  Assert.assertArrayEquals(entries, readEntries);

  out.close();
  Assert.assertEquals(size, openSegment.length());
}
 
Example #21
Source File: LogOutputStream.java    From ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Write the given entry to this output stream.
 *
 * Format:
 *   (1) The serialized size of the entry.
 *   (2) The entry.
 *   (3) 4-byte checksum of the entry.
 *
 * Size in bytes to be written:
 *   (size to encode n) + n + (checksum size),
 *   where n is the entry serialized size and the checksum size is 4.
 */
public void write(LogEntryProto entry) throws IOException {
  final int serialized = entry.getSerializedSize();
  final int bufferSize = CodedOutputStream.computeUInt32SizeNoTag(serialized)
      + serialized;

  preallocateIfNecessary(bufferSize + 4);

  byte[] buf = new byte[bufferSize];
  CodedOutputStream cout = CodedOutputStream.newInstance(buf);
  cout.writeUInt32NoTag(serialized);
  entry.writeTo(cout);

  checksum.reset();
  checksum.update(buf, 0, buf.length);
  final int sum = (int) checksum.getValue();

  out.write(buf);
  writeInt(sum);
}
 
Example #22
Source File: RaftLog.java    From ratis with Apache License 2.0 5 votes vote down vote up
/**
 * Validate the term and index of entry w.r.t RaftLog
 */
void validateLogEntry(LogEntryProto entry) {
  if (entry.hasMetadataEntry()) {
    return;
  }
  TermIndex lastTermIndex = getLastEntryTermIndex();
  if (lastTermIndex != null) {
    Preconditions.assertTrue(entry.getTerm() >= lastTermIndex.getTerm(),
        "Entry term less than RaftLog's last term: %d, entry: %s", lastTermIndex.getTerm(), entry);
    Preconditions.assertTrue(entry.getIndex() == lastTermIndex.getIndex() + 1,
        "Difference between entry index and RaftLog's last index %d greater than 1, entry: %s", lastTermIndex.getIndex(), entry);
  }
}
 
Example #23
Source File: RaftLog.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private long appendImpl(long term, RaftConfiguration newConf) {
  checkLogState();
  try(AutoCloseableLock writeLock = writeLock()) {
    final long nextIndex = getNextIndex();
    final LogEntryProto e = ServerProtoUtils.toLogEntryProto(newConf, term,
        nextIndex);
    appendEntry(e);
    return nextIndex;
  }
}
 
Example #24
Source File: RaftLog.java    From ratis with Apache License 2.0 5 votes vote down vote up
private long appendImpl(long term, RaftConfiguration newConf) {
  checkLogState();
  try(AutoCloseableLock writeLock = writeLock()) {
    final long nextIndex = getNextIndex();
    final LogEntryProto e = ServerProtoUtils.toLogEntryProto(newConf, term,
        nextIndex);
    appendEntry(e);
    return nextIndex;
  }
}
 
Example #25
Source File: SegmentedRaftLog.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
/**
 * When the server is null, return the dummy instance of {@link ServerLogMethods}.
 * Otherwise, the server is non-null, return the implementation using the given server.
 */
private ServerLogMethods newServerLogMethods(RaftServerImpl impl) {
  if (impl == null) {
    return ServerLogMethods.DUMMY;
  }

  return new ServerLogMethods() {
    @Override
    public boolean shouldEvictCache() {
      return cache.shouldEvict();
    }

    @Override
    public long[] getFollowerNextIndices() {
      return impl.getFollowerNextIndices();
    }

    @Override
    public long getLastAppliedIndex() {
      return impl.getState().getLastAppliedIndex();
    }

    @Override
    public void notifyTruncatedLogEntry(TermIndex ti) {
      try {
        final LogEntryProto entry = get(ti.getIndex());
        impl.notifyTruncatedLogEntry(entry);
      } catch (RaftLogIOException e) {
        LOG.error("{}: Failed to read log {}", getName(), ti, e);
      }
    }
  };
}
 
Example #26
Source File: LogStateMachine.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  try {
    checkInitialization();
    final LogEntryProto entry = trx.getLogEntry();
    LogServiceRequestProto logServiceRequestProto =
        LogServiceRequestProto.parseFrom(entry.getStateMachineLogEntry().getLogData());
    switch (logServiceRequestProto.getRequestCase()) {
    case CHANGESTATE:
        return recordTime(getCloseLogTimer, new Task(){
          @Override public CompletableFuture<Message> run() {
            return processChangeState(logServiceRequestProto);
          }});
      case APPENDREQUEST:
        return recordTime(appendRequestTimer, new Task(){
            @Override public CompletableFuture<Message> run() {
              return processAppendRequest(trx, logServiceRequestProto);
            }});
      case SYNCREQUEST:
        return recordTime(syncRequesTimer, new Task(){
          @Override public CompletableFuture<Message> run() {
            return processSyncRequest(trx, logServiceRequestProto);
          }});
      case ARCHIVELOG:
        return updateArchiveLogInfo(logServiceRequestProto);
      default:
        //TODO
        return null;
    }
  } catch (IOException e) {
    // TODO exception handling
    throw new RuntimeException(e);
  }
}
 
Example #27
Source File: FileStoreStateMachine.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();

  final long index = entry.getIndex();
  updateLastAppliedTermIndex(entry.getTerm(), index);

  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final FileStoreRequestProto request;
  try {
    request = FileStoreRequestProto.parseFrom(smLog.getLogData());
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(index,
        "Failed to parse logData in" + smLog, e);
  }

  switch(request.getRequestCase()) {
    case DELETE:
      return delete(index, request.getDelete());
    case WRITEHEADER:
      return writeCommit(index, request.getWriteHeader(), smLog.getStateMachineEntry().getStateMachineData().size());
    case WRITE:
      // WRITE should not happen here since
      // startTransaction converts WRITE requests to WRITEHEADER requests.
    default:
      LOG.error(getId() + ": Unexpected request case " + request.getRequestCase());
      return FileStoreCommon.completeExceptionally(index,
          "Unexpected request case " + request.getRequestCase());
  }
}
 
Example #28
Source File: SegmentedRaftLog.java    From ratis with Apache License 2.0 5 votes vote down vote up
private boolean isSegmentFull(LogSegment segment, LogEntryProto entry) {
  if (segment.getTotalSize() >= segmentMaxSize) {
    return true;
  } else {
    final long entrySize = LogSegment.getEntrySize(entry);
    // if entry size is greater than the max segment size, write it directly
    // into the current segment
    return entrySize <= segmentMaxSize &&
        segment.getTotalSize() + entrySize > segmentMaxSize;
  }
}
 
Example #29
Source File: LogSegment.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
public static int readSegmentFile(File file, long start, long end,
    boolean isOpen, CorruptionPolicy corruptionPolicy,
    RaftLogMetrics raftLogMetrics, Consumer<LogEntryProto> entryConsumer) throws
    IOException {
  int count = 0;
  try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file, start, end, isOpen, raftLogMetrics)) {
    for(LogEntryProto prev = null, next; (next = in.nextEntry()) != null; prev = next) {
      if (prev != null) {
        Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1,
            "gap between entry %s and entry %s", prev, next);
      }

      if (entryConsumer != null) {
        entryConsumer.accept(next);
      }
      count++;
    }
  } catch (IOException ioe) {
    switch (corruptionPolicy) {
      case EXCEPTION: throw ioe;
      case WARN_AND_RETURN:
        LOG.warn("Failed to read segment file {} (start={}, end={}, isOpen? {}): only {} entries read successfully",
            file, start, end, isOpen, count, ioe);
        break;
      default:
        throw new IllegalStateException("Unexpected enum value: " + corruptionPolicy
            + ", class=" + CorruptionPolicy.class);
    }
  }

  return count;
}
 
Example #30
Source File: LogSegment.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public LogEntryProto load(LogRecord key) throws IOException {
  final File file = getSegmentFile();
  // note the loading should not exceed the endIndex: it is possible that
  // the on-disk log file should be truncated but has not been done yet.
  readSegmentFile(file, startIndex, endIndex, isOpen, getLogCorruptionPolicy(), raftLogMetrics,
      entry -> entryCache.put(ServerProtoUtils.toTermIndex(entry), entry));
  loadingTimes.incrementAndGet();
  return Objects.requireNonNull(entryCache.get(key.getTermIndex()));
}