org.apache.ratis.protocol.Message Java Examples

The following examples show how to use org.apache.ratis.protocol.Message. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ArithmeticStateMachine.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();
  final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData());

  final long index = entry.getIndex();
  final Double result;
  try(final AutoCloseableLock writeLock = writeLock()) {
    result = assignment.evaluate(variables);
    updateLastAppliedTermIndex(entry.getTerm(), index);
  }
  final Expression r = Expression.Utils.double2Expression(result);
  final CompletableFuture<Message> f = CompletableFuture.completedFuture(Expression.Utils.toMessage(r));

  final RaftPeerRole role = trx.getServerRole();
  if (role == RaftPeerRole.LEADER) {
    LOG.info("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  } else {
    LOG.debug("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}-{}: variables={}", getId(), index, variables);
  }
  return f;
}
 
Example #2
Source File: SimpleStateMachine4Testing.java    From ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Query the n-th log entry.
 * @param request an index represented in a UTF-8 String, or an empty message.
 * @return a completed future of the n-th log entry,
 *         where n is the last applied index if the request is empty,
 *         otherwise, n is the index represented in the request.
 */
@Override
public CompletableFuture<Message> query(Message request) {
  final String string = request.getContent().toStringUtf8();
  Exception exception;
  try {
    LOG.info("query " + string);
    final LogEntryProto entry = dataMap.get(string);
    if (entry != null) {
      return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString()));
    }
    exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string);
  } catch (Exception e) {
    LOG.warn("Failed request " + request, e);
    exception = e;
  }
  return JavaUtils.completeExceptionally(new StateMachineException(
      "Failed request " + request, exception));
}
 
Example #3
Source File: LogStateMachine.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Message> updateArchiveLogInfo(
    LogServiceRequestProto logServiceRequestProto) {
  LogServiceProtos.ArchiveLogRequestProto archiveLog = logServiceRequestProto.getArchiveLog();
  this.isArchivalRequest = !archiveLog.getIsExport();
  Throwable t = null;
  if(isArchivalRequest) {
    archivalInfo.updateArchivalInfo(archiveLog);
    t = verifyState(State.ARCHIVING);
  }else{
    t = verifyState(State.OPEN, State.CLOSED);
    ArchivalInfo info = exportMap.get(archiveLog.getLocation());
    if(info==null) {
      info = new ArchivalInfo(archiveLog.getLocation());
      exportMap.put(archiveLog.getLocation(),info);
    }
    info.updateArchivalInfo(archiveLog);
  }
  return CompletableFuture.completedFuture(
      Message.valueOf(LogServiceProtoUtil.toArchiveLogReplyProto(t).toByteString()));
}
 
Example #4
Source File: TestStateMachine.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  try {
    assertNotNull(trx.getLogEntry());
    assertNotNull(trx.getStateMachineLogEntry());
    Object context = trx.getStateMachineContext();
    if (isLeader.get()) {
      assertNotNull(trx.getClientRequest());
      assertNotNull(context);
      assertTrue(context instanceof Long);
      Long val = (Long)context;
      assertTrue(val <= transactions.get());
      applied.add(val);
    } else {
      assertNull(trx.getClientRequest());
      assertNull(context);
    }
    numApplied.incrementAndGet();
  } catch (Throwable t) {
    throwable.set(t);
  }
  return CompletableFuture.completedFuture(null);
}
 
Example #5
Source File: TestStateMachine.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  try {
    assertNotNull(trx.getLogEntry());
    assertNotNull(trx.getStateMachineLogEntry());
    Object context = trx.getStateMachineContext();
    if (isLeader.get()) {
      assertNotNull(trx.getClientRequest());
      assertNotNull(context);
      assertTrue(context instanceof Long);
      Long val = (Long)context;
      assertTrue(val <= transactions.get());
      applied.add(val);
    } else {
      assertNull(trx.getClientRequest());
      assertNull(context);
    }
    numApplied.incrementAndGet();
  } catch (Throwable t) {
    throwable.set(t);
  }
  return CompletableFuture.completedFuture(null);
}
 
Example #6
Source File: SimpleStateMachine4Testing.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
/**
 * Query the n-th log entry.
 * @param request an index represented in a UTF-8 String, or an empty message.
 * @return a completed future of the n-th log entry,
 *         where n is the last applied index if the request is empty,
 *         otherwise, n is the index represented in the request.
 */
@Override
public CompletableFuture<Message> query(Message request) {
  final String string = request.getContent().toStringUtf8();
  Exception exception;
  try {
    LOG.info("query " + string);
    final LogEntryProto entry = dataMap.get(string);
    if (entry != null) {
      return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString()));
    }
    exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string);
  } catch (Exception e) {
    LOG.warn("Failed request " + request, e);
    exception = e;
  }
  return JavaUtils.completeExceptionally(new StateMachineException(
      "Failed request " + request, exception));
}
 
Example #7
Source File: OrderedAsync.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
CompletableFuture<RaftClientReply> send(RaftClientRequest.Type type, Message message, RaftPeerId server) {
  if (!type.is(TypeCase.WATCH) && !type.is(TypeCase.STREAM)) {
    Objects.requireNonNull(message, "message == null");
  }
  try {
    requestSemaphore.acquire();
  } catch (InterruptedException e) {
    return JavaUtils.completeExceptionally(IOUtils.toInterruptedIOException(
        "Interrupted when sending " + type + ", message=" + message, e));
  }

  final long callId = RaftClientImpl.nextCallId();
  final LongFunction<PendingOrderedRequest> constructor = seqNum -> new PendingOrderedRequest(callId, seqNum,
      slidingWindowEntry -> client.newRaftClientRequest(server, callId, message, type, slidingWindowEntry));
  return getSlidingWindow(server).submitNewRequest(constructor, this::sendRequestWithRetry
  ).getReplyFuture(
  ).thenApply(reply -> RaftClientImpl.handleRaftException(reply, CompletionException::new)
  ).whenComplete((r, e) -> requestSemaphore.release());
}
 
Example #8
Source File: RaftOutputStream.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
/** Non-blocking flush call */
private void flushAsync() {
  final long pos = byteFlushed;
  if (byteCount == 0) {
    return;
  }

  final CompletableFuture<Long> f = getClient().sendAsync(
      Message.valueOf(ProtoUtils.toByteString(buffer, 0, byteCount))
  ).thenApply(reply -> RaftClientImpl.handleRaftException(reply, CompletionException::new)
  ).thenApply(reply -> reply != null && reply.isSuccess()? pos: null);
  flushFutures.offer(f);

  byteFlushed += byteCount;
  byteCount = 0;
}
 
Example #9
Source File: ArithmeticStateMachine.java    From ratis with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Message> query(Message request) {
  final Expression q = Expression.Utils.bytes2Expression(request.getContent().toByteArray(), 0);
  final Double result;
  try(final AutoCloseableLock readLock = readLock()) {
    result = q.evaluate(variables);
  }
  final Expression r = Expression.Utils.double2Expression(result);
  LOG.debug("QUERY: {} = {}", q, r);
  return CompletableFuture.completedFuture(Expression.Utils.toMessage(r));
}
 
Example #10
Source File: StreamImpl.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<RaftClientReply> streamAsync(Message message, SizeInBytes subSize) {
  final int n = subSize.getSizeInt();
  final MessageOutputStream out = stream();
  final ByteString bytes = message.getContent();
  for(int i = 0; i < bytes.size(); ) {
    final int j = Math.min(i + n, bytes.size());
    final ByteString sub = bytes.substring(i, j);
    out.sendAsync(Message.valueOf(sub));
    i = j;
  }
  return out.closeAsync();
}
 
Example #11
Source File: FileStoreClient.java    From ratis with Apache License 2.0 5 votes vote down vote up
static CompletableFuture<ByteString> sendAsync(
    ByteString request, Function<Message, CompletableFuture<RaftClientReply>> sendFunction)
    {
  return sendFunction.apply(() -> request
  ).thenApply(reply -> {
    final StateMachineException sme = reply.getStateMachineException();
    if (sme != null) {
      throw new CompletionException("Failed to send request " + request, sme);
    }
    Preconditions.assertTrue(reply.isSuccess(), () -> "Failed " + request + ", reply=" + reply);
    return reply.getMessage().getContent();
  });
}
 
Example #12
Source File: FileStoreClient.java    From ratis with Apache License 2.0 5 votes vote down vote up
static ByteString send(
    ByteString request, CheckedFunction<Message, RaftClientReply, IOException> sendFunction)
    throws IOException {
  final RaftClientReply reply = sendFunction.apply(Message.valueOf(request));
  final StateMachineException sme = reply.getStateMachineException();
  if (sme != null) {
    throw new IOException("Failed to send request " + request, sme);
  }
  Preconditions.assertTrue(reply.isSuccess(), () -> "Failed " + request + ", reply=" + reply);
  return reply.getMessage().getContent();
}
 
Example #13
Source File: FileStoreStateMachine.java    From ratis with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Message> delete(long index, DeleteRequestProto request) {
  final String path = request.getPath().toStringUtf8();
  return files.delete(index, path).thenApply(resolved ->
      Message.valueOf(DeleteReplyProto.newBuilder().setResolvedPath(
          FileStoreCommon.toByteString(resolved)).build().toByteString(),
          () -> "Message:" + resolved));
}
 
Example #14
Source File: StateMachineUpdater.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private void checkAndTakeSnapshot(MemoizedSupplier<List<CompletableFuture<Message>>> futures)
    throws ExecutionException, InterruptedException {
  // check if need to trigger a snapshot
  if (shouldTakeSnapshot()) {
    if (futures.isInitialized()) {
      JavaUtils.allOf(futures.get()).get();
    }

    takeSnapshot();
  }
}
 
Example #15
Source File: BaseStateMachine.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
protected CompletableFuture<Message> recordTime(Timer timer, Task task) {
  final Timer.Context timerContext = timer.time();
  try {
    return task.run();
  } finally {
    timerContext.stop();
  }
}
 
Example #16
Source File: ContainerStateMachine.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Message> writeStateMachineData(LogEntryProto entry) {
  try {
    metrics.incNumWriteStateMachineOps();
    long writeStateMachineStartTime = Time.monotonicNowNanos();
    ContainerCommandRequestProto requestProto =
        getContainerCommandRequestProto(gid,
            entry.getStateMachineLogEntry().getLogData());
    WriteChunkRequestProto writeChunk =
        WriteChunkRequestProto.newBuilder(requestProto.getWriteChunk())
            .setData(getStateMachineData(entry.getStateMachineLogEntry()))
            .build();
    requestProto = ContainerCommandRequestProto.newBuilder(requestProto)
        .setWriteChunk(writeChunk).build();
    Type cmdType = requestProto.getCmdType();

    // For only writeChunk, there will be writeStateMachineData call.
    // CreateContainer will happen as a part of writeChunk only.
    switch (cmdType) {
    case WriteChunk:
      return handleWriteChunk(requestProto, entry.getIndex(),
          entry.getTerm(), writeStateMachineStartTime);
    default:
      throw new IllegalStateException("Cmd Type:" + cmdType
          + " should not have state machine data");
    }
  } catch (IOException e) {
    metrics.incNumWriteStateMachineFails();
    return completeExceptionally(e);
  }
}
 
Example #17
Source File: BaseStateMachine.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Message> queryStale(Message request, long minIndex) {
  if (getLastAppliedTermIndex().getIndex() < minIndex) {
    synchronized (transactionFutures) {
      if (getLastAppliedTermIndex().getIndex() < minIndex) {
        return transactionFutures.computeIfAbsent(minIndex, key -> new CompletableFuture<>())
            .thenCompose(v -> query(request));
      }
    }
  }
  return query(request);
}
 
Example #18
Source File: PendingRequests.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
Permit tryAcquire(Message message) {
  final ResourceSemaphore.ResourceAcquireStatus acquired = resource.tryAcquire(message);
  LOG.trace("tryAcquire? {}", acquired);
  if (acquired == ResourceSemaphore.ResourceAcquireStatus.FAILED_IN_ELEMENT_LIMIT) {
    raftServerMetrics.onRequestQueueLimitHit();
    raftServerMetrics.onResourceLimitHit();
    return null;
  } else if (acquired == ResourceSemaphore.ResourceAcquireStatus.FAILED_IN_BYTE_SIZE_LIMIT) {
    raftServerMetrics.onRequestByteSizeLimitHit();
    raftServerMetrics.onResourceLimitHit();
    return null;
  }
  return putPermit();
}
 
Example #19
Source File: StreamRequests.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
synchronized CompletableFuture<ByteString> append(long messageId, Message message) {
  if (nextId == -1) {
    nextId = messageId;
  } else if (messageId != nextId) {
    return JavaUtils.completeExceptionally(new StreamException(
        "Unexpected message id in " + key + ": messageId = " + messageId + " != nextId = " + nextId));
  }
  nextId++;
  bytes = bytes.concat(message.getContent());
  return CompletableFuture.completedFuture(bytes);
}
 
Example #20
Source File: LogStateMachine.java    From ratis with Apache License 2.0 5 votes vote down vote up
/**
 * Process read log entries request
 * @param msg message
 * @return reply message
 */
private CompletableFuture<Message> processReadRequest(LogServiceRequestProto proto) {

  ReadLogRequestProto msgProto = proto.getReadNextQuery();
  long startRecordId = msgProto.getStartRecordId();
  int num = msgProto.getNumRecords();
  Throwable t = verifyState(State.OPEN);
  List<byte[]> list = new ArrayList<byte[]>();
  LOG.info("Start Index: {}", startRecordId);
  LOG.info("Total to read: {}", num);
  if (t == null) {
    for (long index = startRecordId; index < startRecordId + num; index++) {
      try {
        LogEntryProto entry = log.get(index);
        LOG.info("Index: {} Entry: {}", index, entry);
        if (entry == null || entry.hasConfigurationEntry()) {
          continue;
        }
        //TODO: how to distinguish log records from
        // DML commands logged by the service?
        list.add(entry.getStateMachineLogEntry().getLogData().toByteArray());
      } catch (RaftLogIOException e) {
        t = e;
        list = null;
        break;
      }
    }
  }
  return CompletableFuture.completedFuture(
    Message.valueOf(LogServiceProtoUtil.toReadLogReplyProto(list, t).toByteString()));
}
 
Example #21
Source File: MetaStateMachine.java    From ratis with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Message> processListLogsRequest() {
    return CompletableFuture.completedFuture(Message.valueOf(MetaServiceProtoUtil
            .toListLogLogsReplyProto(
                    map.entrySet()
                            .stream()
                            .map(log -> new LogInfo(log.getKey(), log.getValue()))
                            .collect(Collectors.toList())).toByteString()));
}
 
Example #22
Source File: BaseStateMachine.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  // return the same message contained in the entry
  RaftProtos.LogEntryProto entry = Objects.requireNonNull(trx.getLogEntry());
  updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex());
  return CompletableFuture.completedFuture(
      Message.valueOf(trx.getLogEntry().getStateMachineLogEntry().getLogData()));
}
 
Example #23
Source File: MetaStateMachine.java    From ratis with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Message>
processArchiveLog(MetaServiceProtos.MetaServiceRequestProto logServiceRequestProto) {
    ArchiveLogRequestProto archiveLog = logServiceRequestProto.getArchiveLog();
    LogName logName = LogServiceProtoUtil.toLogName(archiveLog.getLogName());
    // Handle log archiving.
    return CompletableFuture.completedFuture(Message
            .valueOf(ArchiveLogReplyProto.newBuilder().build().toByteString()));
}
 
Example #24
Source File: StateMachineUpdater.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private MemoizedSupplier<List<CompletableFuture<Message>>> applyLog() throws RaftLogIOException {
  final MemoizedSupplier<List<CompletableFuture<Message>>> futures = MemoizedSupplier.valueOf(ArrayList::new);
  final long committed = raftLog.getLastCommittedIndex();
  for(long applied; (applied = getLastAppliedIndex()) < committed && state == State.RUNNING && !shouldStop(); ) {
    final long nextIndex = applied + 1;
    final LogEntryProto next = raftLog.get(nextIndex);
    if (next != null) {
      if (LOG.isTraceEnabled()) {
        LOG.trace("{}: applying nextIndex={}, nextLog={}", this, nextIndex, ServerProtoUtils.toString(next));
      } else {
        LOG.debug("{}: applying nextIndex={}", this, nextIndex);
      }

      final CompletableFuture<Message> f = server.applyLogToStateMachine(next);
      if (f != null) {
        futures.get().add(f);
      }
      final long incremented = appliedIndex.incrementAndGet(debugIndexChange);
      Preconditions.assertTrue(incremented == nextIndex);
    } else {
      LOG.debug("{}: logEntry {} is null. There may be snapshot to load. state:{}",
          this, nextIndex, state);
      break;
    }
  }
  return futures;
}
 
Example #25
Source File: LogStreamImpl.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public long getLength() throws IOException {
  RaftClientReply reply = raftClient
      .sendReadOnly(Message.valueOf(LogServiceProtoUtil
          .toGetLengthRequestProto(name).toByteString()));
  GetLogLengthReplyProto proto =
      GetLogLengthReplyProto.parseFrom(reply.getMessage().getContent());
  if (proto.hasException()) {
    LogServiceException e = proto.getException();
    throw new IOException(e.getErrorMsg());
  }
  return proto.getLength();
}
 
Example #26
Source File: LogStreamImpl.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public long getSize() throws IOException{
  RaftClientReply reply = raftClient
      .sendReadOnly(Message.valueOf(LogServiceProtoUtil
          .toGetSizeRequestProto(name).toByteString()));
  GetLogSizeReplyProto proto =
      GetLogSizeReplyProto.parseFrom(reply.getMessage().getContent());
  if (proto.hasException()) {
    LogServiceException e = proto.getException();
    throw new IOException(e.getErrorMsg());
  }
  return proto.getSize();
}
 
Example #27
Source File: LogStateMachine.java    From ratis with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Message> processGetStateRequest(
    LogServiceRequestProto logServiceRequestProto) {
  GetStateRequestProto getState = logServiceRequestProto.getGetState();
  LogName logName = LogServiceProtoUtil.toLogName(getState.getLogName());
  return CompletableFuture.completedFuture(Message.valueOf(LogServiceProtoUtil
      .toGetStateReplyProto(state == State.OPEN).toByteString()));
}
 
Example #28
Source File: LogStateMachine.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Message> processChangeState(LogServiceRequestProto logServiceRequestProto) {
  LogServiceProtos.ChangeStateLogRequestProto changeState = logServiceRequestProto.getChangeState();
  // Need to check whether the file is opened if opened close it.
  // TODO need to handle exceptions while operating with files.

  State targetState = State.valueOf(changeState.getState().name());
  Throwable t = null;
  //if forced skip checking states
  if(!changeState.getForce()) {
    switch (targetState) {
    case OPEN:
      if (state != null) {
        t = verifyState(State.OPEN, State.CLOSED);
      }
      break;
    case CLOSED:
      t =  verifyState(State.OPEN);
      break;
    case ARCHIVED:
      t = verifyState(State.ARCHIVING);
      break;
    case ARCHIVING:
      t = verifyState(State.CLOSED);
      break;
    case DELETED:
      t = verifyState(State.CLOSED);
      break;
    default:
    }
  }
  if(t != null) {
    return CompletableFuture.completedFuture(Message
            .valueOf(LogServiceProtos.ChangeStateReplyProto.newBuilder().
                    setException(LogServiceProtoUtil.toLogException(t)).build().toByteString()));
  }
  this.state = targetState;
  return CompletableFuture.completedFuture(Message
      .valueOf(LogServiceProtos.ChangeStateReplyProto.newBuilder().build().toByteString()));
}
 
Example #29
Source File: LogStateMachine.java    From ratis with Apache License 2.0 5 votes vote down vote up
/**
 * Process get last committed record index
 * @param msg message
 * @return reply message
 */
private CompletableFuture<Message>
    processGetLastCommittedIndexRequest(LogServiceRequestProto proto)
{

  Throwable t = verifyState(State.OPEN);
  long lastIndex = log.getLastCommittedIndex();
  return CompletableFuture.completedFuture(Message
    .valueOf(LogServiceProtoUtil.toGetLogLastIndexReplyProto(lastIndex, t).toByteString()));
}
 
Example #30
Source File: LogStateMachine.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Message> processAppendRequest(TransactionContext trx,
    LogServiceRequestProto logProto) {

  final LogEntryProto entry = trx.getLogEntry();
  AppendLogEntryRequestProto proto = logProto.getAppendRequest();
  final long index = entry.getIndex();
  long newSize = 0;
  Throwable t = verifyState(State.OPEN);
  final List<Long> ids = new ArrayList<Long>();
  if (t == null) {
    try (AutoCloseableLock writeLock = writeLock()) {
        List<byte[]> entries = LogServiceProtoUtil.toListByteArray(proto.getDataList());
        for (byte[] bb : entries) {
          ids.add(this.length);
          newSize += bb.length;
          this.length++;
        }
        this.dataRecordsSize += newSize;
        // TODO do we need this for other write request (close, sync)
        updateLastAppliedTermIndex(entry.getTerm(), index);
    }
  }
  final CompletableFuture<Message> f =
      CompletableFuture.completedFuture(
        Message.valueOf(LogServiceProtoUtil.toAppendLogReplyProto(ids, t).toByteString()));
  final RaftProtos.RaftPeerRole role = trx.getServerRole();
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}:{}-{}: {} new length {}", role, getId(), index,
        TextFormat.shortDebugString(proto), dataRecordsSize);
  }
  return f;
}