org.apache.ratis.thirdparty.com.google.protobuf.ByteString Java Examples

The following examples show how to use org.apache.ratis.thirdparty.com.google.protobuf.ByteString. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FileStoreStateMachine.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Integer> write(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  final CompletableFuture<Integer> f = files.write(entry.getIndex(),
      h.getPath().toStringUtf8(), h.getClose(), h.getOffset(), smLog.getStateMachineEntry().getStateMachineData());
  // sync only if closing the file
  return h.getClose()? f: null;
}
 
Example #2
Source File: FileStoreStateMachine.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<ByteString> read(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  CompletableFuture<ExamplesProtos.ReadReplyProto> reply =
      files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength());

  return reply.thenApply(ExamplesProtos.ReadReplyProto::getData);
}
 
Example #3
Source File: ContainerCommandResponseBuilders.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
/**
 * Gets a response to the read small file call.
 * @param msg - Msg
 * @param data  - Data
 * @param info  - Info
 * @return    Response.
 */
public static ContainerCommandResponseProto getGetSmallFileResponseSuccess(
    ContainerCommandRequestProto msg, ByteString data, ChunkInfo info) {

  Preconditions.checkNotNull(msg);

  ReadChunkResponseProto.Builder readChunk =
      ReadChunkResponseProto.newBuilder()
          .setChunkData(info)
          .setData((data))
          .setBlockID(msg.getGetSmallFile().getBlock().getBlockID());

  GetSmallFileResponseProto.Builder getSmallFile =
      GetSmallFileResponseProto.newBuilder().setData(readChunk);

  return getSuccessResponseBuilder(msg)
      .setCmdType(Type.GetSmallFile)
      .setGetSmallFile(getSmallFile)
      .build();
}
 
Example #4
Source File: FileStoreStateMachine.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
@Override
public TransactionContext startTransaction(RaftClientRequest request) throws IOException {
  final ByteString content = request.getMessage().getContent();
  final FileStoreRequestProto proto = FileStoreRequestProto.parseFrom(content);
  final TransactionContext.Builder b = TransactionContext.newBuilder()
      .setStateMachine(this)
      .setClientRequest(request);

  if (proto.getRequestCase() == FileStoreRequestProto.RequestCase.WRITE) {
    final WriteRequestProto write = proto.getWrite();
    final FileStoreRequestProto newProto = FileStoreRequestProto.newBuilder()
        .setWriteHeader(write.getHeader()).build();
    b.setLogData(newProto.toByteString()).setStateMachineData(write.getData());
  } else {
    b.setLogData(content);
  }
  return b.build();
}
 
Example #5
Source File: Client.java    From ratis with Apache License 2.0 6 votes vote down vote up
@Override
public void run() throws Exception {
  RaftProperties raftProperties = new RaftProperties();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);


}
 
Example #6
Source File: FileStoreClient.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
private static <OUTPUT, THROWABLE extends Throwable> OUTPUT writeImpl(
    CheckedFunction<ByteString, OUTPUT, THROWABLE> sendFunction,
    String path, long offset, boolean close, ByteBuffer data)
    throws THROWABLE {
  final WriteRequestHeaderProto.Builder header = WriteRequestHeaderProto.newBuilder()
      .setPath(ProtoUtils.toByteString(path))
      .setOffset(offset)
      .setLength(data.position())
      .setClose(close);

  final WriteRequestProto.Builder write = WriteRequestProto.newBuilder()
      .setHeader(header)
      .setData(ByteString.copyFrom(data));

  final FileStoreRequestProto request = FileStoreRequestProto.newBuilder().setWrite(write).build();
  return sendFunction.apply(request.toByteString());
}
 
Example #7
Source File: ContainerCommandRequestMessage.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
public static ContainerCommandRequestProto toProto(
    ByteString bytes, RaftGroupId groupId)
    throws InvalidProtocolBufferException {
  final int i = Integer.BYTES + bytes.substring(0, Integer.BYTES)
      .asReadOnlyByteBuffer().getInt();
  final ContainerCommandRequestProto header
      = ContainerCommandRequestProto
      .parseFrom(bytes.substring(Integer.BYTES, i));
  // TODO: setting pipeline id can be avoided if the client is sending it.
  //       In such case, just have to validate the pipeline id.
  final ContainerCommandRequestProto.Builder b = header.toBuilder();
  if (groupId != null) {
    b.setPipelineID(groupId.getUuid().toString());
  }
  final ByteString data = bytes.substring(i);
  if (header.getCmdType() == Type.WriteChunk) {
    b.setWriteChunk(b.getWriteChunkBuilder().setData(data));
  } else if (header.getCmdType() == Type.PutSmallFile) {
    b.setPutSmallFile(b.getPutSmallFileBuilder().setData(data));
  }
  return b.build();
}
 
Example #8
Source File: FileStoreClient.java    From ratis with Apache License 2.0 6 votes vote down vote up
private static <OUTPUT, THROWABLE extends Throwable> OUTPUT writeImpl(
    CheckedFunction<ByteString, OUTPUT, THROWABLE> sendFunction,
    String path, long offset, boolean close, ByteBuffer data)
    throws THROWABLE {
  final WriteRequestHeaderProto.Builder header = WriteRequestHeaderProto.newBuilder()
      .setPath(ProtoUtils.toByteString(path))
      .setOffset(offset)
      .setLength(data.position())
      .setClose(close);

  final WriteRequestProto.Builder write = WriteRequestProto.newBuilder()
      .setHeader(header)
      .setData(ByteString.copyFrom(data));

  final FileStoreRequestProto request = FileStoreRequestProto.newBuilder().setWrite(write).build();
  return sendFunction.apply(request.toByteString());
}
 
Example #9
Source File: ContainerProtocolCalls.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
/**
 * Calls the container protocol to write a chunk.
 *
 * @param xceiverClient client to perform call
 * @param chunk information about chunk to write
 * @param blockID ID of the block
 * @param data the data of the chunk to write
 * @throws IOException if there is an I/O error while performing the call
 */
public static XceiverClientReply writeChunkAsync(
    XceiverClientSpi xceiverClient, ChunkInfo chunk, BlockID blockID,
    ByteString data)
    throws IOException, ExecutionException, InterruptedException {
  WriteChunkRequestProto.Builder writeChunkRequest =
      WriteChunkRequestProto.newBuilder()
          .setBlockID(blockID.getDatanodeBlockIDProtobuf())
          .setChunkData(chunk).setData(data);
  String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
  ContainerCommandRequestProto.Builder builder =
      ContainerCommandRequestProto.newBuilder().setCmdType(Type.WriteChunk)
          .setContainerID(blockID.getContainerID())
          .setDatanodeUuid(id).setWriteChunk(writeChunkRequest);
  String encodedToken = getEncodedBlockToken(new Text(blockID.
      getContainerBlockID().toString()));
  if (encodedToken != null) {
    builder.setEncodedToken(encodedToken);
  }
  ContainerCommandRequestProto request = builder.build();
  return xceiverClient.sendCommandAsync(request);
}
 
Example #10
Source File: LogServiceProtoUtil.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
public static LogServiceRequestProto toAppendBBEntryLogRequestProto(LogName name,
    List<ByteBuffer> entries) {
  LogNameProto logNameProto =
      LogNameProto.newBuilder().setName(name.getName()).build();
  AppendLogEntryRequestProto.Builder builder = AppendLogEntryRequestProto.newBuilder();
  builder.setLogName(logNameProto);
  for (int i=0; i < entries.size(); i++) {
    ByteBuffer currentBuf = entries.get(i);
    // Save the current position
    int pos = currentBuf.position();
    builder.addData(ByteString.copyFrom(currentBuf));
    // Reset it after we're done reading the bytes
    currentBuf.position(pos);
  }
  return LogServiceRequestProto.newBuilder().setAppendRequest(builder.build()).build();
}
 
Example #11
Source File: LogServiceRaftLogReader.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the next log entry. Ensure {@link #hasNext()} returns true before
 * calling this method.
 */
@Override
public byte[] next() throws RaftLogIOException, InvalidProtocolBufferException {
  if (currentRecord == null) {
    throw new NoSuchElementException();
  }
  ByteString current = currentRecord;
  currentRecord = null;
  loadNext();
  return current.toByteArray();
}
 
Example #12
Source File: HadoopRpcService.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private <REQUEST, REPLY> REPLY processRequest(
    REQUEST request, ByteString replyId,
    CheckedFunction<RaftServerProtocolPB, REPLY, ServiceException> f)
    throws IOException {
  CodeInjectionForTesting.execute(SEND_SERVER_REQUEST, getId(), null, request);

  final RaftServerProtocolPB proxy = getProxies().getProxy(
      RaftPeerId.valueOf(replyId)).getProtocol();
  try {
    return f.apply(proxy);
  } catch (ServiceException se) {
    throw ProtoUtils.toIOException(se);
  }
}
 
Example #13
Source File: FileStoreBaseTest.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
Writer verify() throws IOException {
  final Random r = new Random(seed);
  final int size = fileSize.getSizeInt();

  for(int offset = 0; offset < size; ) {
    final int remaining = size - offset;
    final int n = Math.min(remaining, buffer.length);
    final ByteString read = client.read(fileName, offset, n);
    final ByteBuffer expected = randomBytes(n, r);
    verify(read, offset, n, expected);
    offset += n;
  }
  return this;
}
 
Example #14
Source File: ServerProtoUtils.java    From ratis with Apache License 2.0 5 votes vote down vote up
static StateMachineLogEntryProto toStateMachineLogEntryProto(
    ClientId clientId, long callId, ByteString logData, ByteString stateMachineData) {
  final StateMachineLogEntryProto.Builder b = StateMachineLogEntryProto.newBuilder()
      .setClientId(clientId.toByteString())
      .setCallId(callId)
      .setLogData(logData);
  if (stateMachineData != null) {
    b.setStateMachineEntry(toStateMachineEntryProtoBuilder(stateMachineData));
  }
  return b.build();
}
 
Example #15
Source File: ServerProtoUtils.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
static StateMachineLogEntryProto toStateMachineLogEntryProto(
    RaftClientRequest request, ByteString logData, ByteString stateMachineData) {
  if (logData == null) {
    logData = request.getMessage().getContent();
  }
  return toStateMachineLogEntryProto(request.getClientId(), request.getCallId(), logData, stateMachineData);
}
 
Example #16
Source File: Client.java    From ratis with Apache License 2.0 5 votes vote down vote up
@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setMaxCachedSegmentNum(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setSchedulerThreads(raftProperties, 10);
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 
Example #17
Source File: FileInfo.java    From ratis with Apache License 2.0 5 votes vote down vote up
CompletableFuture<Integer> submitCreate(
    CheckedFunction<Path, Path, IOException> resolver, ByteString data, boolean close,
    ExecutorService executor, RaftPeerId id, long index) {
  final Supplier<String> name = () -> "create(" + getRelativePath() + ", "
      + close + ") @" + id + ":" + index;
  final CheckedSupplier<Integer, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    if (out == null) {
      out = new FileOut(resolver.apply(getRelativePath()));
    }
    return write(0L, data, close);
  }, name);
  return submitWrite(task, executor, id, index);
}
 
Example #18
Source File: RaftId.java    From ratis with Apache License 2.0 5 votes vote down vote up
private static ByteString toByteString(UUID uuid) {
  Objects.requireNonNull(uuid, "uuid == null");
  final ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
  buf.putLong(uuid.getMostSignificantBits());
  buf.putLong(uuid.getLeastSignificantBits());
  return ByteString.copyFrom(buf.array());
}
 
Example #19
Source File: TestRaftId.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Test
public void testRaftPeerId() {
  final RaftPeerId id = RaftPeerId.valueOf("abc");
  final ByteString bytes = id.toByteString();
  Assert.assertEquals(bytes, id.toByteString());
  Assert.assertEquals(id, RaftPeerId.valueOf(bytes));
}
 
Example #20
Source File: LogServiceProtoUtil.java    From ratis with Apache License 2.0 5 votes vote down vote up
public static ReadLogReplyProto toReadLogReplyProto(List<byte[]> entries, Throwable t) {
  ReadLogReplyProto.Builder builder = ReadLogReplyProto.newBuilder();
  if (t != null) {
    builder.setException(toLogException(t));
  } else {
    for(byte[] record: entries) {
      builder.addLogRecord( ByteString.copyFrom(record));
    }
  }
  return builder.build();
}
 
Example #21
Source File: AssignmentMessage.java    From ratis with Apache License 2.0 5 votes vote down vote up
@Override
public ByteString getContent() {
  final int length = variable.length() + expression.length();
  final byte[] bytes = new byte[length];
  final int offset = variable.toBytes(bytes, 0);
  expression.toBytes(bytes, offset);
  return toByteString(bytes);
}
 
Example #22
Source File: Checksum.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
private static Function<ByteBuffer, ByteString> newChecksumByteBufferFunction(
    Supplier<ChecksumByteBuffer> constructor) {
  final ChecksumByteBuffer algorithm = constructor.get();
  return  data -> {
    algorithm.reset();
    algorithm.update(data);
    return int2ByteString((int)algorithm.getValue());
  };
}
 
Example #23
Source File: ServerProtoUtils.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
static StateMachineLogEntryProto toStateMachineLogEntryProto(
    ClientId clientId, long callId, ByteString logData, ByteString stateMachineData) {
  final StateMachineLogEntryProto.Builder b = StateMachineLogEntryProto.newBuilder()
      .setClientId(clientId.toByteString())
      .setCallId(callId)
      .setLogData(logData);
  if (stateMachineData != null) {
    b.setStateMachineEntry(toStateMachineEntryProtoBuilder(stateMachineData));
  }
  return b.build();
}
 
Example #24
Source File: Checksum.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
/**
 * Compute checksum using the algorithm for the data upto the max length.
 * @param data input data
 * @param function the checksum function
 * @param maxLength the max length of data
 * @return computed checksum ByteString
 */
private static ByteString computeChecksum(ByteBuffer data,
    Function<ByteBuffer, ByteString> function, int maxLength) {
  final int limit = data.limit();
  try {
    final int maxIndex = data.position() + maxLength;
    if (limit > maxIndex) {
      data.limit(maxIndex);
    }
    return function.apply(data);
  } finally {
    data.limit(limit);
  }
}
 
Example #25
Source File: RaftId.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private static ByteString toByteString(UUID uuid) {
  Objects.requireNonNull(uuid, "uuid == null");
  final ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
  buf.putLong(uuid.getMostSignificantBits());
  buf.putLong(uuid.getLeastSignificantBits());
  return ByteString.copyFrom(buf.array());
}
 
Example #26
Source File: FileInfo.java    From ratis with Apache License 2.0 5 votes vote down vote up
private int write(long offset, ByteString data, boolean close) throws IOException {
  if (offset != writeSize) {
    throw new IOException("Offset/size mismatched: offset = " + offset
        + " != writeSize = " + writeSize + ", path=" + getRelativePath());
  }
  if (out == null) {
    throw new IOException("File output is not initialized, path=" + getRelativePath());
  }

  synchronized (out) {
    int n = 0;
    if (data != null) {
      final ByteBuffer buffer = data.asReadOnlyByteBuffer();
      try {
        for (; buffer.remaining() > 0; ) {
          n += out.write(buffer);
        }
      } finally {
        writeSize += n;
      }
    }

    if (close) {
      out.close();
    }
    return n;
  }
}
 
Example #27
Source File: FileInfo.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private int write(long offset, ByteString data, boolean close) throws IOException {
  // If leader finish write data with offset = 4096 and writeSize become 8192,
  // and 2 follower has not written the data with offset = 4096,
  // then start a leader election. So client will retry send the data with offset = 4096,
  // then offset < writeSize in leader.
  if (offset < writeSize) {
    return data.size();
  }
  if (offset != writeSize) {
    throw new IOException("Offset/size mismatched: offset = " + offset
        + " != writeSize = " + writeSize + ", path=" + getRelativePath());
  }
  if (out == null) {
    throw new IOException("File output is not initialized, path=" + getRelativePath());
  }

  synchronized (out) {
    int n = 0;
    if (data != null) {
      final ByteBuffer buffer = data.asReadOnlyByteBuffer();
      try {
        for (; buffer.remaining() > 0; ) {
          n += out.write(buffer);
        }
      } finally {
        writeSize += n;
      }
    }

    if (close) {
      out.close();
    }
    return n;
  }
}
 
Example #28
Source File: FileStore.java    From ratis with Apache License 2.0 5 votes vote down vote up
CompletableFuture<ReadReplyProto> read(String relative, long offset, long length) {
  final Supplier<String> name = () -> "read(" + relative
      + ", " + offset + ", " + length + ") @" + getId();
  final CheckedSupplier<ReadReplyProto, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    final FileInfo info = files.get(relative);
    final ReadReplyProto.Builder reply = ReadReplyProto.newBuilder()
        .setResolvedPath(FileStoreCommon.toByteString(info.getRelativePath()))
        .setOffset(offset);

    final ByteString bytes = info.read(this::resolve, offset, length);
    return reply.setData(bytes).build();
  }, name);
  return submit(task, reader);
}
 
Example #29
Source File: TestContainerCommandRequestMessage.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
static void runTest(int length,
    BiFunction<BlockID, ByteString, ContainerCommandRequestProto> method)
    throws Exception {
  System.out.println("length=" + length);
  final BlockID blockID = new BlockID(RANDOM.nextLong(), RANDOM.nextLong());
  final ByteString data = newData(length);

  final ContainerCommandRequestProto original = method.apply(blockID, data);
  final ContainerCommandRequestMessage message
      = ContainerCommandRequestMessage.toMessage(original, null);
  final ContainerCommandRequestProto computed
      = ContainerCommandRequestMessage.toProto(message.getContent(), null);
  Assert.assertEquals(original, computed);
}
 
Example #30
Source File: LogServiceProtoUtil.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
public static List<byte[]> toListByteArray(List<ByteString> list) {
  List<byte[]> retVal = new ArrayList<byte[]>(list.size());
  for(int i=0; i < list.size(); i++) {
    retVal.add(list.get(i).toByteArray());
  }
  return retVal;
}