Java Code Examples for org.apache.ratis.thirdparty.com.google.protobuf.ByteString

The following examples show how to use org.apache.ratis.thirdparty.com.google.protobuf.ByteString. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: ratis   Source File: Client.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run() throws Exception {
  RaftProperties raftProperties = new RaftProperties();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);


}
 
Example 2
Source Project: incubator-ratis   Source File: LogServiceProtoUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static LogServiceRequestProto toAppendBBEntryLogRequestProto(LogName name,
    List<ByteBuffer> entries) {
  LogNameProto logNameProto =
      LogNameProto.newBuilder().setName(name.getName()).build();
  AppendLogEntryRequestProto.Builder builder = AppendLogEntryRequestProto.newBuilder();
  builder.setLogName(logNameProto);
  for (int i=0; i < entries.size(); i++) {
    ByteBuffer currentBuf = entries.get(i);
    // Save the current position
    int pos = currentBuf.position();
    builder.addData(ByteString.copyFrom(currentBuf));
    // Reset it after we're done reading the bytes
    currentBuf.position(pos);
  }
  return LogServiceRequestProto.newBuilder().setAppendRequest(builder.build()).build();
}
 
Example 3
Source Project: ratis   Source File: FileStoreClient.java    License: Apache License 2.0 6 votes vote down vote up
private static <OUTPUT, THROWABLE extends Throwable> OUTPUT writeImpl(
    CheckedFunction<ByteString, OUTPUT, THROWABLE> sendFunction,
    String path, long offset, boolean close, ByteBuffer data)
    throws THROWABLE {
  final WriteRequestHeaderProto.Builder header = WriteRequestHeaderProto.newBuilder()
      .setPath(ProtoUtils.toByteString(path))
      .setOffset(offset)
      .setLength(data.position())
      .setClose(close);

  final WriteRequestProto.Builder write = WriteRequestProto.newBuilder()
      .setHeader(header)
      .setData(ByteString.copyFrom(data));

  final FileStoreRequestProto request = FileStoreRequestProto.newBuilder().setWrite(write).build();
  return sendFunction.apply(request.toByteString());
}
 
Example 4
/**
 * Gets a response to the read small file call.
 * @param msg - Msg
 * @param data  - Data
 * @param info  - Info
 * @return    Response.
 */
public static ContainerCommandResponseProto getGetSmallFileResponseSuccess(
    ContainerCommandRequestProto msg, ByteString data, ChunkInfo info) {

  Preconditions.checkNotNull(msg);

  ReadChunkResponseProto.Builder readChunk =
      ReadChunkResponseProto.newBuilder()
          .setChunkData(info)
          .setData((data))
          .setBlockID(msg.getGetSmallFile().getBlock().getBlockID());

  GetSmallFileResponseProto.Builder getSmallFile =
      GetSmallFileResponseProto.newBuilder().setData(readChunk);

  return getSuccessResponseBuilder(msg)
      .setCmdType(Type.GetSmallFile)
      .setGetSmallFile(getSmallFile)
      .build();
}
 
Example 5
Source Project: hadoop-ozone   Source File: ContainerProtocolCalls.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Calls the container protocol to write a chunk.
 *
 * @param xceiverClient client to perform call
 * @param chunk information about chunk to write
 * @param blockID ID of the block
 * @param data the data of the chunk to write
 * @throws IOException if there is an I/O error while performing the call
 */
public static XceiverClientReply writeChunkAsync(
    XceiverClientSpi xceiverClient, ChunkInfo chunk, BlockID blockID,
    ByteString data)
    throws IOException, ExecutionException, InterruptedException {
  WriteChunkRequestProto.Builder writeChunkRequest =
      WriteChunkRequestProto.newBuilder()
          .setBlockID(blockID.getDatanodeBlockIDProtobuf())
          .setChunkData(chunk).setData(data);
  String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
  ContainerCommandRequestProto.Builder builder =
      ContainerCommandRequestProto.newBuilder().setCmdType(Type.WriteChunk)
          .setContainerID(blockID.getContainerID())
          .setDatanodeUuid(id).setWriteChunk(writeChunkRequest);
  String encodedToken = getEncodedBlockToken(new Text(blockID.
      getContainerBlockID().toString()));
  if (encodedToken != null) {
    builder.setEncodedToken(encodedToken);
  }
  ContainerCommandRequestProto request = builder.build();
  return xceiverClient.sendCommandAsync(request);
}
 
Example 6
public static ContainerCommandRequestProto toProto(
    ByteString bytes, RaftGroupId groupId)
    throws InvalidProtocolBufferException {
  final int i = Integer.BYTES + bytes.substring(0, Integer.BYTES)
      .asReadOnlyByteBuffer().getInt();
  final ContainerCommandRequestProto header
      = ContainerCommandRequestProto
      .parseFrom(bytes.substring(Integer.BYTES, i));
  // TODO: setting pipeline id can be avoided if the client is sending it.
  //       In such case, just have to validate the pipeline id.
  final ContainerCommandRequestProto.Builder b = header.toBuilder();
  if (groupId != null) {
    b.setPipelineID(groupId.getUuid().toString());
  }
  final ByteString data = bytes.substring(i);
  if (header.getCmdType() == Type.WriteChunk) {
    b.setWriteChunk(b.getWriteChunkBuilder().setData(data));
  } else if (header.getCmdType() == Type.PutSmallFile) {
    b.setPutSmallFile(b.getPutSmallFileBuilder().setData(data));
  }
  return b.build();
}
 
Example 7
Source Project: incubator-ratis   Source File: FileStoreClient.java    License: Apache License 2.0 6 votes vote down vote up
private static <OUTPUT, THROWABLE extends Throwable> OUTPUT writeImpl(
    CheckedFunction<ByteString, OUTPUT, THROWABLE> sendFunction,
    String path, long offset, boolean close, ByteBuffer data)
    throws THROWABLE {
  final WriteRequestHeaderProto.Builder header = WriteRequestHeaderProto.newBuilder()
      .setPath(ProtoUtils.toByteString(path))
      .setOffset(offset)
      .setLength(data.position())
      .setClose(close);

  final WriteRequestProto.Builder write = WriteRequestProto.newBuilder()
      .setHeader(header)
      .setData(ByteString.copyFrom(data));

  final FileStoreRequestProto request = FileStoreRequestProto.newBuilder().setWrite(write).build();
  return sendFunction.apply(request.toByteString());
}
 
Example 8
Source Project: incubator-ratis   Source File: FileStoreStateMachine.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TransactionContext startTransaction(RaftClientRequest request) throws IOException {
  final ByteString content = request.getMessage().getContent();
  final FileStoreRequestProto proto = FileStoreRequestProto.parseFrom(content);
  final TransactionContext.Builder b = TransactionContext.newBuilder()
      .setStateMachine(this)
      .setClientRequest(request);

  if (proto.getRequestCase() == FileStoreRequestProto.RequestCase.WRITE) {
    final WriteRequestProto write = proto.getWrite();
    final FileStoreRequestProto newProto = FileStoreRequestProto.newBuilder()
        .setWriteHeader(write.getHeader()).build();
    b.setLogData(newProto.toByteString()).setStateMachineData(write.getData());
  } else {
    b.setLogData(content);
  }
  return b.build();
}
 
Example 9
Source Project: incubator-ratis   Source File: FileStoreStateMachine.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Integer> write(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  final CompletableFuture<Integer> f = files.write(entry.getIndex(),
      h.getPath().toStringUtf8(), h.getClose(), h.getOffset(), smLog.getStateMachineEntry().getStateMachineData());
  // sync only if closing the file
  return h.getClose()? f: null;
}
 
Example 10
Source Project: incubator-ratis   Source File: FileStoreStateMachine.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<ByteString> read(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  CompletableFuture<ExamplesProtos.ReadReplyProto> reply =
      files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength());

  return reply.thenApply(ExamplesProtos.ReadReplyProto::getData);
}
 
Example 11
Source Project: ratis   Source File: RaftId.java    License: Apache License 2.0 5 votes vote down vote up
private static ByteString toByteString(UUID uuid) {
  Objects.requireNonNull(uuid, "uuid == null");
  final ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
  buf.putLong(uuid.getMostSignificantBits());
  buf.putLong(uuid.getLeastSignificantBits());
  return ByteString.copyFrom(buf.array());
}
 
Example 12
Source Project: ratis   Source File: FileInfo.java    License: Apache License 2.0 5 votes vote down vote up
CompletableFuture<Integer> submitCreate(
    CheckedFunction<Path, Path, IOException> resolver, ByteString data, boolean close,
    ExecutorService executor, RaftPeerId id, long index) {
  final Supplier<String> name = () -> "create(" + getRelativePath() + ", "
      + close + ") @" + id + ":" + index;
  final CheckedSupplier<Integer, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    if (out == null) {
      out = new FileOut(resolver.apply(getRelativePath()));
    }
    return write(0L, data, close);
  }, name);
  return submitWrite(task, executor, id, index);
}
 
Example 13
Source Project: ratis   Source File: Client.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setMaxCachedSegmentNum(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setSchedulerThreads(raftProperties, 10);
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 
Example 14
Source Project: ratis   Source File: LogServiceProtoUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static LogServiceRequestProto toAppendEntryLogRequestProto(LogName name,
    List<byte[]> entries) {
  LogNameProto logNameProto =
      LogNameProto.newBuilder().setName(name.getName()).build();
  AppendLogEntryRequestProto.Builder builder = AppendLogEntryRequestProto.newBuilder();
  builder.setLogName(logNameProto);
  for (int i=0; i < entries.size(); i++) {
    builder.addData(ByteString.copyFrom(entries.get(i)));
  }
  return LogServiceRequestProto.newBuilder().setAppendRequest(builder.build()).build();
}
 
Example 15
Source Project: incubator-ratis   Source File: StreamRequests.java    License: Apache License 2.0 5 votes vote down vote up
synchronized CompletableFuture<ByteString> append(long messageId, Message message) {
  if (nextId == -1) {
    nextId = messageId;
  } else if (messageId != nextId) {
    return JavaUtils.completeExceptionally(new StreamException(
        "Unexpected message id in " + key + ": messageId = " + messageId + " != nextId = " + nextId));
  }
  nextId++;
  bytes = bytes.concat(message.getContent());
  return CompletableFuture.completedFuture(bytes);
}
 
Example 16
Source Project: ratis   Source File: ServerProtoUtils.java    License: Apache License 2.0 5 votes vote down vote up
static StateMachineLogEntryProto toStateMachineLogEntryProto(
    ClientId clientId, long callId, ByteString logData, ByteString stateMachineData) {
  final StateMachineLogEntryProto.Builder b = StateMachineLogEntryProto.newBuilder()
      .setClientId(clientId.toByteString())
      .setCallId(callId)
      .setLogData(logData);
  if (stateMachineData != null) {
    b.setStateMachineEntry(toStateMachineEntryProtoBuilder(stateMachineData));
  }
  return b.build();
}
 
Example 17
Source Project: incubator-ratis   Source File: HadoopRpcService.java    License: Apache License 2.0 5 votes vote down vote up
private <REQUEST, REPLY> REPLY processRequest(
    REQUEST request, ByteString replyId,
    CheckedFunction<RaftServerProtocolPB, REPLY, ServiceException> f)
    throws IOException {
  CodeInjectionForTesting.execute(SEND_SERVER_REQUEST, getId(), null, request);

  final RaftServerProtocolPB proxy = getProxies().getProxy(
      RaftPeerId.valueOf(replyId)).getProtocol();
  try {
    return f.apply(proxy);
  } catch (ServiceException se) {
    throw ProtoUtils.toIOException(se);
  }
}
 
Example 18
Source Project: ratis   Source File: AssignmentMessage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ByteString getContent() {
  final int length = variable.length() + expression.length();
  final byte[] bytes = new byte[length];
  final int offset = variable.toBytes(bytes, 0);
  expression.toBytes(bytes, offset);
  return toByteString(bytes);
}
 
Example 19
Source Project: hadoop-ozone   Source File: Checksum.java    License: Apache License 2.0 5 votes vote down vote up
private static Function<ByteBuffer, ByteString> newChecksumByteBufferFunction(
    Supplier<ChecksumByteBuffer> constructor) {
  final ChecksumByteBuffer algorithm = constructor.get();
  return  data -> {
    algorithm.reset();
    algorithm.update(data);
    return int2ByteString((int)algorithm.getValue());
  };
}
 
Example 20
Source Project: hadoop-ozone   Source File: Checksum.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Compute checksum using the algorithm for the data upto the max length.
 * @param data input data
 * @param function the checksum function
 * @param maxLength the max length of data
 * @return computed checksum ByteString
 */
private static ByteString computeChecksum(ByteBuffer data,
    Function<ByteBuffer, ByteString> function, int maxLength) {
  final int limit = data.limit();
  try {
    final int maxIndex = data.position() + maxLength;
    if (limit > maxIndex) {
      data.limit(maxIndex);
    }
    return function.apply(data);
  } finally {
    data.limit(limit);
  }
}
 
Example 21
Source Project: incubator-ratis   Source File: RaftId.java    License: Apache License 2.0 5 votes vote down vote up
private static ByteString toByteString(UUID uuid) {
  Objects.requireNonNull(uuid, "uuid == null");
  final ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
  buf.putLong(uuid.getMostSignificantBits());
  buf.putLong(uuid.getLeastSignificantBits());
  return ByteString.copyFrom(buf.array());
}
 
Example 22
Source Project: ratis   Source File: FileInfo.java    License: Apache License 2.0 5 votes vote down vote up
private int write(long offset, ByteString data, boolean close) throws IOException {
  if (offset != writeSize) {
    throw new IOException("Offset/size mismatched: offset = " + offset
        + " != writeSize = " + writeSize + ", path=" + getRelativePath());
  }
  if (out == null) {
    throw new IOException("File output is not initialized, path=" + getRelativePath());
  }

  synchronized (out) {
    int n = 0;
    if (data != null) {
      final ByteBuffer buffer = data.asReadOnlyByteBuffer();
      try {
        for (; buffer.remaining() > 0; ) {
          n += out.write(buffer);
        }
      } finally {
        writeSize += n;
      }
    }

    if (close) {
      out.close();
    }
    return n;
  }
}
 
Example 23
Source Project: incubator-ratis   Source File: FileInfo.java    License: Apache License 2.0 5 votes vote down vote up
private int write(long offset, ByteString data, boolean close) throws IOException {
  // If leader finish write data with offset = 4096 and writeSize become 8192,
  // and 2 follower has not written the data with offset = 4096,
  // then start a leader election. So client will retry send the data with offset = 4096,
  // then offset < writeSize in leader.
  if (offset < writeSize) {
    return data.size();
  }
  if (offset != writeSize) {
    throw new IOException("Offset/size mismatched: offset = " + offset
        + " != writeSize = " + writeSize + ", path=" + getRelativePath());
  }
  if (out == null) {
    throw new IOException("File output is not initialized, path=" + getRelativePath());
  }

  synchronized (out) {
    int n = 0;
    if (data != null) {
      final ByteBuffer buffer = data.asReadOnlyByteBuffer();
      try {
        for (; buffer.remaining() > 0; ) {
          n += out.write(buffer);
        }
      } finally {
        writeSize += n;
      }
    }

    if (close) {
      out.close();
    }
    return n;
  }
}
 
Example 24
Source Project: ratis   Source File: FileStore.java    License: Apache License 2.0 5 votes vote down vote up
CompletableFuture<ReadReplyProto> read(String relative, long offset, long length) {
  final Supplier<String> name = () -> "read(" + relative
      + ", " + offset + ", " + length + ") @" + getId();
  final CheckedSupplier<ReadReplyProto, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    final FileInfo info = files.get(relative);
    final ReadReplyProto.Builder reply = ReadReplyProto.newBuilder()
        .setResolvedPath(FileStoreCommon.toByteString(info.getRelativePath()))
        .setOffset(offset);

    final ByteString bytes = info.read(this::resolve, offset, length);
    return reply.setData(bytes).build();
  }, name);
  return submit(task, reader);
}
 
Example 25
Source Project: incubator-ratis   Source File: LogServiceProtoUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static List<byte[]> toListByteArray(List<ByteString> list) {
  List<byte[]> retVal = new ArrayList<byte[]>(list.size());
  for(int i=0; i < list.size(); i++) {
    retVal.add(list.get(i).toByteArray());
  }
  return retVal;
}
 
Example 26
static void runTest(int length,
    BiFunction<BlockID, ByteString, ContainerCommandRequestProto> method)
    throws Exception {
  System.out.println("length=" + length);
  final BlockID blockID = new BlockID(RANDOM.nextLong(), RANDOM.nextLong());
  final ByteString data = newData(length);

  final ContainerCommandRequestProto original = method.apply(blockID, data);
  final ContainerCommandRequestMessage message
      = ContainerCommandRequestMessage.toMessage(original, null);
  final ContainerCommandRequestProto computed
      = ContainerCommandRequestMessage.toProto(message.getContent(), null);
  Assert.assertEquals(original, computed);
}
 
Example 27
Source Project: incubator-ratis   Source File: ServerProtoUtils.java    License: Apache License 2.0 5 votes vote down vote up
static StateMachineLogEntryProto toStateMachineLogEntryProto(
    ClientId clientId, long callId, ByteString logData, ByteString stateMachineData) {
  final StateMachineLogEntryProto.Builder b = StateMachineLogEntryProto.newBuilder()
      .setClientId(clientId.toByteString())
      .setCallId(callId)
      .setLogData(logData);
  if (stateMachineData != null) {
    b.setStateMachineEntry(toStateMachineEntryProtoBuilder(stateMachineData));
  }
  return b.build();
}
 
Example 28
Source Project: ratis   Source File: LogServiceProtoUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static ReadLogReplyProto toReadLogReplyProto(List<byte[]> entries, Throwable t) {
  ReadLogReplyProto.Builder builder = ReadLogReplyProto.newBuilder();
  if (t != null) {
    builder.setException(toLogException(t));
  } else {
    for(byte[] record: entries) {
      builder.addLogRecord( ByteString.copyFrom(record));
    }
  }
  return builder.build();
}
 
Example 29
Source Project: incubator-ratis   Source File: LogServiceRaftLogReader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the next log entry. Ensure {@link #hasNext()} returns true before
 * calling this method.
 */
@Override
public byte[] next() throws RaftLogIOException, InvalidProtocolBufferException {
  if (currentRecord == null) {
    throw new NoSuchElementException();
  }
  ByteString current = currentRecord;
  currentRecord = null;
  loadNext();
  return current.toByteArray();
}
 
Example 30
Source Project: incubator-ratis   Source File: FileStoreBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
Writer verify() throws IOException {
  final Random r = new Random(seed);
  final int size = fileSize.getSizeInt();

  for(int offset = 0; offset < size; ) {
    final int remaining = size - offset;
    final int n = Math.min(remaining, buffer.length);
    final ByteString read = client.read(fileName, offset, n);
    final ByteBuffer expected = randomBytes(n, r);
    verify(read, offset, n, expected);
    offset += n;
  }
  return this;
}