Java Code Examples for com.google.protobuf.ByteString#newInput()

The following examples show how to use com.google.protobuf.ByteString#newInput() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: EmptyInputStreamFactoryTest.java    From bazel-buildfarm with Apache License 2.0 6 votes vote down vote up
@Test
public void nonEmptyDigestIsDelegated() throws IOException, InterruptedException {
  ByteString content = ByteString.copyFromUtf8("Hello, World");
  Digest contentDigest = DIGEST_UTIL.compute(content);
  EmptyInputStreamFactory emptyFactory =
      new EmptyInputStreamFactory(
          new InputStreamFactory() {
            @Override
            public InputStream newInput(Digest digest, long offset) throws IOException {
              if (digest.equals(contentDigest)) {
                return content.newInput();
              }
              throw new IOException("invalid");
            }
          });
  InputStream in = emptyFactory.newInput(contentDigest, /* offset=*/ 0);
  assertThat(ByteString.readFrom(in)).isEqualTo(content);
}
 
Example 2
Source File: SparkOperationContext.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public OperationContext getClone() throws IOException, ClassNotFoundException{
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream oos = new ObjectOutputStream(baos);
    oos.writeObject(this);
    oos.flush();
    oos.close();
    ByteString bs = ZeroCopyLiteralByteString.wrap(baos.toByteArray());

    // Deserialize activation to clone it
    try (InputStream is = bs.newInput();
         ObjectInputStream ois = new ObjectInputStream(is)) {
        SparkOperationContext operationContext = (SparkOperationContext) ois.readObject();
        BroadcastedActivation broadcastedActivation = operationContext.broadcastedActivation;
        BroadcastedActivation.ActivationHolderAndBytes activationHolderAndBytes = broadcastedActivation.readActivationHolder();
        broadcastedActivation.setActivationHolder(activationHolderAndBytes.getActivationHolder());
        operationContext.op = broadcastedActivation.getActivationHolder().getOperationsMap().get(op.resultSetNumber());
        operationContext.activation = operationContext.broadcastedActivation.getActivationHolder().getActivation();
        return operationContext;
    }
}
 
Example 3
Source File: SparkLeanOperationContext.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public OperationContext getClone() throws IOException, ClassNotFoundException{
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream oos = new ObjectOutputStream(baos);
    oos.writeObject(this);
    oos.flush();
    oos.close();
    ByteString bs = ZeroCopyLiteralByteString.wrap(baos.toByteArray());

    // Deserialize activation to clone it
    try (InputStream is = bs.newInput();
         ObjectInputStream ois = new ObjectInputStream(is)) {
        SparkLeanOperationContext operationContext = (SparkLeanOperationContext) ois.readObject();
        BroadcastedActivation broadcastedActivation = operationContext.broadcastedActivation;
        BroadcastedActivation.ActivationHolderAndBytes activationHolderAndBytes = broadcastedActivation.readActivationHolder();
        broadcastedActivation.setActivationHolder(activationHolderAndBytes.getActivationHolder());
        operationContext.op = broadcastedActivation.getActivationHolder().getOperationsMap().get(op.resultSetNumber());
        operationContext.activation = operationContext.broadcastedActivation.getActivationHolder().getActivation();
        return operationContext;
    }
}
 
Example 4
Source File: BlockRepoSaver.java    From jelectrum with MIT License 5 votes vote down vote up
private void saveFile(String key, ByteString data, int cache_seconds)
{
  ObjectMetadata omd = new ObjectMetadata();
  omd.setCacheControl("max-age=" + cache_seconds);
  omd.setContentLength(data.size());


  PutObjectRequest put = new PutObjectRequest(bucket, key, data.newInput(), omd);
  put.setCannedAcl(CannedAccessControlList.PublicRead);
  put.setStorageClass(com.amazonaws.services.s3.model.StorageClass.StandardInfrequentAccess.toString());

  s3.putObject(put);

}
 
Example 5
Source File: TezUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
public static void addToConfFromByteString(Configuration configuration, ByteString byteString)
    throws IOException {
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
    readConfFromPB(confProto, configuration);
  }
}
 
Example 6
Source File: TezUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
public static Configuration createConfFromBaseConfAndPayload(TaskContext context)
    throws IOException {
  Configuration baseConf = context.getContainerConfiguration();
  Configuration configuration = new Configuration(baseConf);
  UserPayload payload = context.getUserPayload();
  ByteString byteString = ByteString.copyFrom(payload.getPayload());
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
    readConfFromPB(confProto, configuration);
    return configuration;
  }
}
 
Example 7
Source File: TezUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Convert a byte string to a Configuration object
 *
 * @param byteString byteString representation of the conf created using {@link
 *                   #createByteStringFromConf(org.apache.hadoop.conf.Configuration)}
 * @return Configuration
 * @throws java.io.IOException
 */
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
  Objects.requireNonNull(byteString, "ByteString must be specified");
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput());) {
    CodedInputStream in = CodedInputStream.newInstance(uncompressIs);
    in.setSizeLimit(Integer.MAX_VALUE);
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(in);
    Configuration conf = new Configuration(false);
    readConfFromPB(confProto, conf);
    return conf;
  }
}
 
Example 8
Source File: TezCommonUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Private
public static byte[] decompressByteStringToByteArray(ByteString byteString, Inflater inflater) throws IOException {
  inflater.reset();
  try (InflaterInputStream inflaterInputStream = new InflaterInputStream(byteString.newInput(), inflater)) {
    return IOUtils.toByteArray(inflaterInputStream);
  }
}
 
Example 9
Source File: OlapSerializationUtils.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static <R extends Serializable> R decode(ByteString commandBytes) throws IOException{
    InputStream is = commandBytes.newInput();
    ObjectInputStream ois = new ObjectInputStream(is);
    try{
        return (R)ois.readObject(); //shouldn't be a problem with any IOExceptions
    }catch(ClassNotFoundException e){
        throw new IOException(e); //shouldn't happen
    }
}
 
Example 10
Source File: TezUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * Convert compressed byte string to a Configuration object using protocol
 * buffer
 * 
 * @param byteString
 *          :compressed conf in Protocol buffer
 * @return Configuration
 * @throws IOException
 */
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
  Preconditions.checkNotNull(byteString, "ByteString must be specified");
  // SnappyInputStream uncompressIs = new
  // SnappyInputStream(byteString.newInput());
  InflaterInputStream uncompressIs = new InflaterInputStream(byteString.newInput());
  ConfigurationProto confProto = ConfigurationProto.parseFrom(uncompressIs);
  Configuration conf = new Configuration(false);
  readConfFromPB(confProto, conf);
  return conf;
}
 
Example 11
Source File: ProtoListUtil.java    From OpenYOLO-Android with Apache License 2.0 5 votes vote down vote up
/**
 * Reads a list of protos, using the provided parser, from the provided {@link ByteString}.
 * @throws IOException if the proto list could not be parsed.
 */
public static <T extends MessageLite> List<T> readMessageList(
        ByteString bytes,
        Parser<T> parser)
        throws IOException {
    InputStream stream = bytes.newInput();
    return readMessageList(stream, parser);
}
 
Example 12
Source File: UtxoTrieNode.java    From jelectrum with MIT License 5 votes vote down vote up
/**
 * Deserialize from a byte string
 */
public UtxoTrieNode(ByteString bs)
{
  try
  {
    DataInputStream din=new DataInputStream(bs.newInput());

    long ver = din.readLong();
    Assert.assertEquals(serialVersionUID, ver);

    prefix = SerialUtil.readString(din);
    int count = din.readInt();

    springs = new TreeMap<String, Sha256Hash>();

    for(int i=0; i<count; i++)
    {
      byte hash_bytes[]=new byte[32];
      String sub = SerialUtil.readString(din);
      din.readFully(hash_bytes);
      Sha256Hash hash = new Sha256Hash(hash_bytes);
      
      if (hash.equals(hash_null))
      {
        hash=null;
        springs.put(sub, null);
      }
      else
      {
        springs.put(sub, hash);
      }

    }
  }
  catch(java.io.IOException e)
  {
    throw new RuntimeException(e);
  }
  
}
 
Example 13
Source File: InMemoryCacheClient.java    From bazel with Apache License 2.0 5 votes vote down vote up
@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
  try (InputStream in = data.newInput()) {
    cas.put(digest, data.toByteArray());
  } catch (IOException e) {
    return Futures.immediateFailedFuture(e);
  }
  return Futures.immediateFuture(null);
}
 
Example 14
Source File: DiskCacheClient.java    From bazel with Apache License 2.0 5 votes vote down vote up
@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
  try (InputStream in = data.newInput()) {
    saveFile(digest.getHash(), in);
  } catch (IOException e) {
    return Futures.immediateFailedFuture(e);
  }
  return Futures.immediateFuture(null);
}
 
Example 15
Source File: HttpCacheClient.java    From bazel with Apache License 2.0 5 votes vote down vote up
@Override
public void uploadActionResult(ActionKey actionKey, ActionResult actionResult)
    throws IOException, InterruptedException {
  ByteString serialized = actionResult.toByteString();
  try (InputStream in = serialized.newInput()) {
    uploadBlocking(actionKey.getDigest().getHash(), serialized.size(), in, false);
  }
}
 
Example 16
Source File: HttpCacheClient.java    From bazel with Apache License 2.0 5 votes vote down vote up
@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
  try (InputStream in = data.newInput()) {
    uploadBlocking(digest.getHash(), digest.getSizeBytes(), in, /* casUpload= */ true);
  } catch (IOException | InterruptedException e) {
    return Futures.immediateFailedFuture(e);
  }
  return Futures.immediateFuture(null);
}
 
Example 17
Source File: ByteStringQueueInputStream.java    From bazel-buildfarm with Apache License 2.0 5 votes vote down vote up
private void advance() throws IOException {
  ByteString data = ByteString.EMPTY;
  while (hasNext() && data.isEmpty()) {
    try {
      data = queue.take();
    } catch (InterruptedException e) {
      throw new IOException(e);
    }
  }
  input = data.newInput();
}
 
Example 18
Source File: TezCommonUtils.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Private
public static byte[] decompressByteStringToByteArray(ByteString byteString) throws IOException {
  InflaterInputStream in = new InflaterInputStream(byteString.newInput());
  byte[] bytes = IOUtils.toByteArray(in);
  return bytes;
}
 
Example 19
Source File: ArtifactStoreTest.java    From modeldb with Apache License 2.0 4 votes vote down vote up
@Test
public void getArtifactFromCloudTest() {
  LOGGER.info("get artifact from cloud test start................................");

  try {
    ArtifactStoreBlockingStub artifactStoreBlockingStub =
        ArtifactStoreGrpc.newBlockingStub(channel);

    StoreArtifact storeArtifact =
        StoreArtifact.newBuilder()
            .setKey("verta_logo.png")
            .setPath(
                "https://www.verta.ai/static/logo-landing-424af27a5fc184c64225f604232db39e.png")
            .build();

    StoreArtifact.Response response = artifactStoreBlockingStub.storeArtifact(storeArtifact);

    String cloudFileKey = response.getArtifactStoreKey();
    String cloudFilePath = response.getArtifactStorePath();
    LOGGER.log(
        Level.INFO,
        "StoreArtifact.Response : \n cloudFileKey - "
            + cloudFileKey
            + " \n cloudFilePath - "
            + cloudFilePath);

    assumeTrue(cloudFileKey != null && !cloudFileKey.isEmpty());
    assumeTrue(cloudFilePath != null && !cloudFilePath.isEmpty());

    GetArtifact getArtifactRequest = GetArtifact.newBuilder().setKey(cloudFileKey).build();
    GetArtifact.Response getArtifactResponse =
        artifactStoreBlockingStub.getArtifact(getArtifactRequest);
    ByteString responseByteString = getArtifactResponse.getContents();
    InputStream inputStream = responseByteString.newInput();

    String rootPath = System.getProperty("user.dir");
    FileOutputStream fileOutputStream =
        new FileOutputStream(new File(rootPath + File.separator + cloudFileKey));
    IOUtils.copy(inputStream, fileOutputStream);
    fileOutputStream.close();
    inputStream.close();

    File downloadedFile = new File(rootPath + File.separator + cloudFileKey);
    if (!downloadedFile.exists()) {
      fail("File not fount at download destination");
    }
    downloadedFile.delete();

    DeleteArtifact deleteArtifact = DeleteArtifact.newBuilder().setKey(cloudFileKey).build();
    DeleteArtifact.Response deleteArtifactResponse =
        artifactStoreBlockingStub.deleteArtifact(deleteArtifact);
    assertTrue(deleteArtifactResponse.getStatus());

  } catch (Exception e) {
    e.printStackTrace();
    Status status = Status.fromThrowable(e);
    LOGGER.warning(
        "Error Code : " + status.getCode() + " Description : " + status.getDescription());
    fail();
  }

  LOGGER.info("get artifact from cloud test stop................................");
}