Java Code Examples for com.google.protobuf.ByteString#newInput()

@Test
public void nonEmptyDigestIsDelegated() throws IOException, InterruptedException {
  ByteString content = ByteString.copyFromUtf8("Hello, World");
  Digest contentDigest = DIGEST_UTIL.compute(content);
  EmptyInputStreamFactory emptyFactory =
      new EmptyInputStreamFactory(
          new InputStreamFactory() {
            @Override
            public InputStream newInput(Digest digest, long offset) throws IOException {
              if (digest.equals(contentDigest)) {
                return content.newInput();
              }
              throw new IOException("invalid");
            }
          });
  InputStream in = emptyFactory.newInput(contentDigest, /* offset=*/ 0);
  assertThat(ByteString.readFrom(in)).isEqualTo(content);
}
 

@Override
public OperationContext getClone() throws IOException, ClassNotFoundException{
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream oos = new ObjectOutputStream(baos);
    oos.writeObject(this);
    oos.flush();
    oos.close();
    ByteString bs = ZeroCopyLiteralByteString.wrap(baos.toByteArray());

    // Deserialize activation to clone it
    try (InputStream is = bs.newInput();
         ObjectInputStream ois = new ObjectInputStream(is)) {
        SparkOperationContext operationContext = (SparkOperationContext) ois.readObject();
        BroadcastedActivation broadcastedActivation = operationContext.broadcastedActivation;
        BroadcastedActivation.ActivationHolderAndBytes activationHolderAndBytes = broadcastedActivation.readActivationHolder();
        broadcastedActivation.setActivationHolder(activationHolderAndBytes.getActivationHolder());
        operationContext.op = broadcastedActivation.getActivationHolder().getOperationsMap().get(op.resultSetNumber());
        operationContext.activation = operationContext.broadcastedActivation.getActivationHolder().getActivation();
        return operationContext;
    }
}
 

@Override
public OperationContext getClone() throws IOException, ClassNotFoundException{
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream oos = new ObjectOutputStream(baos);
    oos.writeObject(this);
    oos.flush();
    oos.close();
    ByteString bs = ZeroCopyLiteralByteString.wrap(baos.toByteArray());

    // Deserialize activation to clone it
    try (InputStream is = bs.newInput();
         ObjectInputStream ois = new ObjectInputStream(is)) {
        SparkLeanOperationContext operationContext = (SparkLeanOperationContext) ois.readObject();
        BroadcastedActivation broadcastedActivation = operationContext.broadcastedActivation;
        BroadcastedActivation.ActivationHolderAndBytes activationHolderAndBytes = broadcastedActivation.readActivationHolder();
        broadcastedActivation.setActivationHolder(activationHolderAndBytes.getActivationHolder());
        operationContext.op = broadcastedActivation.getActivationHolder().getOperationsMap().get(op.resultSetNumber());
        operationContext.activation = operationContext.broadcastedActivation.getActivationHolder().getActivation();
        return operationContext;
    }
}
 

private void saveFile(String key, ByteString data, int cache_seconds)
{
  ObjectMetadata omd = new ObjectMetadata();
  omd.setCacheControl("max-age=" + cache_seconds);
  omd.setContentLength(data.size());


  PutObjectRequest put = new PutObjectRequest(bucket, key, data.newInput(), omd);
  put.setCannedAcl(CannedAccessControlList.PublicRead);
  put.setStorageClass(com.amazonaws.services.s3.model.StorageClass.StandardInfrequentAccess.toString());

  s3.putObject(put);

}
 

public static void addToConfFromByteString(Configuration configuration, ByteString byteString)
    throws IOException {
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
    readConfFromPB(confProto, configuration);
  }
}
 

public static Configuration createConfFromBaseConfAndPayload(TaskContext context)
    throws IOException {
  Configuration baseConf = context.getContainerConfiguration();
  Configuration configuration = new Configuration(baseConf);
  UserPayload payload = context.getUserPayload();
  ByteString byteString = ByteString.copyFrom(payload.getPayload());
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
    readConfFromPB(confProto, configuration);
    return configuration;
  }
}
 

/**
 * Convert a byte string to a Configuration object
 *
 * @param byteString byteString representation of the conf created using {@link
 *                   #createByteStringFromConf(org.apache.hadoop.conf.Configuration)}
 * @return Configuration
 * @throws java.io.IOException
 */
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
  Objects.requireNonNull(byteString, "ByteString must be specified");
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput());) {
    CodedInputStream in = CodedInputStream.newInstance(uncompressIs);
    in.setSizeLimit(Integer.MAX_VALUE);
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(in);
    Configuration conf = new Configuration(false);
    readConfFromPB(confProto, conf);
    return conf;
  }
}
 

@Private
public static byte[] decompressByteStringToByteArray(ByteString byteString, Inflater inflater) throws IOException {
  inflater.reset();
  try (InflaterInputStream inflaterInputStream = new InflaterInputStream(byteString.newInput(), inflater)) {
    return IOUtils.toByteArray(inflaterInputStream);
  }
}
 

@SuppressWarnings("unchecked")
public static <R extends Serializable> R decode(ByteString commandBytes) throws IOException{
    InputStream is = commandBytes.newInput();
    ObjectInputStream ois = new ObjectInputStream(is);
    try{
        return (R)ois.readObject(); //shouldn't be a problem with any IOExceptions
    }catch(ClassNotFoundException e){
        throw new IOException(e); //shouldn't happen
    }
}
 

/**
 * Convert compressed byte string to a Configuration object using protocol
 * buffer
 * 
 * @param byteString
 *          :compressed conf in Protocol buffer
 * @return Configuration
 * @throws IOException
 */
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
  Preconditions.checkNotNull(byteString, "ByteString must be specified");
  // SnappyInputStream uncompressIs = new
  // SnappyInputStream(byteString.newInput());
  InflaterInputStream uncompressIs = new InflaterInputStream(byteString.newInput());
  ConfigurationProto confProto = ConfigurationProto.parseFrom(uncompressIs);
  Configuration conf = new Configuration(false);
  readConfFromPB(confProto, conf);
  return conf;
}
 

/**
 * Reads a list of protos, using the provided parser, from the provided {@link ByteString}.
 * @throws IOException if the proto list could not be parsed.
 */
public static <T extends MessageLite> List<T> readMessageList(
        ByteString bytes,
        Parser<T> parser)
        throws IOException {
    InputStream stream = bytes.newInput();
    return readMessageList(stream, parser);
}
 

/**
 * Deserialize from a byte string
 */
public UtxoTrieNode(ByteString bs)
{
  try
  {
    DataInputStream din=new DataInputStream(bs.newInput());

    long ver = din.readLong();
    Assert.assertEquals(serialVersionUID, ver);

    prefix = SerialUtil.readString(din);
    int count = din.readInt();

    springs = new TreeMap<String, Sha256Hash>();

    for(int i=0; i<count; i++)
    {
      byte hash_bytes[]=new byte[32];
      String sub = SerialUtil.readString(din);
      din.readFully(hash_bytes);
      Sha256Hash hash = new Sha256Hash(hash_bytes);
      
      if (hash.equals(hash_null))
      {
        hash=null;
        springs.put(sub, null);
      }
      else
      {
        springs.put(sub, hash);
      }

    }
  }
  catch(java.io.IOException e)
  {
    throw new RuntimeException(e);
  }
  
}
 

@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
  try (InputStream in = data.newInput()) {
    cas.put(digest, data.toByteArray());
  } catch (IOException e) {
    return Futures.immediateFailedFuture(e);
  }
  return Futures.immediateFuture(null);
}
 

@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
  try (InputStream in = data.newInput()) {
    saveFile(digest.getHash(), in);
  } catch (IOException e) {
    return Futures.immediateFailedFuture(e);
  }
  return Futures.immediateFuture(null);
}
 

@Override
public void uploadActionResult(ActionKey actionKey, ActionResult actionResult)
    throws IOException, InterruptedException {
  ByteString serialized = actionResult.toByteString();
  try (InputStream in = serialized.newInput()) {
    uploadBlocking(actionKey.getDigest().getHash(), serialized.size(), in, false);
  }
}
 

@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
  try (InputStream in = data.newInput()) {
    uploadBlocking(digest.getHash(), digest.getSizeBytes(), in, /* casUpload= */ true);
  } catch (IOException | InterruptedException e) {
    return Futures.immediateFailedFuture(e);
  }
  return Futures.immediateFuture(null);
}
 

private void advance() throws IOException {
  ByteString data = ByteString.EMPTY;
  while (hasNext() && data.isEmpty()) {
    try {
      data = queue.take();
    } catch (InterruptedException e) {
      throw new IOException(e);
    }
  }
  input = data.newInput();
}
 

@Private
public static byte[] decompressByteStringToByteArray(ByteString byteString) throws IOException {
  InflaterInputStream in = new InflaterInputStream(byteString.newInput());
  byte[] bytes = IOUtils.toByteArray(in);
  return bytes;
}
 

@Test
public void getArtifactFromCloudTest() {
  LOGGER.info("get artifact from cloud test start................................");

  try {
    ArtifactStoreBlockingStub artifactStoreBlockingStub =
        ArtifactStoreGrpc.newBlockingStub(channel);

    StoreArtifact storeArtifact =
        StoreArtifact.newBuilder()
            .setKey("verta_logo.png")
            .setPath(
                "https://www.verta.ai/static/logo-landing-424af27a5fc184c64225f604232db39e.png")
            .build();

    StoreArtifact.Response response = artifactStoreBlockingStub.storeArtifact(storeArtifact);

    String cloudFileKey = response.getArtifactStoreKey();
    String cloudFilePath = response.getArtifactStorePath();
    LOGGER.log(
        Level.INFO,
        "StoreArtifact.Response : \n cloudFileKey - "
            + cloudFileKey
            + " \n cloudFilePath - "
            + cloudFilePath);

    assumeTrue(cloudFileKey != null && !cloudFileKey.isEmpty());
    assumeTrue(cloudFilePath != null && !cloudFilePath.isEmpty());

    GetArtifact getArtifactRequest = GetArtifact.newBuilder().setKey(cloudFileKey).build();
    GetArtifact.Response getArtifactResponse =
        artifactStoreBlockingStub.getArtifact(getArtifactRequest);
    ByteString responseByteString = getArtifactResponse.getContents();
    InputStream inputStream = responseByteString.newInput();

    String rootPath = System.getProperty("user.dir");
    FileOutputStream fileOutputStream =
        new FileOutputStream(new File(rootPath + File.separator + cloudFileKey));
    IOUtils.copy(inputStream, fileOutputStream);
    fileOutputStream.close();
    inputStream.close();

    File downloadedFile = new File(rootPath + File.separator + cloudFileKey);
    if (!downloadedFile.exists()) {
      fail("File not fount at download destination");
    }
    downloadedFile.delete();

    DeleteArtifact deleteArtifact = DeleteArtifact.newBuilder().setKey(cloudFileKey).build();
    DeleteArtifact.Response deleteArtifactResponse =
        artifactStoreBlockingStub.deleteArtifact(deleteArtifact);
    assertTrue(deleteArtifactResponse.getStatus());

  } catch (Exception e) {
    e.printStackTrace();
    Status status = Status.fromThrowable(e);
    LOGGER.warning(
        "Error Code : " + status.getCode() + " Description : " + status.getDescription());
    fail();
  }

  LOGGER.info("get artifact from cloud test stop................................");
}