Java Code Examples for com.google.protobuf.ByteString#newOutput()

The following examples show how to use com.google.protobuf.ByteString#newOutput() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ProtobufByteStringSerDe.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
/**
 * Serialize the given value to byte string using the given mapper, employing the given codec algorithm.
 *
 * @param mapper object mapper
 * @param value  value to serialize
 * @param codec  codec
 * @return serialized bytes
 * @throws JsonGenerationException in case of serialization errors
 */
public static ByteString writeValue(ObjectMapper mapper, Object value, Codec codec)
    throws JsonGenerationException {
  final Output output = ByteString.newOutput();

  try {
    final OutputStream os = codec.compress(output);
    try {
      mapper.writer()
          .without(SerializationFeature.INDENT_OUTPUT)
          .writeValue(os, value);
    } finally {
      os.close();
    }
  } catch (IOException e) {
    // Should not happen but...
    throw new JsonGenerationException(e, null);
  }

  // Javadoc says data is copied, but it's more of a transfer of ownership!
  return output.toByteString();
}
 
Example 2
Source File: FlinkStateInternals.java    From flink-dataflow with Apache License 2.0 6 votes vote down vote up
@Override
public void persistState(StateCheckpointWriter checkpointBuilder) throws IOException {
	if (!contents.isEmpty()) {
		// serialize the coder.
		byte[] coder = InstantiationUtil.serializeObject(elemCoder);

		checkpointBuilder.addListUpdatesBuilder()
				.setTag(stateKey)
				.setData(coder)
				.writeInt(contents.size());

		for (T item : contents) {
			// encode the element
			ByteString.Output stream = ByteString.newOutput();
			elemCoder.encode(item, stream, Coder.Context.OUTER);
			ByteString data = stream.toByteString();

			// add the data to the checkpoint.
			checkpointBuilder.setData(data);
		}
	}
}
 
Example 3
Source File: FlinkStateInternals.java    From flink-dataflow with Apache License 2.0 6 votes vote down vote up
@Override
public void persistState(StateCheckpointWriter checkpointBuilder) throws IOException {
	if (!isClear) {
		// serialize the coder.
		byte[] coder = InstantiationUtil.serializeObject(accumCoder);

		// serialize the combiner.
		byte[] combiner = InstantiationUtil.serializeObject(combineFn);

		// encode the accumulator into a ByteString
		ByteString.Output stream = ByteString.newOutput();
		accumCoder.encode(accum, stream, Coder.Context.OUTER);
		ByteString data = stream.toByteString();

		// put the flag that the next serialized element is an accumulator
		checkpointBuilder.addAccumulatorBuilder()
			.setTag(stateKey)
			.setData(coder)
			.setData(combiner)
			.setData(data);
	}
}
 
Example 4
Source File: MRInputHelpers.java    From tez with Apache License 2.0 6 votes vote down vote up
@InterfaceStability.Evolving
@InterfaceAudience.LimitedPrivate({"hive, pig"})
public static MRRuntimeProtos.MRSplitProto createSplitProto(
    org.apache.hadoop.mapred.InputSplit oldSplit) throws IOException {
  MRRuntimeProtos.MRSplitProto.Builder builder = MRRuntimeProtos.MRSplitProto.newBuilder();

  builder.setSplitClassName(oldSplit.getClass().getName());

  ByteString.Output os = ByteString
      .newOutput(SPLIT_SERIALIZED_LENGTH_ESTIMATE);
  oldSplit.write(new NonSyncDataOutputStream(os));
  ByteString splitBs = os.toByteString();
  builder.setSplitBytes(splitBs);

  return builder.build();
}
 
Example 5
Source File: MemoryWriteOutputStream.java    From bazel-buildfarm with Apache License 2.0 6 votes vote down vote up
MemoryWriteOutputStream(
    ContentAddressableStorage storage,
    Digest digest,
    ListenableFuture<ByteString> writtenFuture) {
  this.storage = storage;
  this.digest = digest;
  this.writtenFuture = writtenFuture;
  if (digest.getSizeBytes() > Integer.MAX_VALUE) {
    throw new IllegalArgumentException(
        String.format(
            "content size %d exceeds maximum of %d", digest.getSizeBytes(), Integer.MAX_VALUE));
  }
  out = ByteString.newOutput((int) digest.getSizeBytes());
  hashOut = DigestUtil.forDigest(digest).newHashingOutputStream(out);
  addListener(
      () -> {
        future.set(null);
        try {
          hashOut.close();
        } catch (IOException e) {
          // ignore
        }
      },
      directExecutor());
}
 
Example 6
Source File: PersisterHelper.java    From kogito-runtimes with Apache License 2.0 6 votes vote down vote up
private static void writeStrategiesIndex(MarshallerWriteContext context,
                                          ProtobufMessages.Header.Builder _header) throws IOException {
     for( Entry<ObjectMarshallingStrategy,Integer> entry : context.usedStrategies.entrySet() ) {
Builder _strat = ProtobufMessages.Header.StrategyIndex.newBuilder()
                                  .setId( entry.getValue().intValue() )
                                  .setName( entry.getKey().getName()  );

         Context ctx = context.strategyContext.get( entry.getKey() );
         if( ctx != null ) {
             Output os = ByteString.newOutput();
             ctx.write( new DroolsObjectOutputStream( os ) );
             _strat.setData( os.toByteString() );
             os.close();
         }
         _header.addStrategy( _strat.build() );
     }
 }
 
Example 7
Source File: SimpleUtxoMgr.java    From jelectrum with MIT License 6 votes vote down vote up
public static ByteString getKey(ByteString scriptHash, Sha256Hash tx_id, int idx)
{
  try
  {
    ByteString.Output key_out = ByteString.newOutput(32+32+4);

    key_out.write(scriptHash.toByteArray());
    key_out.write(tx_id.getBytes());

    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.order(java.nio.ByteOrder.LITTLE_ENDIAN);
    bb.putInt(idx);
    key_out.write(bb.array());
    return key_out.toByteString();
  }
  catch(java.io.IOException e)
  {
    throw new RuntimeException(e);
  }

}
 
Example 8
Source File: RocksDBMapMutationSet.java    From jelectrum with MIT License 6 votes vote down vote up
private ByteString getDBKey(ByteString key)
{
  try
  {
    ByteString.Output out = ByteString.newOutput(100);
    out.write(name_bytes);
    out.write(key.toByteArray());
    out.write(sep);
    ByteString w = out.toByteString();
    return w;
  }
  catch(java.io.IOException e)
  {
    throw new RuntimeException(e);
  }
}
 
Example 9
Source File: GenQueryOutputStream.java    From bazel with Apache License 2.0 6 votes vote down vote up
private void maybeStartCompression(int additionalBytes) throws IOException {
  if (!compressionEnabled) {
    return;
  }

  if (compressed) {
    return;
  }

  if (bytesWritten + additionalBytes < COMPRESSION_THRESHOLD) {
    return;
  }

  ByteString.Output compressedBytesOut = ByteString.newOutput();
  GZIPOutputStream gzipOut = new GZIPOutputStream(compressedBytesOut);
  bytesOut.writeTo(gzipOut);
  bytesOut = compressedBytesOut;
  out = gzipOut;
  compressed = true;
}
 
Example 10
Source File: TezCommonUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Private
public static ByteString compressByteArrayToByteString(byte[] inBytes) throws IOException {
  ByteString.Output os = ByteString.newOutput();
  DeflaterOutputStream compressOs = new DeflaterOutputStream(os, new Deflater(
      Deflater.BEST_COMPRESSION));
  compressOs.write(inBytes);
  compressOs.finish();
  ByteString byteString = os.toByteString();
  return byteString;
}
 
Example 11
Source File: MetadataProtoUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public static ByteString toProtobuf(BytesOutput out) {
  ByteString.Output output = ByteString.newOutput();
  try {
    out.writeTo(output);
    return output.toByteString();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 12
Source File: TestUtils.java    From bazel with Apache License 2.0 5 votes vote down vote up
public static <T> ByteString toBytes(SerializationContext serializationContext, T value)
    throws IOException, SerializationException {
  ByteString.Output output = ByteString.newOutput();
  CodedOutputStream codedOut = CodedOutputStream.newInstance(output);
  serializationContext.serialize(value, codedOut);
  codedOut.flush();
  return output.toByteString();
}
 
Example 13
Source File: GenQueryOutputStream.java    From bazel with Apache License 2.0 5 votes vote down vote up
@Override
public ByteString getBytes() throws IOException {
  ByteString.Output out = ByteString.newOutput(size);
  try (GZIPInputStream gzipIn = new GZIPInputStream(compressedData.newInput())) {
    ByteStreams.copy(gzipIn, out);
  }
  return out.toByteString();
}
 
Example 14
Source File: TezUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Convert a Configuration to compressed ByteString using Protocol buffer
 *
 * @param conf
 *          : Configuration to be converted
 * @return PB ByteString (compressed)
 * @throws java.io.IOException
 */
public static ByteString createByteStringFromConf(Configuration conf) throws IOException {
  Objects.requireNonNull(conf, "Configuration must be specified");
  ByteString.Output os = ByteString.newOutput();
  SnappyOutputStream compressOs = new SnappyOutputStream(os);
  try {
    writeConfInPB(compressOs, conf);
  } finally {
    if (compressOs != null) {
      compressOs.close();
    }
  }
  return os.toByteString();
}
 
Example 15
Source File: BuildOptions.java    From bazel with Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(
    SerializationContext context,
    OptionsDiffForReconstruction diff,
    CodedOutputStream codedOut)
    throws SerializationException, IOException {
  OptionsDiffCache cache = context.getDependency(OptionsDiffCache.class);
  ByteString bytes = cache.getBytesFromOptionsDiff(diff);
  if (bytes == null) {
    context = context.getNewNonMemoizingContext();
    ByteString.Output byteStringOut = ByteString.newOutput();
    CodedOutputStream bytesOut = CodedOutputStream.newInstance(byteStringOut);
    context.serialize(diff.differingOptions, bytesOut);
    context.serialize(diff.extraFirstFragmentClasses, bytesOut);
    context.serialize(diff.extraSecondFragments, bytesOut);
    bytesOut.writeByteArrayNoTag(diff.baseFingerprint);
    context.serialize(diff.checksum, bytesOut);
    context.serialize(diff.differingStarlarkOptions, bytesOut);
    context.serialize(diff.extraFirstStarlarkOptions, bytesOut);
    context.serialize(diff.extraSecondStarlarkOptions, bytesOut);
    bytesOut.flush();
    byteStringOut.flush();
    int optionsDiffSize = byteStringOut.size();
    bytes = byteStringOut.toByteString();
    cache.putBytesFromOptionsDiff(diff, bytes);
    logger.atFine().log(
        "Serialized OptionsDiffForReconstruction %s. Diff took %d bytes.",
        diff, optionsDiffSize);
  }
  codedOut.writeBytesNoTag(bytes);
}
 
Example 16
Source File: TestUtils.java    From bazel with Apache License 2.0 5 votes vote down vote up
public static ByteString toBytesMemoized(Object original, ObjectCodecRegistry registry)
    throws IOException, SerializationException {
  ByteString.Output output = ByteString.newOutput();
  CodedOutputStream codedOut = CodedOutputStream.newInstance(output);
  new ObjectCodecs(registry).serializeMemoized(original, codedOut);
  codedOut.flush();
  return output.toByteString();
}
 
Example 17
Source File: ObjectCodecs.java    From bazel with Apache License 2.0 5 votes vote down vote up
private static ByteString serializeToByteString(Object subject, SerializeCall wrapped)
    throws SerializationException {
  ByteString.Output resultOut = ByteString.newOutput();
  CodedOutputStream codedOut = CodedOutputStream.newInstance(resultOut);
  wrapped.serialize(subject, codedOut);
  try {
    codedOut.flush();
    return resultOut.toByteString();
  } catch (IOException e) {
    throw new SerializationException("Failed to serialize " + subject, e);
  }
}
 
Example 18
Source File: BlockListAsLongs.java    From hadoop with Apache License 2.0 4 votes vote down vote up
Builder() {
  out = ByteString.newOutput(64*1024);
  cos = CodedOutputStream.newInstance(out);
}
 
Example 19
Source File: BinTools.java    From bazel with Apache License 2.0 4 votes vote down vote up
@Override
public ByteString getBytes() throws IOException {
  ByteString.Output out = ByteString.newOutput();
  writeTo(out);
  return out.toByteString();
}
 
Example 20
Source File: ByteStreamServiceWriter.java    From bazel-buildfarm with Apache License 2.0 4 votes vote down vote up
public ByteStreamServiceWriter(
    String resourceName, SettableFuture<ByteString> content, int initialCapacity) {
  this.resourceName = resourceName;
  this.content = content;
  out = ByteString.newOutput(initialCapacity);
}