Java Code Examples for com.google.common.io.ByteStreams#read()

The following examples show how to use com.google.common.io.ByteStreams#read() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   File: ParquetCompressionUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static Slice decompressGzip(Slice input, int uncompressedSize)
        throws IOException
{
    if (uncompressedSize == 0) {
        return EMPTY_SLICE;
    }

    try (GZIPInputStream gzipInputStream = new GZIPInputStream(input.getInput(), min(GZIP_BUFFER_SIZE, input.length()))) {
        byte[] buffer = new byte[uncompressedSize];
        int bytesRead = ByteStreams.read(gzipInputStream, buffer, 0, buffer.length);
        if (bytesRead != uncompressedSize) {
            throw new IllegalArgumentException(format("Invalid uncompressedSize for GZIP input. Expected %s, actual: %s", uncompressedSize, bytesRead));
        }
        // Verify we're at EOF and aren't truncating the input
        checkArgument(gzipInputStream.read() == -1, "Invalid uncompressedSize for GZIP input. Actual size exceeds %s bytes", uncompressedSize);
        return wrappedBuffer(buffer, 0, bytesRead);
    }
}
 
Example 2
List<byte[]> fragmentInternal(InputStream is, int fragmentSizeB, int maxSizeB) throws IOException {
  if (fragmentSizeB != maxSizeB) {
    throw new IOException(Utils.format(
        "Invalid configuration, fragmentSize '{}' and maxSize '{}' should be the same", fragmentSizeB, maxSizeB));
  }
  byte[] buffer = new byte[fragmentSizeB];
  int read = ByteStreams.read(is, buffer, 0, fragmentSizeB);
  if (is.read() > -1) {
    throw new IOException(Utils.format("Maximum data size '{}' exceeded", maxSizeB));
  }
  byte[] data = buffer;
  if (read < buffer.length) {
    data = new byte[read];
    System.arraycopy(buffer, 0, data, 0, read);
  }
  return ImmutableList.of(data);
}
 
Example 3
Source Project: jkube   File: LogRequestor.java    License: Eclipse Public License 2.0 5 votes vote down vote up
/**
 * This is a copy of ByteStreams.readFully(), with the slight change that it throws
 * NoBytesReadException if zero bytes are read. Otherwise it is identical.
 *
 * @param in
 * @param bytes
 * @throws IOException
 */
private void readFully(InputStream in, byte[] bytes) throws IOException {
    int read = ByteStreams.read(in, bytes, 0, bytes.length);
    if (read == 0) {
        throw new NoBytesReadException();
    } else if (read != bytes.length) {
        throw new EOFException("reached end of stream after reading "
                + read + " bytes; " + bytes.length + " bytes expected");
    }
}
 
Example 4
Source Project: emodb   File: DefaultBlobStore.java    License: Apache License 2.0 5 votes vote down vote up
private StorageSummary putObject(Table table, String blobId, InputSupplier<? extends InputStream> in, Map<String, String> attributes) throws IOException {
    long timestamp = _storageProvider.getCurrentTimestamp(table);
    int chunkSize = _storageProvider.getDefaultChunkSize();
    checkArgument(chunkSize > 0);
    DigestInputStream md5In = new DigestInputStream(in.getInput(), getMessageDigest("MD5"));
    DigestInputStream sha1In = new DigestInputStream(md5In, getMessageDigest("SHA-1"));

    // A more aggressive solution like the Astyanax ObjectWriter recipe would improve performance by pipelining
    // reading the input stream and writing chunks, and issuing the writes in parallel.
    byte[] bytes = new byte[chunkSize];
    long length = 0;
    int chunkCount = 0;
    for (; ; ) {
        int chunkLength;
        try {
            chunkLength = ByteStreams.read(sha1In, bytes, 0, bytes.length);
        } catch (IOException e) {
            LOGGER.error("Failed to read input stream", e);
            throw Throwables.propagate(e);
        }
        if (chunkLength == 0) {
            break;
        }
        ByteBuffer buffer = ByteBuffer.wrap(bytes, 0, chunkLength);
        _storageProvider.writeChunk(table, blobId, chunkCount, buffer, timestamp);
        length += chunkLength;
        chunkCount++;
    }

    // Include two types of hash: md5 (because it's common) and sha1 (because it's secure)
    String md5 = Hex.encodeHexString(md5In.getMessageDigest().digest());
    String sha1 = Hex.encodeHexString(sha1In.getMessageDigest().digest());

    return new StorageSummary(length, chunkCount, chunkSize, md5, sha1, attributes, timestamp);
}
 
Example 5
Source Project: emodb   File: StashFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int read(long position, byte[] buffer, int offset, int length)
        throws IOException {
    try (InputStream in = _stashReader.getRawSplitPart(_stashSplit, Range.closedOpen(position, position + length))) {
        return ByteStreams.read(in, buffer, offset, length);
    }
}
 
Example 6
private void perform ( final Path file, final Map<String, String> metadata ) throws IOException
{
    try ( final GZIPInputStream gis = new GZIPInputStream ( new FileInputStream ( file.toFile () ) );
          final TarArchiveInputStream tis = new TarArchiveInputStream ( gis ) )
    {
        TarArchiveEntry entry;
        while ( ( entry = tis.getNextTarEntry () ) != null )
        {
            if ( entry.getName ().equals ( "package/package.json" ) )
            {
                final byte[] data = new byte[(int)entry.getSize ()];
                ByteStreams.read ( tis, data, 0, data.length );

                final String str = StandardCharsets.UTF_8.decode ( ByteBuffer.wrap ( data ) ).toString ();

                try
                {
                    // test parse
                    new JsonParser ().parse ( str );
                    // store
                    metadata.put ( "package.json", str );
                }
                catch ( final JsonParseException e )
                {
                    // ignore
                }

                break; // stop parsing the archive
            }
        }

    }
}
 
Example 7
Source Project: osm-lib   File: VEXBlock.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
/** */
private void readHeader(InputStream in) {
    byte[] fourBytes = new byte[4];
    try {
        int nRead = ByteStreams.read(in, fourBytes, 0, 4);
        if (nRead == 0) {
            // ByteStreams.read attempts to fully read 4 bytes and returns the number of bytes read,
            // which is only less than 4 if upstream.read() returned a negative value (EOF).
            LOG.debug("Hit end of file, no more blocks to read.");
            nBytes = 0;
            entityType = VexFormat.VEX_NONE; // indicates no more blocks
            return;
        }
        String s = new String(fourBytes);
        if (s.equals("VEXN")) {
            entityType = VexFormat.VEX_NODE;
        } else if (s.equals("VEXW")) {
            entityType = VexFormat.VEX_WAY;
        } else if (s.equals("VEXR")) {
            entityType = VexFormat.VEX_RELATION;
        } else {
            LOG.error("Unrecognized block type '{}', aborting VEX read.", entityType);
            throw new RuntimeException("Uncrecoginzed VEX block type.");
        }
        ByteStreams.readFully(in, fourBytes);
        nEntities = Ints.fromByteArray(fourBytes);
        ByteStreams.readFully(in, fourBytes);
        nBytes = Ints.fromByteArray(fourBytes);
        if (nBytes < 0 || nBytes > BUFFER_SIZE) {
            throw new RuntimeException("Block has impossible compressed data size, it is probably corrupted.");
        }
        if (nEntities < 0 || nEntities > BUFFER_SIZE) {
            throw new RuntimeException("Block contains impossible number of entities, it is probably corrupted.");
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example 8
Source Project: docker-client   File: LogReader.java    License: Apache License 2.0 5 votes vote down vote up
public LogMessage nextMessage() throws IOException {
  stream.mark(HEADER_SIZE);

  // Read header
  final byte[] headerBytes = new byte[HEADER_SIZE];
  final int n = ByteStreams.read(stream, headerBytes, 0, HEADER_SIZE);
  if (n == 0) {
    return null;
  }
  final ByteBuffer header = ByteBuffer.wrap(headerBytes);
  int streamId = header.get();
  final int idZ = header.getInt(0);

  // Read frame
  final byte[] frame;
  // Header format is : {STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4}
  if (idZ == 0 || idZ == 0x01000000 || idZ == 0x02000000) {
    header.position(FRAME_SIZE_OFFSET);
    final int frameSize = header.getInt();
    frame = new byte[frameSize];
  } else {
    stream.reset();
    streamId = Stream.STDOUT.id();
    frame = new byte[stream.available()];
  }
  ByteStreams.readFully(stream, frame);
  return new LogMessage(streamId, ByteBuffer.wrap(frame));
}
 
Example 9
/**
 * This is a copy of ByteStreams.readFully(), with the slight change that it throws
 * NoBytesReadException if zero bytes are read. Otherwise it is identical.
 *
 * @param in
 * @param bytes
 * @throws IOException
 */
private void readFully(InputStream in, byte[] bytes) throws IOException {
    int read = ByteStreams.read(in, bytes, 0, bytes.length);
    if (read == 0) {
        throw new NoBytesReadException();
    } else if (read != bytes.length) {
        throw new EOFException("reached end of stream after reading "
                + read + " bytes; " + bytes.length + " bytes expected");
    }
}
 
Example 10
Source Project: bazel   File: FileSystemUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads at most {@code limit} bytes from {@code inputFile} and returns it as a byte array.
 *
 * @throws IOException if there was an error.
 */
public static byte[] readContentWithLimit(Path inputFile, int limit) throws IOException {
  Preconditions.checkArgument(limit >= 0, "limit needs to be >=0, but it is %s", limit);
  ByteSource byteSource = asByteSource(inputFile);
  byte[] buffer = new byte[limit];
  try (InputStream inputStream = byteSource.openBufferedStream()) {
    int read = ByteStreams.read(inputStream, buffer, 0, limit);
    return read == limit ? buffer : Arrays.copyOf(buffer, read);
  }
}
 
Example 11
Source Project: Strata   File: UnicodeBom.java    License: Apache License 2.0 5 votes vote down vote up
BomReader(InputStream inputStream) throws IOException {
  super(inputStream);

  Charset encoding;
  byte[] bom = new byte[MAX_BOM_SIZE];

  // read first 3 bytes such that they can be pushed back later
  PushbackInputStream pushbackStream = new PushbackInputStream(inputStream, MAX_BOM_SIZE);
  int bytesRead = ByteStreams.read(pushbackStream, bom, 0, 3);

  // look for BOM and adapt, defauling to UTF-8
  if (bytesRead >= 3 && bom[0] == X_EF && bom[1] == X_BB && bom[2] == X_BF) {
    encoding = StandardCharsets.UTF_8;
    pushbackStream.unread(bom, 3, (bytesRead - 3));

  } else if (bytesRead >= 2 && bom[0] == X_FE && bom[1] == X_FF) {
    encoding = StandardCharsets.UTF_16BE;
    pushbackStream.unread(bom, 2, (bytesRead - 2));

  } else if (bytesRead >= 2 && bom[0] == X_FF && bom[1] == X_FE) {
    encoding = StandardCharsets.UTF_16LE;
    pushbackStream.unread(bom, 2, (bytesRead - 2));

  } else {
    encoding = StandardCharsets.UTF_8;
    pushbackStream.unread(bom, 0, bytesRead);
  }

  // use Java standard code now we know the encoding
  this.underlying = new InputStreamReader(pushbackStream, encoding);
}
 
Example 12
Source Project: buck   File: MoreAsserts.java    License: Apache License 2.0 5 votes vote down vote up
public static void assertContentsEqual(Path one, Path two) throws IOException {
  Preconditions.checkNotNull(one);
  Preconditions.checkNotNull(two);

  if (one.equals(two)) {
    return;
  }

  if (Files.size(one) != Files.size(two)) {
    fail(
        String.format(
            "File sizes differ: %s (%d bytes), %s (%d bytes)",
            one, Files.size(one), two, Files.size(two)));
  }

  try (InputStream ois = Files.newInputStream(one);
      InputStream tis = Files.newInputStream(two)) {
    byte[] bo = new byte[BUFFER_SIZE];
    byte[] bt = new byte[BUFFER_SIZE];

    while (true) {
      int read1 = ByteStreams.read(ois, bo, 0, BUFFER_SIZE);
      int read2 = ByteStreams.read(tis, bt, 0, BUFFER_SIZE);
      if (read1 != read2 || !Arrays.equals(bo, bt)) {
        fail(String.format("Contents of files differ: %s, %s", one, two));
      } else if (read1 != BUFFER_SIZE) {
        return;
      }
    }
  }
}
 
Example 13
Source Project: buck   File: BserDeserializer.java    License: Apache License 2.0 4 votes vote down vote up
private ByteBuffer readBserBuffer(InputStream inputStream) throws IOException {
  ByteBuffer sniffBuffer = ByteBuffer.allocate(SNIFF_BUFFER_SIZE).order(ByteOrder.nativeOrder());
  Preconditions.checkState(sniffBuffer.hasArray());

  int sniffBytesRead = ByteStreams.read(inputStream, sniffBuffer.array(), 0, INITIAL_SNIFF_LEN);
  if (sniffBytesRead < INITIAL_SNIFF_LEN) {
    throw new BserEofException(
        String.format(
            "Invalid BSER header (expected %d bytes, got %d bytes)",
            INITIAL_SNIFF_LEN, sniffBytesRead));
  }

  if (sniffBuffer.get() != 0x00 || sniffBuffer.get() != 0x01) {
    throw new IOException("Invalid BSER header");
  }

  byte lengthType = sniffBuffer.get();
  int lengthBytesRemaining;
  switch (lengthType) {
    case BSER_INT8:
      lengthBytesRemaining = 1;
      break;
    case BSER_INT16:
      lengthBytesRemaining = 2;
      break;
    case BSER_INT32:
      lengthBytesRemaining = 4;
      break;
    case BSER_INT64:
      lengthBytesRemaining = 8;
      break;
    default:
      throw new IOException(String.format("Unrecognized BSER header length type %d", lengthType));
  }
  int lengthBytesRead =
      ByteStreams.read(
          inputStream, sniffBuffer.array(), sniffBuffer.position(), lengthBytesRemaining);
  if (lengthBytesRead < lengthBytesRemaining) {
    throw new BserEofException(
        String.format(
            "Invalid BSER header length (expected %d bytes, got %d bytes)",
            lengthBytesRemaining, lengthBytesRead));
  }
  int bytesRemaining = deserializeIntLen(sniffBuffer, lengthType);

  ByteBuffer bserBuffer = ByteBuffer.allocate(bytesRemaining).order(ByteOrder.nativeOrder());
  Preconditions.checkState(bserBuffer.hasArray());

  int remainingBytesRead = ByteStreams.read(inputStream, bserBuffer.array(), 0, bytesRemaining);

  if (remainingBytesRead < bytesRemaining) {
    throw new IOException(
        String.format(
            "Invalid BSER header (expected %d bytes, got %d bytes)",
            bytesRemaining, remainingBytesRead));
  }

  return bserBuffer;
}