Java Code Examples for com.google.common.io.ByteStreams#read()

private static Slice decompressGzip(Slice input, int uncompressedSize)
        throws IOException
{
    if (uncompressedSize == 0) {
        return EMPTY_SLICE;
    }

    try (GZIPInputStream gzipInputStream = new GZIPInputStream(input.getInput(), min(GZIP_BUFFER_SIZE, input.length()))) {
        byte[] buffer = new byte[uncompressedSize];
        int bytesRead = ByteStreams.read(gzipInputStream, buffer, 0, buffer.length);
        if (bytesRead != uncompressedSize) {
            throw new IllegalArgumentException(format("Invalid uncompressedSize for GZIP input. Expected %s, actual: %s", uncompressedSize, bytesRead));
        }
        // Verify we're at EOF and aren't truncating the input
        checkArgument(gzipInputStream.read() == -1, "Invalid uncompressedSize for GZIP input. Actual size exceeds %s bytes", uncompressedSize);
        return wrappedBuffer(buffer, 0, bytesRead);
    }
}
 

List<byte[]> fragmentInternal(InputStream is, int fragmentSizeB, int maxSizeB) throws IOException {
  if (fragmentSizeB != maxSizeB) {
    throw new IOException(Utils.format(
        "Invalid configuration, fragmentSize '{}' and maxSize '{}' should be the same", fragmentSizeB, maxSizeB));
  }
  byte[] buffer = new byte[fragmentSizeB];
  int read = ByteStreams.read(is, buffer, 0, fragmentSizeB);
  if (is.read() > -1) {
    throw new IOException(Utils.format("Maximum data size '{}' exceeded", maxSizeB));
  }
  byte[] data = buffer;
  if (read < buffer.length) {
    data = new byte[read];
    System.arraycopy(buffer, 0, data, 0, read);
  }
  return ImmutableList.of(data);
}
 

/**
 * This is a copy of ByteStreams.readFully(), with the slight change that it throws
 * NoBytesReadException if zero bytes are read. Otherwise it is identical.
 *
 * @param in
 * @param bytes
 * @throws IOException
 */
private void readFully(InputStream in, byte[] bytes) throws IOException {
    int read = ByteStreams.read(in, bytes, 0, bytes.length);
    if (read == 0) {
        throw new NoBytesReadException();
    } else if (read != bytes.length) {
        throw new EOFException("reached end of stream after reading "
                + read + " bytes; " + bytes.length + " bytes expected");
    }
}
 

private StorageSummary putObject(Table table, String blobId, InputSupplier<? extends InputStream> in, Map<String, String> attributes) throws IOException {
    long timestamp = _storageProvider.getCurrentTimestamp(table);
    int chunkSize = _storageProvider.getDefaultChunkSize();
    checkArgument(chunkSize > 0);
    DigestInputStream md5In = new DigestInputStream(in.getInput(), getMessageDigest("MD5"));
    DigestInputStream sha1In = new DigestInputStream(md5In, getMessageDigest("SHA-1"));

    // A more aggressive solution like the Astyanax ObjectWriter recipe would improve performance by pipelining
    // reading the input stream and writing chunks, and issuing the writes in parallel.
    byte[] bytes = new byte[chunkSize];
    long length = 0;
    int chunkCount = 0;
    for (; ; ) {
        int chunkLength;
        try {
            chunkLength = ByteStreams.read(sha1In, bytes, 0, bytes.length);
        } catch (IOException e) {
            LOGGER.error("Failed to read input stream", e);
            throw Throwables.propagate(e);
        }
        if (chunkLength == 0) {
            break;
        }
        ByteBuffer buffer = ByteBuffer.wrap(bytes, 0, chunkLength);
        _storageProvider.writeChunk(table, blobId, chunkCount, buffer, timestamp);
        length += chunkLength;
        chunkCount++;
    }

    // Include two types of hash: md5 (because it's common) and sha1 (because it's secure)
    String md5 = Hex.encodeHexString(md5In.getMessageDigest().digest());
    String sha1 = Hex.encodeHexString(sha1In.getMessageDigest().digest());

    return new StorageSummary(length, chunkCount, chunkSize, md5, sha1, attributes, timestamp);
}
 

@Override
public int read(long position, byte[] buffer, int offset, int length)
        throws IOException {
    try (InputStream in = _stashReader.getRawSplitPart(_stashSplit, Range.closedOpen(position, position + length))) {
        return ByteStreams.read(in, buffer, offset, length);
    }
}
 

private void perform ( final Path file, final Map<String, String> metadata ) throws IOException
{
    try ( final GZIPInputStream gis = new GZIPInputStream ( new FileInputStream ( file.toFile () ) );
          final TarArchiveInputStream tis = new TarArchiveInputStream ( gis ) )
    {
        TarArchiveEntry entry;
        while ( ( entry = tis.getNextTarEntry () ) != null )
        {
            if ( entry.getName ().equals ( "package/package.json" ) )
            {
                final byte[] data = new byte[(int)entry.getSize ()];
                ByteStreams.read ( tis, data, 0, data.length );

                final String str = StandardCharsets.UTF_8.decode ( ByteBuffer.wrap ( data ) ).toString ();

                try
                {
                    // test parse
                    new JsonParser ().parse ( str );
                    // store
                    metadata.put ( "package.json", str );
                }
                catch ( final JsonParseException e )
                {
                    // ignore
                }

                break; // stop parsing the archive
            }
        }

    }
}
 

/** */
private void readHeader(InputStream in) {
    byte[] fourBytes = new byte[4];
    try {
        int nRead = ByteStreams.read(in, fourBytes, 0, 4);
        if (nRead == 0) {
            // ByteStreams.read attempts to fully read 4 bytes and returns the number of bytes read,
            // which is only less than 4 if upstream.read() returned a negative value (EOF).
            LOG.debug("Hit end of file, no more blocks to read.");
            nBytes = 0;
            entityType = VexFormat.VEX_NONE; // indicates no more blocks
            return;
        }
        String s = new String(fourBytes);
        if (s.equals("VEXN")) {
            entityType = VexFormat.VEX_NODE;
        } else if (s.equals("VEXW")) {
            entityType = VexFormat.VEX_WAY;
        } else if (s.equals("VEXR")) {
            entityType = VexFormat.VEX_RELATION;
        } else {
            LOG.error("Unrecognized block type '{}', aborting VEX read.", entityType);
            throw new RuntimeException("Uncrecoginzed VEX block type.");
        }
        ByteStreams.readFully(in, fourBytes);
        nEntities = Ints.fromByteArray(fourBytes);
        ByteStreams.readFully(in, fourBytes);
        nBytes = Ints.fromByteArray(fourBytes);
        if (nBytes < 0 || nBytes > BUFFER_SIZE) {
            throw new RuntimeException("Block has impossible compressed data size, it is probably corrupted.");
        }
        if (nEntities < 0 || nEntities > BUFFER_SIZE) {
            throw new RuntimeException("Block contains impossible number of entities, it is probably corrupted.");
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

public LogMessage nextMessage() throws IOException {
  stream.mark(HEADER_SIZE);

  // Read header
  final byte[] headerBytes = new byte[HEADER_SIZE];
  final int n = ByteStreams.read(stream, headerBytes, 0, HEADER_SIZE);
  if (n == 0) {
    return null;
  }
  final ByteBuffer header = ByteBuffer.wrap(headerBytes);
  int streamId = header.get();
  final int idZ = header.getInt(0);

  // Read frame
  final byte[] frame;
  // Header format is : {STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4}
  if (idZ == 0 || idZ == 0x01000000 || idZ == 0x02000000) {
    header.position(FRAME_SIZE_OFFSET);
    final int frameSize = header.getInt();
    frame = new byte[frameSize];
  } else {
    stream.reset();
    streamId = Stream.STDOUT.id();
    frame = new byte[stream.available()];
  }
  ByteStreams.readFully(stream, frame);
  return new LogMessage(streamId, ByteBuffer.wrap(frame));
}
 

/**
 * This is a copy of ByteStreams.readFully(), with the slight change that it throws
 * NoBytesReadException if zero bytes are read. Otherwise it is identical.
 *
 * @param in
 * @param bytes
 * @throws IOException
 */
private void readFully(InputStream in, byte[] bytes) throws IOException {
    int read = ByteStreams.read(in, bytes, 0, bytes.length);
    if (read == 0) {
        throw new NoBytesReadException();
    } else if (read != bytes.length) {
        throw new EOFException("reached end of stream after reading "
                + read + " bytes; " + bytes.length + " bytes expected");
    }
}
 

/**
 * Reads at most {@code limit} bytes from {@code inputFile} and returns it as a byte array.
 *
 * @throws IOException if there was an error.
 */
public static byte[] readContentWithLimit(Path inputFile, int limit) throws IOException {
  Preconditions.checkArgument(limit >= 0, "limit needs to be >=0, but it is %s", limit);
  ByteSource byteSource = asByteSource(inputFile);
  byte[] buffer = new byte[limit];
  try (InputStream inputStream = byteSource.openBufferedStream()) {
    int read = ByteStreams.read(inputStream, buffer, 0, limit);
    return read == limit ? buffer : Arrays.copyOf(buffer, read);
  }
}
 

BomReader(InputStream inputStream) throws IOException {
  super(inputStream);

  Charset encoding;
  byte[] bom = new byte[MAX_BOM_SIZE];

  // read first 3 bytes such that they can be pushed back later
  PushbackInputStream pushbackStream = new PushbackInputStream(inputStream, MAX_BOM_SIZE);
  int bytesRead = ByteStreams.read(pushbackStream, bom, 0, 3);

  // look for BOM and adapt, defauling to UTF-8
  if (bytesRead >= 3 && bom[0] == X_EF && bom[1] == X_BB && bom[2] == X_BF) {
    encoding = StandardCharsets.UTF_8;
    pushbackStream.unread(bom, 3, (bytesRead - 3));

  } else if (bytesRead >= 2 && bom[0] == X_FE && bom[1] == X_FF) {
    encoding = StandardCharsets.UTF_16BE;
    pushbackStream.unread(bom, 2, (bytesRead - 2));

  } else if (bytesRead >= 2 && bom[0] == X_FF && bom[1] == X_FE) {
    encoding = StandardCharsets.UTF_16LE;
    pushbackStream.unread(bom, 2, (bytesRead - 2));

  } else {
    encoding = StandardCharsets.UTF_8;
    pushbackStream.unread(bom, 0, bytesRead);
  }

  // use Java standard code now we know the encoding
  this.underlying = new InputStreamReader(pushbackStream, encoding);
}
 

public static void assertContentsEqual(Path one, Path two) throws IOException {
  Preconditions.checkNotNull(one);
  Preconditions.checkNotNull(two);

  if (one.equals(two)) {
    return;
  }

  if (Files.size(one) != Files.size(two)) {
    fail(
        String.format(
            "File sizes differ: %s (%d bytes), %s (%d bytes)",
            one, Files.size(one), two, Files.size(two)));
  }

  try (InputStream ois = Files.newInputStream(one);
      InputStream tis = Files.newInputStream(two)) {
    byte[] bo = new byte[BUFFER_SIZE];
    byte[] bt = new byte[BUFFER_SIZE];

    while (true) {
      int read1 = ByteStreams.read(ois, bo, 0, BUFFER_SIZE);
      int read2 = ByteStreams.read(tis, bt, 0, BUFFER_SIZE);
      if (read1 != read2 || !Arrays.equals(bo, bt)) {
        fail(String.format("Contents of files differ: %s, %s", one, two));
      } else if (read1 != BUFFER_SIZE) {
        return;
      }
    }
  }
}
 

private ByteBuffer readBserBuffer(InputStream inputStream) throws IOException {
  ByteBuffer sniffBuffer = ByteBuffer.allocate(SNIFF_BUFFER_SIZE).order(ByteOrder.nativeOrder());
  Preconditions.checkState(sniffBuffer.hasArray());

  int sniffBytesRead = ByteStreams.read(inputStream, sniffBuffer.array(), 0, INITIAL_SNIFF_LEN);
  if (sniffBytesRead < INITIAL_SNIFF_LEN) {
    throw new BserEofException(
        String.format(
            "Invalid BSER header (expected %d bytes, got %d bytes)",
            INITIAL_SNIFF_LEN, sniffBytesRead));
  }

  if (sniffBuffer.get() != 0x00 || sniffBuffer.get() != 0x01) {
    throw new IOException("Invalid BSER header");
  }

  byte lengthType = sniffBuffer.get();
  int lengthBytesRemaining;
  switch (lengthType) {
    case BSER_INT8:
      lengthBytesRemaining = 1;
      break;
    case BSER_INT16:
      lengthBytesRemaining = 2;
      break;
    case BSER_INT32:
      lengthBytesRemaining = 4;
      break;
    case BSER_INT64:
      lengthBytesRemaining = 8;
      break;
    default:
      throw new IOException(String.format("Unrecognized BSER header length type %d", lengthType));
  }
  int lengthBytesRead =
      ByteStreams.read(
          inputStream, sniffBuffer.array(), sniffBuffer.position(), lengthBytesRemaining);
  if (lengthBytesRead < lengthBytesRemaining) {
    throw new BserEofException(
        String.format(
            "Invalid BSER header length (expected %d bytes, got %d bytes)",
            lengthBytesRemaining, lengthBytesRead));
  }
  int bytesRemaining = deserializeIntLen(sniffBuffer, lengthType);

  ByteBuffer bserBuffer = ByteBuffer.allocate(bytesRemaining).order(ByteOrder.nativeOrder());
  Preconditions.checkState(bserBuffer.hasArray());

  int remainingBytesRead = ByteStreams.read(inputStream, bserBuffer.array(), 0, bytesRemaining);

  if (remainingBytesRead < bytesRemaining) {
    throw new IOException(
        String.format(
            "Invalid BSER header (expected %d bytes, got %d bytes)",
            bytesRemaining, remainingBytesRead));
  }

  return bserBuffer;
}