java.util.zip.CheckedInputStream Java Examples

private SFVInfo getSFVFile(Slave slave, String path) throws IOException {
    BufferedReader reader = null;
    CRC32 checksum;
    try {
        File file = slave.getRoots().getFile(path);
        checksum = new CRC32();
        reader = new BufferedReader(new InputStreamReader(new CheckedInputStream(new FileInputStream(file), checksum)));
        SFVInfo sfvInfo = SFVInfo.importSFVInfoFromFile(reader);
        sfvInfo.setSFVFileName(file.getName());
        sfvInfo.setChecksum(checksum.getValue());
        return sfvInfo;
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
}
 

@Override
public Long getFileChecksum(File file) {
    long checksum = -1;
    byte[] buffer = new byte[512];
    Checksum check = new Adler32();

    try {
        FileInputStream fis = new FileInputStream(file);
        CheckedInputStream cis = new CheckedInputStream(fis, check);
        
        int read = cis.read(buffer);
        while(read != -1){
            read = cis.read(buffer);
        }

        checksum = check.getValue();
    } catch (IOException ex) {
        logger.error("Error getting file checksum: "+ex);
    }
    return checksum;
}
 

@Override
public Long getFileChecksum(File file) {
    
    long checksum = -1;
    byte[] buffer = new byte[512];

    try {
        FileInputStream fis = new FileInputStream(file);
        CheckedInputStream cis = new CheckedInputStream(fis, check);
        
        int read = cis.read(buffer);
        while(read != -1){
            read = cis.read(buffer);
        }
        
        fis.close();
        cis.close();

        checksum = check.getValue();
    } catch (IOException ex) {
        logger.error("Error getting file checksum: "+ex);
    }
    return checksum;
}
 

private long calculateChecksum(URL layer) {
    if (layer == null) {
        return -1;
    }
    try {
        InputStream is = layer.openStream();
        try {
            CheckedInputStream cis = new CheckedInputStream(is, new CRC32());
            // Compute the CRC32 checksum
            byte[] buf = new byte[1024];
            while (cis.read(buf) >= 0) {
            }
            cis.close();
            return cis.getChecksum().getValue();
        } finally {
            is.close();
        }
    } catch (IOException e) {
        return -1;
    }
}
 

/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
  this.logVersion = logVersion;
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = DataChecksum.newCrc32();
  } else {
    this.checksum = null;
  }
  // It is possible that the logVersion is actually a future layoutversion
  // during the rolling upgrade (e.g., the NN gets upgraded first). We
  // assume future layout will also support length of editlog op.
  this.supportEditLogLength = NameNodeLayoutVersion.supports(
      NameNodeLayoutVersion.Feature.EDITLOG_LENGTH, logVersion)
      || logVersion < NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
  this.limiter = limiter;
  this.cache = new OpInstanceCache();
  this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT;
}
 

private static long getApkFileChecksum(Context context) {
    String apkPath = context.getPackageCodePath();
    Long chksum = null;
    try {
        // Open the file and build a CRC32 checksum.
        FileInputStream fis = new FileInputStream(new File(apkPath));
        CRC32 chk = new CRC32();
        CheckedInputStream cis = new CheckedInputStream(fis, chk);
        byte[] buff = new byte[80];
        while (cis.read(buff) >= 0) ;
        chksum = chk.getValue();
    } catch (Exception e) {
        e.printStackTrace();
    }
    return chksum;
}
 

public static void decompress(final String sourceFile, final String outputDir, final Checksum checksum)
                                                                                                       throws IOException {
    try (final FileInputStream fis = new FileInputStream(sourceFile);
            final CheckedInputStream cis = new CheckedInputStream(fis, checksum);
            final ZipInputStream zis = new ZipInputStream(new BufferedInputStream(cis))) {
        ZipEntry entry;
        while ((entry = zis.getNextEntry()) != null) {
            final String fileName = entry.getName();
            final File entryFile = new File(Paths.get(outputDir, fileName).toString());
            FileUtils.forceMkdir(entryFile.getParentFile());
            try (final FileOutputStream fos = new FileOutputStream(entryFile);
                    final BufferedOutputStream bos = new BufferedOutputStream(fos)) {
                IOUtils.copy(zis, bos);
                bos.flush();
                fos.getFD().sync();
            }
        }
        // Continue to read all remaining bytes(extra metadata of ZipEntry) directly from the checked stream,
        // Otherwise, the checksum value maybe unexpected.
        //
        // See https://coderanch.com/t/279175/java/ZipInputStream
        IOUtils.copy(cis, NullOutputStream.NULL_OUTPUT_STREAM);
    }
}
 

@Override
protected void parse(InputStream is) throws IOException {
	// don't close CheckedInputStream, as it will close the underlying inputstream
	try (UncloseableInputStream uis = new UncloseableInputStream(is);
			CheckedInputStream cis = new CheckedInputStream(uis, new CRC16())) {

		checkControl(cis, HDTPart.Type.TRIPLES);
		checkFormat(cis, FORMAT_BITMAP);

		properties = getProperties(cis);

		int i = getIntegerProperty(properties, ORDER, "order");
		if (i != HDTTriples.Order.SPO.getValue()) {
			throw new UnsupportedOperationException(
					"Triples section: order " + Integer.toString(i) + ", but only SPO order is supported");
		}

		checkCRC(cis, is, 2);
	}
}
 

@Override
protected void parse(InputStream is) throws IOException {
	super.parse(is);

	// don't close CheckedInputStream, as it will close the underlying inputstream
	try (UncloseableInputStream uis = new UncloseableInputStream(is);
			CheckedInputStream cis = new CheckedInputStream(uis, new CRC32())) {
		// read bytes, minimum 1
		long bytes = (nrbits * entries + 7) / 8;
		if (bytes > Integer.MAX_VALUE) {
			throw new UnsupportedOperationException("Maximum number of bytes in array exceeded: " + bytes);
		}

		buffer = new byte[(int) bytes];
		cis.read(buffer);

		checkCRC(cis, is, 4);
	}
}
 

/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
@SuppressWarnings("deprecation")
public Reader(DataInputStream in, int logVersion) {
  this.logVersion = logVersion;
  if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = FSEditLog.getChecksumForRead();
  } else {
    this.checksum = null;
  }

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
}
 

@Override
protected void parse(InputStream is) throws IOException {
	CRC8 crc8 = new CRC8();
	crc8.update(getType());

	// don't close CheckedInputStream, as it will close the underlying inputstream
	try (UncloseableInputStream uis = new UncloseableInputStream(is);
			CheckedInputStream cis = new CheckedInputStream(uis, crc8)) {

		nrbits = cis.read();
		long l = VByte.decode(cis);
		if (l > Integer.MAX_VALUE) {
			throw new UnsupportedOperationException("Maximum number of bytes in array exceeded: " + l);
		}
		entries = (int) l;

		checkCRC(cis, is, 1);
	}
}
 

@Test
public void testChecksumStreams() throws IOException {
    byte[] content = new byte[33333];
    new Random().nextBytes(content);

    Murmur3F murmur3F = new Murmur3F();
    murmur3F.update(content);
    String hash = murmur3F.getValueHexString();

    murmur3F.reset();
    CheckedOutputStream out = new CheckedOutputStream(new ByteArrayOutputStream(), murmur3F);
    out.write(content);
    Assert.assertEquals(hash, murmur3F.getValueHexString());

    murmur3F.reset();
    CheckedInputStream in = new CheckedInputStream(new ByteArrayInputStream(content), murmur3F);
    IoUtils.readAllBytes(in);
    Assert.assertEquals(hash, murmur3F.getValueHexString());
}
 

/**
 * Compare the calculated checksum to the expected one.
 *
 * @param cis checked input stream
 * @param is  (unchecked) input stream
 * @param len number of bytes of the checksum
 * @throws IOException
 */
protected static void checkCRC(CheckedInputStream cis, InputStream is, int len) throws IOException {
	long calc = cis.getChecksum().getValue();

	byte[] checksum = new byte[len];
	is.read(checksum);

	long expect = 0L;
	// little-endian to big-endian, e.g. HDT-It stores checksum 7635 as 0x35 0x76 (at least on x86)
	for (int i = len - 1; i >= 0; i--) {
		expect <<= 8;
		expect |= checksum[i] & 0xFF;
	}

	if (calc != expect) {
		throw new IOException("CRC does not match: calculated " +
				Long.toHexString(calc) + " instead of " + Long.toHexString(expect));
	}
}
 

private Long getCRC(URL zipUrl) {
    Long result = -1l;
    try {
        CRC32 crc = new CRC32();
        CheckedInputStream cis = new CheckedInputStream(zipUrl.openStream(), crc);

        byte[] buffer = new byte[1024];
        int length;

        //read the entry from zip file and extract it to disk
        while( (length = cis.read(buffer)) > 0);

        cis.close();

        result = crc.getValue();
    } catch (IOException e) {
        LOG.warn("Unable to calculate CRC, resource doesn't exist?", e);
    }
    return result;
}
 

public void test_read$byteII() throws Exception {
    // testing that the return by skip is valid
    InputStream checkInput = Support_Resources
            .getStream("hyts_checkInput.txt");
    CheckedInputStream checkIn = new CheckedInputStream(checkInput,
            new CRC32());
    byte buff[] = new byte[50];
    checkIn.read(buff, 10, 5);
    checkIn.close();
    try {
        checkIn.read(buff, 10, 5);
        fail("IOException expected.");
    } catch (IOException ee) {
        // expected
    }
    checkInput.close();
}
 

public void test_read() throws Exception {
    // testing that the return by skip is valid
    InputStream checkInput = Support_Resources
            .getStream("hyts_checkInput.txt");
    CheckedInputStream checkIn = new CheckedInputStream(checkInput,
            new CRC32());
    checkIn.read();
    checkIn.close();
    try {
        checkIn.read();
        fail("IOException expected.");
    } catch (IOException ee) {
        // expected
    }
    checkInput.close();
}
 

/**
 * java.util.zip.CheckedInputStream#skip(long)
 */
public void test_skipJ() throws Exception {
    // testing that the return by skip is valid
    InputStream checkInput = Support_Resources.getStream("hyts_checkInput.txt");
    CheckedInputStream checkIn = new CheckedInputStream(checkInput, new CRC32());
    long skipValue = 5;
    assertEquals("the value returned by skip(n) is not the same as its parameter",
            skipValue, checkIn.skip(skipValue));
    checkIn.skip(skipValue);
    // ran JDK and found the checkSum value is 2235765342
    // System.out.print(checkIn.getChecksum().getValue());
    assertEquals("checkSum value is not correct", 2235765342L, checkIn.getChecksum()
            .getValue());
    assertEquals(0, checkIn.skip(0));
    checkInput.close();
}
 

/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) {
  this.logVersion = logVersion;
  if (NameNodeLayoutVersion.supports(
      LayoutVersion.Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = DataChecksum.newCrc32();
  } else {
    this.checksum = null;
  }
  // It is possible that the logVersion is actually a future layoutversion
  // during the rolling upgrade (e.g., the NN gets upgraded first). We
  // assume future layout will also support length of editlog op.
  this.supportEditLogLength = NameNodeLayoutVersion.supports(
      NameNodeLayoutVersion.Feature.EDITLOG_LENGTH, logVersion)
      || logVersion < NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
  this.limiter = limiter;
  this.cache = new OpInstanceCache();
  this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT;
}
 

public TezSpillRecord(Path indexFileName, Configuration job, Checksum crc,
                   String expectedIndexOwner)
    throws IOException {

  final FileSystem rfs = FileSystem.getLocal(job).getRaw();
  final FSDataInputStream in = rfs.open(indexFileName);
  try {
    final long length = rfs.getFileStatus(indexFileName).getLen();
    final int partitions = 
        (int) length / Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH;
    final int size = partitions * Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH;

    buf = ByteBuffer.allocate(size);
    if (crc != null) {
      crc.reset();
      CheckedInputStream chk = new CheckedInputStream(in, crc);
      IOUtils.readFully(chk, buf.array(), 0, size);
      if (chk.getChecksum().getValue() != in.readLong()) {
        throw new ChecksumException("Checksum error reading spill index: " +
                              indexFileName, -1);
      }
    } else {
      IOUtils.readFully(in, buf.array(), 0, size);
    }
    entries = buf.asLongBuffer();
  } finally {
    in.close();
  }
}
 

/** internally formats the given file and computes their checksum
 * 
 * @param file the file
 * @param original checksum of the original file
 * @param formatted checksum of the formatted file
 * @return the formatted bytes
 * @throws IOException if an error occurs */
private byte[] format(File file, Checksum original, Checksum formatted) throws IOException {
    try (InputStream in = new CheckedInputStream(new BufferedInputStream(new FileInputStream(file)), original)) {
        @SuppressWarnings("resource")
        ByteArrayOutputStream buffer = formattingBuffer != null ? formattingBuffer.get() : null;
        if (buffer == null) {
            buffer = new ByteArrayOutputStream();
            formattingBuffer = new WeakReference<>(buffer);
        } else {
            buffer.reset();
        }

        try (OutputStream out = new CheckedOutputStream(buffer, formatted);
             FormattingXmlStreamWriter writer = FormattingXmlStreamWriter.create(out, format)) {
            // cannot use XMlStreamReader due to comment handling:
            // https://stackoverflow.com/questions/15792007/why-does-xmlstreamreader-staxsource-strip-comments-from-xml
            TransformerFactory tf = TransformerFactory.newInstance();
            tf.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true);
            SAXSource saxSource = new SAXSource(new InputSource(in));
            SAXParserFactory sf = SAXParserFactory.newInstance();
            sf.setNamespaceAware(true);
            sf.setFeature("http://xml.org/sax/features/namespace-prefixes", true);
            sf.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, true);
            saxSource.setXMLReader(new NormalizingSaxFilter(sf.newSAXParser().getXMLReader()));
            Transformer t = tf.newTransformer();
            StAXResult result = new StAXResult(writer);
            t.transform(saxSource, result);
        }
        return buffer.toByteArray();
    } catch (TransformerException | XMLStreamException | FactoryConfigurationError | ParserConfigurationException | SAXException ex) {
        throw new IOException(ex);
    }
}
 

static void createTestFile() throws Throwable {

		/**
		 * Generate a temporary file for uploading/downloading
		 */
		sourceFile = new File(System.getProperty("user.home"), "sftp-file");
		java.util.Random rnd = new java.util.Random();

		FileOutputStream out = new FileOutputStream(sourceFile);
		byte[] buf = new byte[1024000];
		for (int i = 0; i < 100; i++) {
			rnd.nextBytes(buf);
			out.write(buf);
		}
		out.close();

		CheckedInputStream cis = new CheckedInputStream(new FileInputStream(
				sourceFile), new Adler32());

		try {
			byte[] tempBuf = new byte[16384];
			while (cis.read(tempBuf) >= 0) {
			}
			sourceFileChecksum = cis.getChecksum().getValue();
		} catch (IOException e) {
		} finally {
			cis.close();
		}
	}
 

public ChunkEnumeration(File file) throws FileNotFoundException {
    this.check = new Adler32();

    this.fis = new FileInputStream(file);
    this.cis = new CheckedInputStream(fis, check);
    this.closed = false;
    this.number = 0;
}
 

public static long getFileCRCCode(File file) throws Exception {

        FileInputStream fileinputstream = new FileInputStream(file);
        CRC32 crc32 = new CRC32();
        for (CheckedInputStream checkedinputstream =
             new CheckedInputStream(fileinputstream, crc32);
             checkedinputstream.read() != -1;
                ) {
        }
        return crc32.getValue();


    }
 

private void readHeader(InputStream inStream, RpcCRC32Checksum crc) throws IOException {
	CheckedInputStream in = new CheckedInputStream(inStream, crc);
	crc.reset();
	// Check header magic
	if (readUShort(in) != GZIP_MAGIC) {
	    throw new IOException("Not in GZIP format");
	}
	// Check compression method
	if (readUByte(in) != 8) {
	    throw new IOException("Unsupported compression method");
	}
	// Read flags
	int flg = readUByte(in);
	// Skip MTIME, XFL, and OS fields
	skipBytes(in, 6);
	// Skip optional extra field
	if ((flg & FEXTRA) == FEXTRA) {
	    skipBytes(in, readUShort(in));
	}
	// Skip optional file name
	if ((flg & FNAME) == FNAME) {
	    while (readUByte(in) != 0) ;
	}
	// Skip optional file comment
	if ((flg & FCOMMENT) == FCOMMENT) {
	    while (readUByte(in) != 0) ;
	}
	// Check optional header CRC
	if ((flg & FHCRC) == FHCRC) {
	    int v = (int)crc.getValue() & 0xffff;
	    if (readUShort(in) != v) {
		throw new IOException("Corrupt GZIP header");
	    }
	}
}
 

private void readHeader(InputStream inStream, RpcCRC32Checksum crc) throws IOException {
	CheckedInputStream in = new CheckedInputStream(inStream, crc);
	crc.reset();
	// Check header magic
	if (readUShort(in) != GZIP_MAGIC) {
		throw new IOException("Not in GZIP format");
	}
	// Check compression method
	if (readUByte(in) != 8) {
		throw new IOException("Unsupported compression method");
	}
	// Read flags
	int flg = readUByte(in);
	// Skip MTIME, XFL, and OS fields
	skipBytes(in, 6);
	// Skip optional extra field
	if ((flg & FEXTRA) == FEXTRA) {
		skipBytes(in, readUShort(in));
	}
	// Skip optional file name
	if ((flg & FNAME) == FNAME) {
		while (readUByte(in) != 0) ;
	}
	// Skip optional file comment
	if ((flg & FCOMMENT) == FCOMMENT) {
		while (readUByte(in) != 0) ;
	}
	// Check optional header CRC
	if ((flg & FHCRC) == FHCRC) {
		int v = (int) crc.getValue() & 0xffff;
		if (readUShort(in) != v) {
			throw new IOException("Corrupt GZIP header");
		}
	}
}
 

private void readHeader(InputStream inStream, RpcCRC32Checksum crc) throws IOException {
	CheckedInputStream in = new CheckedInputStream(inStream, crc);
	crc.reset();
	// Check header magic
	if (readUShort(in) != GZIP_MAGIC) {
		throw new IOException("Not in GZIP format");
	}
	// Check compression method
	if (readUByte(in) != 8) {
		throw new IOException("Unsupported compression method");
	}
	// Read flags
	int flg = readUByte(in);
	// Skip MTIME, XFL, and OS fields
	skipBytes(in, 6);
	// Skip optional extra field
	if ((flg & FEXTRA) == FEXTRA) {
		skipBytes(in, readUShort(in));
	}
	// Skip optional file name
	if ((flg & FNAME) == FNAME) {
		while (readUByte(in) != 0) ;
	}
	// Skip optional file comment
	if ((flg & FCOMMENT) == FCOMMENT) {
		while (readUByte(in) != 0) ;
	}
	// Check optional header CRC
	if ((flg & FHCRC) == FHCRC) {
		int v = (int) crc.getValue() & 0xffff;
		if (readUShort(in) != v) {
			throw new IOException("Corrupt GZIP header");
		}
	}
}
 

public SpillRecord(Path indexFileName, JobConf job, Checksum crc)
    throws IOException {

  final FileSystem rfs = FileSystem.getLocal(job).getRaw();
  final FSDataInputStream in = rfs.open(indexFileName);
  try {
    final long length = rfs.getFileStatus(indexFileName).getLen();
    final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
    final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;

    buf = ByteBuffer.allocate(size);
    if (crc != null) {
      crc.reset();
      CheckedInputStream chk = new CheckedInputStream(in, crc);
      IOUtils.readFully(chk, buf.array(), 0, size);
      if (chk.getChecksum().getValue() != in.readLong()) {
        throw new ChecksumException("Checksum error reading spill index: " +
                              indexFileName, -1);
      }
    } else {
      IOUtils.readFully(in, buf.array(), 0, size);
    }
    entries = buf.asLongBuffer();
  } finally {
    in.close();
  }
}
 

private void readHeader(InputStream inStream, RpcCRC32Checksum crc) throws IOException {
	CheckedInputStream in = new CheckedInputStream(inStream, crc);
	crc.reset();
	// Check header magic
	if (readUShort(in) != GZIP_MAGIC) {
		throw new IOException("Not in GZIP format");
	}
	// Check compression method
	if (readUByte(in) != 8) {
		throw new IOException("Unsupported compression method");
	}
	// Read flags
	int flg = readUByte(in);
	// Skip MTIME, XFL, and OS fields
	skipBytes(in, 6);
	// Skip optional extra field
	if ((flg & FEXTRA) == FEXTRA) {
		skipBytes(in, readUShort(in));
	}
	// Skip optional file name
	if ((flg & FNAME) == FNAME) {
		while (readUByte(in) != 0) ;
	}
	// Skip optional file comment
	if ((flg & FCOMMENT) == FCOMMENT) {
		while (readUByte(in) != 0) ;
	}
	// Check optional header CRC
	if ((flg & FHCRC) == FHCRC) {
		int v = (int) crc.getValue() & 0xffff;
		if (readUShort(in) != v) {
			throw new IOException("Corrupt GZIP header");
		}
	}
}
 

private void readHeader(InputStream inStream, RpcCRC32Checksum crc) throws IOException {
	CheckedInputStream in = new CheckedInputStream(inStream, crc);
	crc.reset();
	// Check header magic
	if (readUShort(in) != GZIP_MAGIC) {
		throw new IOException("Not in GZIP format");
	}
	// Check compression method
	if (readUByte(in) != 8) {
		throw new IOException("Unsupported compression method");
	}
	// Read flags
	int flg = readUByte(in);
	// Skip MTIME, XFL, and OS fields
	skipBytes(in, 6);
	// Skip optional extra field
	if ((flg & FEXTRA) == FEXTRA) {
		skipBytes(in, readUShort(in));
	}
	// Skip optional file name
	if ((flg & FNAME) == FNAME) {
		while (readUByte(in) != 0) ;
	}
	// Skip optional file comment
	if ((flg & FCOMMENT) == FCOMMENT) {
		while (readUByte(in) != 0) ;
	}
	// Check optional header CRC
	if ((flg & FHCRC) == FHCRC) {
		int v = (int) crc.getValue() & 0xffff;
		if (readUShort(in) != v) {
			throw new IOException("Corrupt GZIP header");
		}
	}
}
 

public long recalculateSchemaChecksum() {
  if (getMondrianSchemaFilename() != null) {
    try {
      CheckedInputStream cis = new CheckedInputStream(new FileInputStream(getMondrianSchemaFilename()), new CRC32());
      byte[] buf = new byte[1024];
      while(cis.read(buf) >= 0) {
      }
      return cis.getChecksum().getValue();
    } catch (IOException e) {
        e.printStackTrace();
    }
  }
  return -1;
}