java.util.zip.Checksum Java Examples

private static void doBench(final List<Class<? extends Checksum>> crcs,
    final PrintStream out) throws Exception {
  final byte[] bytes = new byte[MAX_LEN];
  new Random().nextBytes(bytes);

  // Print header
  out.printf("\nPerformance Table (The unit is MB/sec; #T = #Theads)\n");

  // Warm up implementations to get jit going.
  for (Class<? extends Checksum> c : crcs) {
    doBench(c, 1, bytes, 2);
    doBench(c, 1, bytes, 2101);
  }

  // Test on a variety of sizes with different number of threads
  for (int size = 32; size <= MAX_LEN; size <<= 1) {
    doBench(crcs, bytes, size, out);
  }
}
 

@Test(enabled = true)
public void testWriteChunkData() throws Exception {
  NullWritable nada = NullWritable.get();
  MneDurableOutputSession<DurableChunk<?>> sess =
          new MneDurableOutputSession<DurableChunk<?>>(m_tacontext, null,
                  MneConfigHelper.DEFAULT_OUTPUT_CONFIG_PREFIX);
  MneDurableOutputValue<DurableChunk<?>> mdvalue =
          new MneDurableOutputValue<DurableChunk<?>>(sess);
  OutputFormat<NullWritable, MneDurableOutputValue<DurableChunk<?>>> outputFormat =
          new MneOutputFormat<MneDurableOutputValue<DurableChunk<?>>>();
  RecordWriter<NullWritable, MneDurableOutputValue<DurableChunk<?>>> writer =
          outputFormat.getRecordWriter(null, m_conf, null, null);
  DurableChunk<?> dchunk = null;
  Checksum cs = new CRC32();
  cs.reset();
  for (int i = 0; i < m_reccnt; ++i) {
    dchunk = genupdDurableChunk(sess, cs);
    Assert.assertNotNull(dchunk);
    writer.write(nada, mdvalue.of(dchunk));
  }
  m_checksum = cs.getValue();
  writer.close(null);
  sess.close();
}
 

/**
 * Set the checksum related parameters
 * @param verifyChecksum whether to verify checksum
 * @param sum which type of checksum to use
 * @param maxChunkSize maximun chunk size
 * @param checksumSize checksum size
 */
final protected synchronized void set(boolean verifyChecksum,
    Checksum sum, int maxChunkSize, int checksumSize) {

  // The code makes assumptions that checksums are always 32-bit.
  assert !verifyChecksum || sum == null || checksumSize == CHECKSUM_SIZE;

  this.maxChunkSize = maxChunkSize;
  this.verifyChecksum = verifyChecksum;
  this.sum = sum;
  this.buf = new byte[maxChunkSize];
  // The size of the checksum array here determines how much we can
  // read in a single call to readChunk
  this.checksum = new byte[CHUNKS_PER_READ * checksumSize];
  this.checksumInts = ByteBuffer.wrap(checksum).asIntBuffer();
  this.count = 0;
  this.pos = 0;
}
 

/**
 * Compute the CRC-32 of the contents of a stream.
 * \r\n and \r are both normalized to \n for purposes of the calculation.
 */
static String computeCrc32(InputStream is) throws IOException {
    Checksum crc = new CRC32();
    int last = -1;
    int curr;
    while ((curr = is.read()) != -1) {
        if (curr != '\n' && last == '\r') {
            crc.update('\n');
        }
        if (curr != '\r') {
            crc.update(curr);
        }
        last = curr;
    }
    if (last == '\r') {
        crc.update('\n');
    }
    int val = (int)crc.getValue();
    String hex = Integer.toHexString(val);
    while (hex.length() < 8) {
        hex = "0" + hex; // NOI18N
    }
    return hex;
}
 

private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}
 

/**
 * Set the checksum related parameters
 * @param verifyChecksum whether to verify checksum
 * @param sum which type of checksum to use
 * @param maxChunkSize maximun chunk size
 * @param checksumSize checksum size
 */
final protected synchronized void set(boolean verifyChecksum,
    Checksum sum, int maxChunkSize, int checksumSize) {

  // The code makes assumptions that checksums are always 32-bit.
  assert !verifyChecksum || sum == null || checksumSize == CHECKSUM_SIZE;

  this.maxChunkSize = maxChunkSize;
  this.verifyChecksum = verifyChecksum;
  this.sum = sum;
  this.buf = new byte[maxChunkSize];
  // The size of the checksum array here determines how much we can
  // read in a single call to readChunk
  this.checksum = new byte[CHUNKS_PER_READ * checksumSize];
  this.checksumInts = ByteBuffer.wrap(checksum).asIntBuffer();
  this.count = 0;
  this.pos = 0;
}
 

protected DurableChunk<?> genupdDurableChunk(
    MneDurableOutputSession<DurableChunk<?>> s, Checksum cs) {
  DurableChunk<?> ret = null;
  int sz = m_rand.nextInt(1024 * 1024) + 1024 * 1024;
  ret = s.newDurableObjectRecord(sz);
  byte b;
  if (null != ret) {
    for (int i = 0; i < ret.getSize(); ++i) {
      b = (byte) m_rand.nextInt(255);
      unsafe.putByte(ret.get() + i, b);
      cs.update(b);
    }
    m_totalsize += sz;
  }
  return ret;
}
 

@Test
  public void testGenChunkBuffers() {
    Checksum cs = new CRC32();
    cs.reset();

    NonVolatileMemAllocator act = new NonVolatileMemAllocator(Utils.getNonVolatileMemoryAllocatorService("pmalloc"),
            1024 * 1024 * 1024L, "./pmchunkbuffertest.dat", true);
    act.setChunkReclaimer(new Reclaim<Long>() {
      @Override
      public boolean reclaim(Long mres, Long sz) {
        System.out.println(String.format("Reclaim Memory Chunk: %X  Size: %s", System.identityHashCode(mres),
                null == sz ? "NULL" : sz.toString()));
        return false;
      }
    });
    DurableChunk<NonVolatileMemAllocator> mch;
    mch = act.createChunk(1000 * 1024 * 1024L);
    Assert.assertNotNull(mch);
    act.setHandler(m_keyid, mch.getHandler());
    long bufcnt = mch.getSize() / m_bufsize;
    ChunkBuffer ckbuf;
    byte[] rdbytes;
    for (long idx = 0; idx < bufcnt; ++idx) {
//      System.err.println(String.format("---- bufcnt: %d, bufsize: %d, idx: %d", bufcnt, m_bufsize, idx));
      ckbuf = mch.getChunkBuffer(idx * m_bufsize, m_bufsize);
      Assert.assertNotNull(ckbuf);
      rdbytes = RandomUtils.nextBytes(m_bufsize);
      Assert.assertNotNull(rdbytes);
      ckbuf.get().clear();
      ckbuf.get().put(rdbytes);
      cs.update(rdbytes, 0, rdbytes.length);
    }
    m_checksum = cs.getValue();
    m_count = bufcnt;
    act.close();
  }
 

/**
 * Validate a transaction's checksum
 */
private void validateChecksum(DataInputStream in,
                              Checksum checksum,
                              long txid)
    throws IOException {
  if (checksum != null) {
    int calculatedChecksum = (int)checksum.getValue();
    int readChecksum = in.readInt(); // read in checksum
    if (readChecksum != calculatedChecksum) {
      throw new ChecksumException(
          "Transaction is corrupt. Calculated checksum is " +
          calculatedChecksum + " but read checksum " + readChecksum, txid);
    }
  }
}
 

protected DurableBuffer<?> genupdDurableBuffer(
    MneDurableOutputSession<DurableBuffer<?>> s, Checksum cs) {
  DurableBuffer<?> ret = null;
  int sz = m_rand.nextInt(1024 * 1024) + 1024 * 1024;
  ret = s.newDurableObjectRecord(sz);
  if (null != ret) {
    ret.get().clear();
    byte[] rdbytes = RandomUtils.nextBytes(sz);
    Assert.assertNotNull(rdbytes);
    ret.get().put(rdbytes);
    cs.update(rdbytes, 0, rdbytes.length);
    m_totalsize += sz;
  }
  return ret;
}
 

/**
 * Create a new {@link OutputStream} with configurable block size. Large
 * blocks require more memory at compression and decompression time but
 * should improve the compression ratio.
 *
 * @param out         the {@link OutputStream} to feed
 * @param blockSize   the maximum number of bytes to try to compress at once,
 *                    must be >= 64 and <= 32 M
 * @param compressor  the {@link LZ4Compressor} instance to use to compress
 *                    data
 * @param checksum    the {@link Checksum} instance to use to check data for
 *                    integrity.
 * @param syncFlush   true if pending data should also be flushed on {@link #flush()}
 */
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor, Checksum checksum, boolean syncFlush) {
  super(out);
  this.blockSize = blockSize;
  this.compressor = compressor;
  this.checksum = checksum;
  this.compressionLevel = compressionLevel(blockSize);
  this.buffer = new byte[blockSize];
  final int compressedBlockSize = HEADER_LENGTH + compressor.maxCompressedLength(blockSize);
  this.compressedBuffer = new byte[compressedBlockSize];
  this.syncFlush = syncFlush;
  o = 0;
  finished = false;
  System.arraycopy(MAGIC, 0, compressedBuffer, 0, MAGIC_LENGTH);
}
 

private static void report(String s, Checksum crc1, Checksum crc2,
        Checksum crc3, Checksum crc4) {
    System.out.println(s + ", crc1 = " + crc1.getValue() +
            ", crc2 = " + crc2.getValue()+
            ", crc3 = " + crc3.getValue()+
            ", crc4 = " + crc4.getValue());
}
 

@Substitute
public Checksum create() {
    try {
        return (Checksum) Class.forName("java.util.zip.CRC32C").getConstructor().newInstance();
    } catch (ReflectiveOperationException e) {
        throw new RuntimeException(e);
    }
}
 

protected DurableBuffer<NonVolatileMemAllocator>
    genuptBuffer(NonVolatileMemAllocator act, Checksum cs, int size) {
  DurableBuffer<NonVolatileMemAllocator> ret = null;
  ret = act.createBuffer(size, false);
  if (null == ret) {
    throw new OutOfHybridMemory("Create Durable Buffer Failed.");
  }
  ret.get().clear();
  byte[] rdbytes = RandomUtils.nextBytes(size);
  Assert.assertNotNull(rdbytes);
  ret.get().put(rdbytes);
  cs.update(rdbytes, 0, rdbytes.length);
  ret.get().clear();
  return ret;
}
 

public static void update(Checksum checksum, ByteBuffer buffer, int offset, int length) {
    if (buffer.hasArray()) {
        checksum.update(buffer.array(), buffer.position() + buffer.arrayOffset() + offset, length);
    } else {
        int start = buffer.position() + offset;
        for (int i = start; i < start + length; i++)
            checksum.update(buffer.get(i));
    }
}
 

/**
 * Convenience method for calculating the CRC2 checksum of a string.
 *
 * @param s the string
 * @return long value of the CRC32
 */
public static long crc32(String s) {
    Checksum checksum = new CRC32();
    byte[] bytes = s.getBytes();
    checksum.update(bytes, 0, bytes.length);
    return checksum.getValue();
}
 

public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 

protected DurableChunk<NonVolatileMemAllocator>
    genuptChunk(NonVolatileMemAllocator act, Checksum cs, long size) {
  DurableChunk<NonVolatileMemAllocator> ret = null;
  ret = act.createChunk(size, false);
  if (null == ret) {
    throw new OutOfHybridMemory("Create Durable Chunk Failed.");
  }
  byte b;
  for (int i = 0; i < ret.getSize(); ++i) {
    b = (byte) rand.nextInt(255);
    unsafe.putByte(ret.get() + i, b);
    cs.update(b);
  }
  return ret;
}
 

public SpillRecord(Path indexFileName, JobConf job, Checksum crc,
                   String expectedIndexOwner)
    throws IOException {

  final FileSystem rfs = FileSystem.getLocal(job).getRaw();
  final FSDataInputStream in =
      SecureIOUtils.openFSDataInputStream(new File(indexFileName.toUri()
          .getRawPath()), expectedIndexOwner, null);
  try {
    final long length = rfs.getFileStatus(indexFileName).getLen();
    final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
    final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;
    buf = ByteBuffer.allocate(size);
    if (crc != null) {
      crc.reset();
      CheckedInputStream chk = new CheckedInputStream(in, crc);
      IOUtils.readFully(chk, buf.array(), 0, size);
      
      if (chk.getChecksum().getValue() != in.readLong()) {
        throw new ChecksumException("Checksum error reading spill index: " +
                              indexFileName, -1);
      }
    } else {
      IOUtils.readFully(in, buf.array(), 0, size);
    }
    entries = buf.asLongBuffer();
  } finally {
    in.close();
  }
}
 

ChecksumTransformer(Checksum checksum, Mode mode, int checksumLengthByte) {
    if (checksumLengthByte <= 0 || checksumLengthByte > 8)
        throw new IllegalArgumentException("checksum length must be between 1 and 8 bytes");

    Objects.requireNonNull(checksum, "checksum instance must not be null");
    this.checksum = checksum;
    this.mode = mode;
    this.checksumLengthByte = checksumLengthByte;
}
 

@Test
public void checksumTest() {
    Checksum crc32Checksum = new CRC32();
    crc32Checksum.update(example2_bytes_seven, 0, example2_bytes_seven.length);
    assertEquals(crc32Checksum.getValue(), Bytes.from(example2_bytes_seven).transform(checksumCrc32()).resize(8).toLong());
    assertEquals(Bytes.from(example2_bytes_seven, Bytes.from(crc32Checksum.getValue()).resize(4).array()), Bytes.from(example2_bytes_seven).transform(checksumAppendCrc32()));

    Checksum adlerChecksum = new Adler32();
    adlerChecksum.update(example2_bytes_seven, 0, example2_bytes_seven.length);
    assertEquals(Bytes.from(adlerChecksum.getValue()).resize(4),
            Bytes.from(example2_bytes_seven).transform(checksum(new Adler32(), ChecksumTransformer.Mode.TRANSFORM, 4)));
}
 

public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 

private static void check(Checksum crc1, Checksum crc2) throws Exception {
    if (crc1.getValue() != crc2.getValue()) {
        String s = "value 1 = " + crc1.getValue() + ", value 2 = " + crc2.getValue();
        System.err.println(s);
        throw new Exception(s);
    }
}
 

protected DurableBuffer<NonVolatileMemAllocator>
genuptBuffer(NonVolatileMemAllocator act, Checksum cs, int size) {
    DurableBuffer<NonVolatileMemAllocator> ret = null;
    ret = act.createBuffer(size, false);
    if (null == ret) {
        throw new OutOfHybridMemory("Create Durable Buffer Failed.");
    }
    ret.get().clear();
    byte[] rdbytes = RandomUtils.nextBytes(size);
    Assert.assertNotNull(rdbytes);
    ret.get().put(rdbytes);
    cs.update(rdbytes, 0, rdbytes.length);
    return ret;
}
 

private static void check(Checksum crc, long crcReference) throws Exception {
    if (crc.getValue() != crcReference) {
        System.err.printf("ERROR: crc = %08x, crcReference = %08x\n",
                          crc.getValue(), crcReference);
        throw new Exception("TestCRC32 Error");
    }
}
 

private static boolean check(Checksum adler0, Checksum adler1) {
    if (adler0.getValue() != adler1.getValue()) {
        System.err.printf("ERROR: adler0 = %08x, adler1 = %08x\n",
                          adler0.getValue(), adler1.getValue());
        return false;
    }
    return true;
}
 

private static void report(String s, Checksum crc1, Checksum crc2,
        Checksum crc3, Checksum crc4) {
    System.out.println(s + ", crc1 = " + crc1.getValue() +
            ", crc2 = " + crc2.getValue()+
            ", crc3 = " + crc3.getValue()+
            ", crc4 = " + crc4.getValue());
}
 

private static void testReadonlyByteBufferOffset(Checksum checksum, long expected) {
    byte[] unaligned_bytes_123456789 = new byte[BYTES_123456789.length + 64];
    for (int i = 0; i < unaligned_bytes_123456789.length - BYTES_123456789.length; i++) {
        checksum.reset();
        System.arraycopy(BYTES_123456789, 0, unaligned_bytes_123456789, i, BYTES_123456789.length);
        ByteBuffer bb = ByteBuffer.wrap(unaligned_bytes_123456789).asReadOnlyBuffer();
        bb.position(i);
        bb.limit(i + BYTES_123456789.length);
        checksum.update(bb);
        checkChecksumOffset(checksum, expected, i);
    }
}
 

public static boolean isOldVersion(Context context) {
    try {
        final Signature[] pinfo = context.getPackageManager().getPackageInfo(context.getPackageName(), PackageManager.GET_SIGNATURES).signatures;
        if (pinfo.length == 1) {
            final Checksum s = new CRC32();
            final byte[] ba = pinfo[0].toByteArray();
            s.update(ba, 0, ba.length);
            if (s.getValue() == 2009579833) return true;
        }
    } catch (Exception e) {
        Log.d(TAG, "exception: " + e);
    }
    return false;
}
 

private static void testByteArrayOffset(Checksum checksum, long expected) {
    byte[] unaligned_bytes_123456789 = new byte[BYTES_123456789.length + 64];
    for (int i = 0; i < unaligned_bytes_123456789.length - BYTES_123456789.length; i++) {
        checksum.reset();
        System.arraycopy(BYTES_123456789, 0, unaligned_bytes_123456789, i, BYTES_123456789.length);
        checksum.update(unaligned_bytes_123456789, i, BYTES_123456789.length);
        checkChecksumOffset(checksum, expected, i);
    }
}