Java Code Examples for org.apache.hadoop.util.PureJavaCrc32

The following examples show how to use org.apache.hadoop.util.PureJavaCrc32. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up
private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}
 
Example 2
Source Project: big-c   Source File: TestShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up
private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}
 
Example 3
Source Project: RDFS   Source File: BlockXCodingMerger.java    License: Apache License 2.0 6 votes vote down vote up
public BlockXCodingMerger(Block block, int namespaceId,
		DataInputStream[] childInputStreams, long offsetInBlock,
		long length, String[] childAddrs, String myAddr,
		DataTransferThrottler throttler,
		int mergerLevel) throws IOException{
	super();
	this.block = block;
	this.namespaceId = namespaceId;
	this.childInputStreams = childInputStreams;
	this.offsetInBlock = offsetInBlock;
	this.length = length;
	this.childAddrs = childAddrs;
	this.myAddr = myAddr;
	this.throttler = throttler;
	this.mergerLevel = mergerLevel;
	Configuration conf = new Configuration();
	this.packetSize = conf.getInt("raid.blockreconstruct.packetsize", 4096);
	this.bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);
	this.checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
			bytesPerChecksum, new PureJavaCrc32());
	this.checksumSize = checksum.getChecksumSize();
}
 
Example 4
Source Project: hadoop   Source File: GenSort.java    License: Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example 5
Source Project: pravega-samples   Source File: GenSort.java    License: Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example 6
Source Project: big-c   Source File: GenSort.java    License: Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example 7
Source Project: RDFS   Source File: IFileOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a checksum output stream that writes
 * the bytes to the given stream.
 * @param out
 */
public IFileOutputStream(OutputStream out) {
  super(out);
  sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
      Integer.MAX_VALUE, new PureJavaCrc32());
  barray = new byte[sum.getChecksumSize()];
}
 
Example 8
Source Project: RDFS   Source File: IFileInputStream.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a checksum input stream that reads
 * @param in The input stream to be verified for checksum.
 * @param len The length of the input stream including checksum bytes.
 */
public IFileInputStream(InputStream in, long len) {
  this.in = in;
  sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 
      Integer.MAX_VALUE, new PureJavaCrc32());
  checksumSize = sum.getChecksumSize();
  length = len;
  dataLength = length - checksumSize;
}
 
Example 9
Source Project: RDFS   Source File: DFSOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
private DFSOutputStream(DFSClient dfsClient, String src, long blockSize,
    Progressable progress, int bytesPerChecksum, short replication, boolean forceSync,
    boolean doParallelWrites, DatanodeInfo[] favoredNodes)
throws IOException {
  super(new CRC32(), bytesPerChecksum, 4);
  this.dfsClient = dfsClient;
  this.forceSync = forceSync;
  this.doParallelWrites = doParallelWrites;
  this.src = src;
  this.blockSize = blockSize;
  this.blockReplication = replication;
  this.progress = progress;
  
  streamer = new DataStreamer();
  
  packetTimeout =
      dfsClient.conf.getLong("dfs.client.packet.timeout", 15000); // 15 seconds
  // try block recovery 5 times:
  maxRecoveryErrorCount =
      dfsClient.conf.getInt("dfs.client.block.recovery.retries", 5);
  
  if (progress != null) {
    DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "+src);
  }

  this.favoredNodes = favoredNodes;

  if ( bytesPerChecksum < 1 || blockSize % bytesPerChecksum != 0) {
    throw new IOException("io.bytes.per.checksum(" + bytesPerChecksum +
                          ") and blockSize(" + blockSize +
                          ") do not match. " + "blockSize should be a " +
                          "multiple of io.bytes.per.checksum");

  }
  checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
                                          bytesPerChecksum,
                                          new PureJavaCrc32());
}
 
Example 10
Source Project: RDFS   Source File: BlockReaderAccelerator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * object constructor
 */
public BlockReaderAccelerator(
  Configuration conf,
  InetSocketAddress targetAddress,
  DatanodeInfo chosenNode, 
  int dataTransferVersion,
  int namespaceId,
  String clientName,
  Socket sock,
  String hdfsfile, 
  LocatedBlock blk,
  long startOffset, 
  long length,
  boolean verifyChecksum,
  DFSClientMetrics metrics) throws IOException {

  this.conf = conf;
  this.targetAddress = targetAddress;
  this.datanodeInfo = chosenNode;
  this.dataTransferVersion = dataTransferVersion;
  this.namespaceId = namespaceId;
  this.clientName = clientName;
  this.sock = sock;
  this.hdfsfile = hdfsfile;
  this.blk = blk;
  this.startOffset = startOffset;
  this.length = length;
  this.verifyChecksum = verifyChecksum;
  this.metrics = metrics;

   // create a checksum checker
   if (this.verifyChecksum) {
     this.checker = new PureJavaCrc32();
   }
}
 
Example 11
Source Project: incubator-tez   Source File: GenSort.java    License: Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example 12
Source Project: tez   Source File: TestShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up
private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  Checksum crc = new PureJavaCrc32();
  TezSpillRecord tezSpillRecord = new TezSpillRecord(2);
  tezSpillRecord.putIndex(new TezIndexRecord(0, 10, 10), 0);
  tezSpillRecord.putIndex(new TezIndexRecord(10, 10, 10), 1);
  tezSpillRecord.writeToFile(new Path(indexFile.getAbsolutePath()), conf,
      FileSystem.getLocal(conf).getRaw(), crc);
}
 
Example 13
Source Project: hadoop-ozone   Source File: TestChecksumByteBuffer.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testPureJavaCrc32ByteBuffer() {
  final Checksum expected = new PureJavaCrc32();
  final ChecksumByteBuffer testee = new PureJavaCrc32ByteBuffer();
  new VerifyChecksumByteBuffer(expected, testee).testCorrectness();
}
 
Example 14
Source Project: hadoop   Source File: SpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example 15
Source Project: hadoop   Source File: SpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example 16
Source Project: big-c   Source File: SpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example 17
Source Project: big-c   Source File: SpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example 18
Source Project: RDFS   Source File: SpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
public SpillRecord(Path indexFileName, JobConf job) throws IOException {
  this(indexFileName, job, new PureJavaCrc32());
}
 
Example 19
Source Project: RDFS   Source File: SpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example 20
Source Project: RDFS   Source File: BlockReader.java    License: Apache License 2.0 4 votes vote down vote up
public static BlockReader newBlockReader( int dataTransferVersion,
                                   int namespaceId,
                                   Socket sock, String file,
                                   long blockId,
                                   long genStamp,
                                   long startOffset, long len,
                                   int bufferSize, boolean verifyChecksum,
                                   String clientName, long minSpeedBps)
                                   throws IOException {
  // in and out will be closed when sock is closed (by the caller)
  DataOutputStream out = new DataOutputStream(
    new BufferedOutputStream(NetUtils.getOutputStream(sock,HdfsConstants.WRITE_TIMEOUT)));

  //write the header.
  ReadBlockHeader readBlockHeader = new ReadBlockHeader(
      dataTransferVersion, namespaceId, blockId, genStamp, startOffset, len,
      clientName);
  readBlockHeader.writeVersionAndOpCode(out);
  readBlockHeader.write(out);
  out.flush();

  //
  // Get bytes in block, set streams
  //

  DataInputStream in = new DataInputStream(
      new BufferedInputStream(NetUtils.getInputStream(sock),
                              bufferSize));

  if ( in.readShort() != DataTransferProtocol.OP_STATUS_SUCCESS ) {
    throw new IOException("Got error in response to OP_READ_BLOCK " +
                          "self=" + sock.getLocalSocketAddress() +
                          ", remote=" + sock.getRemoteSocketAddress() +
                          " for file " + file +
                          " for block " + blockId);
  }
  DataChecksum checksum = DataChecksum.newDataChecksum( in , new PureJavaCrc32());
  //Warning when we get CHECKSUM_NULL?

  // Read the first chunk offset.
  long firstChunkOffset = in.readLong();

  if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
      firstChunkOffset >= (startOffset + checksum.getBytesPerChecksum())) {
    throw new IOException("BlockReader: error in first chunk offset (" +
                          firstChunkOffset + ") startOffset is " +
                          startOffset + " for file " + file);
  }

  return new BlockReader(file, blockId, in, checksum, verifyChecksum,
      startOffset, firstChunkOffset, sock, minSpeedBps, dataTransferVersion);
}
 
Example 21
Source Project: RDFS   Source File: FSEditLog.java    License: Apache License 2.0 4 votes vote down vote up
protected Checksum initialValue() {
    return new PureJavaCrc32();
}
 
Example 22
Source Project: RDFS   Source File: FSEditLog.java    License: Apache License 2.0 4 votes vote down vote up
protected Checksum initialValue() {
    return new PureJavaCrc32();
}
 
Example 23
Source Project: incubator-tez   Source File: TezSpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
public TezSpillRecord(Path indexFileName, Configuration job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example 24
Source Project: incubator-tez   Source File: TezSpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, Configuration job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example 25
Source Project: tez   Source File: TezSpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
public TezSpillRecord(Path indexFileName, FileSystem fs, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, fs, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example 26
Source Project: tez   Source File: TezSpillRecord.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, Configuration job, FileSystem fs) throws IOException {
  writeToFile(loc, job, fs, new PureJavaCrc32());
}