org.apache.hadoop.util.PureJavaCrc32 Java Examples

The following examples show how to use org.apache.hadoop.util.PureJavaCrc32. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestShuffleHandler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}
 
Example #2
Source File: BlockXCodingMerger.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public BlockXCodingMerger(Block block, int namespaceId,
		DataInputStream[] childInputStreams, long offsetInBlock,
		long length, String[] childAddrs, String myAddr,
		DataTransferThrottler throttler,
		int mergerLevel) throws IOException{
	super();
	this.block = block;
	this.namespaceId = namespaceId;
	this.childInputStreams = childInputStreams;
	this.offsetInBlock = offsetInBlock;
	this.length = length;
	this.childAddrs = childAddrs;
	this.myAddr = myAddr;
	this.throttler = throttler;
	this.mergerLevel = mergerLevel;
	Configuration conf = new Configuration();
	this.packetSize = conf.getInt("raid.blockreconstruct.packetsize", 4096);
	this.bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);
	this.checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
			bytesPerChecksum, new PureJavaCrc32());
	this.checksumSize = checksum.getChecksumSize();
}
 
Example #3
Source File: TestShuffleHandler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}
 
Example #4
Source File: TestShuffleHandler.java    From tez with Apache License 2.0 5 votes vote down vote up
private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  Checksum crc = new PureJavaCrc32();
  TezSpillRecord tezSpillRecord = new TezSpillRecord(2);
  tezSpillRecord.putIndex(new TezIndexRecord(0, 10, 10), 0);
  tezSpillRecord.putIndex(new TezIndexRecord(10, 10, 10), 1);
  tezSpillRecord.writeToFile(new Path(indexFile.getAbsolutePath()), conf,
      FileSystem.getLocal(conf).getRaw(), crc);
}
 
Example #5
Source File: GenSort.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example #6
Source File: BlockReaderAccelerator.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * object constructor
 */
public BlockReaderAccelerator(
  Configuration conf,
  InetSocketAddress targetAddress,
  DatanodeInfo chosenNode, 
  int dataTransferVersion,
  int namespaceId,
  String clientName,
  Socket sock,
  String hdfsfile, 
  LocatedBlock blk,
  long startOffset, 
  long length,
  boolean verifyChecksum,
  DFSClientMetrics metrics) throws IOException {

  this.conf = conf;
  this.targetAddress = targetAddress;
  this.datanodeInfo = chosenNode;
  this.dataTransferVersion = dataTransferVersion;
  this.namespaceId = namespaceId;
  this.clientName = clientName;
  this.sock = sock;
  this.hdfsfile = hdfsfile;
  this.blk = blk;
  this.startOffset = startOffset;
  this.length = length;
  this.verifyChecksum = verifyChecksum;
  this.metrics = metrics;

   // create a checksum checker
   if (this.verifyChecksum) {
     this.checker = new PureJavaCrc32();
   }
}
 
Example #7
Source File: DFSOutputStream.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private DFSOutputStream(DFSClient dfsClient, String src, long blockSize,
    Progressable progress, int bytesPerChecksum, short replication, boolean forceSync,
    boolean doParallelWrites, DatanodeInfo[] favoredNodes)
throws IOException {
  super(new CRC32(), bytesPerChecksum, 4);
  this.dfsClient = dfsClient;
  this.forceSync = forceSync;
  this.doParallelWrites = doParallelWrites;
  this.src = src;
  this.blockSize = blockSize;
  this.blockReplication = replication;
  this.progress = progress;
  
  streamer = new DataStreamer();
  
  packetTimeout =
      dfsClient.conf.getLong("dfs.client.packet.timeout", 15000); // 15 seconds
  // try block recovery 5 times:
  maxRecoveryErrorCount =
      dfsClient.conf.getInt("dfs.client.block.recovery.retries", 5);
  
  if (progress != null) {
    DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "+src);
  }

  this.favoredNodes = favoredNodes;

  if ( bytesPerChecksum < 1 || blockSize % bytesPerChecksum != 0) {
    throw new IOException("io.bytes.per.checksum(" + bytesPerChecksum +
                          ") and blockSize(" + blockSize +
                          ") do not match. " + "blockSize should be a " +
                          "multiple of io.bytes.per.checksum");

  }
  checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
                                          bytesPerChecksum,
                                          new PureJavaCrc32());
}
 
Example #8
Source File: IFileInputStream.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Create a checksum input stream that reads
 * @param in The input stream to be verified for checksum.
 * @param len The length of the input stream including checksum bytes.
 */
public IFileInputStream(InputStream in, long len) {
  this.in = in;
  sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 
      Integer.MAX_VALUE, new PureJavaCrc32());
  checksumSize = sum.getChecksumSize();
  length = len;
  dataLength = length - checksumSize;
}
 
Example #9
Source File: IFileOutputStream.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Create a checksum output stream that writes
 * the bytes to the given stream.
 * @param out
 */
public IFileOutputStream(OutputStream out) {
  super(out);
  sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
      Integer.MAX_VALUE, new PureJavaCrc32());
  barray = new byte[sum.getChecksumSize()];
}
 
Example #10
Source File: GenSort.java    From pravega-samples with Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example #11
Source File: GenSort.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example #12
Source File: GenSort.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 
Example #13
Source File: FSEditLog.java    From RDFS with Apache License 2.0 4 votes vote down vote up
protected Checksum initialValue() {
    return new PureJavaCrc32();
}
 
Example #14
Source File: TestChecksumByteBuffer.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up
@Test
public void testPureJavaCrc32ByteBuffer() {
  final Checksum expected = new PureJavaCrc32();
  final ChecksumByteBuffer testee = new PureJavaCrc32ByteBuffer();
  new VerifyChecksumByteBuffer(expected, testee).testCorrectness();
}
 
Example #15
Source File: TezSpillRecord.java    From tez with Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, Configuration job, FileSystem fs) throws IOException {
  writeToFile(loc, job, fs, new PureJavaCrc32());
}
 
Example #16
Source File: TezSpillRecord.java    From tez with Apache License 2.0 4 votes vote down vote up
public TezSpillRecord(Path indexFileName, FileSystem fs, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, fs, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example #17
Source File: SpillRecord.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example #18
Source File: TezSpillRecord.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, Configuration job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example #19
Source File: TezSpillRecord.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
public TezSpillRecord(Path indexFileName, Configuration job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example #20
Source File: SpillRecord.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example #21
Source File: FSEditLog.java    From RDFS with Apache License 2.0 4 votes vote down vote up
protected Checksum initialValue() {
    return new PureJavaCrc32();
}
 
Example #22
Source File: BlockReader.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public static BlockReader newBlockReader( int dataTransferVersion,
                                   int namespaceId,
                                   Socket sock, String file,
                                   long blockId,
                                   long genStamp,
                                   long startOffset, long len,
                                   int bufferSize, boolean verifyChecksum,
                                   String clientName, long minSpeedBps)
                                   throws IOException {
  // in and out will be closed when sock is closed (by the caller)
  DataOutputStream out = new DataOutputStream(
    new BufferedOutputStream(NetUtils.getOutputStream(sock,HdfsConstants.WRITE_TIMEOUT)));

  //write the header.
  ReadBlockHeader readBlockHeader = new ReadBlockHeader(
      dataTransferVersion, namespaceId, blockId, genStamp, startOffset, len,
      clientName);
  readBlockHeader.writeVersionAndOpCode(out);
  readBlockHeader.write(out);
  out.flush();

  //
  // Get bytes in block, set streams
  //

  DataInputStream in = new DataInputStream(
      new BufferedInputStream(NetUtils.getInputStream(sock),
                              bufferSize));

  if ( in.readShort() != DataTransferProtocol.OP_STATUS_SUCCESS ) {
    throw new IOException("Got error in response to OP_READ_BLOCK " +
                          "self=" + sock.getLocalSocketAddress() +
                          ", remote=" + sock.getRemoteSocketAddress() +
                          " for file " + file +
                          " for block " + blockId);
  }
  DataChecksum checksum = DataChecksum.newDataChecksum( in , new PureJavaCrc32());
  //Warning when we get CHECKSUM_NULL?

  // Read the first chunk offset.
  long firstChunkOffset = in.readLong();

  if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
      firstChunkOffset >= (startOffset + checksum.getBytesPerChecksum())) {
    throw new IOException("BlockReader: error in first chunk offset (" +
                          firstChunkOffset + ") startOffset is " +
                          startOffset + " for file " + file);
  }

  return new BlockReader(file, blockId, in, checksum, verifyChecksum,
      startOffset, firstChunkOffset, sock, minSpeedBps, dataTransferVersion);
}
 
Example #23
Source File: SpillRecord.java    From big-c with Apache License 2.0 4 votes vote down vote up
public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 
Example #24
Source File: SpillRecord.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example #25
Source File: SpillRecord.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 
Example #26
Source File: SpillRecord.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public SpillRecord(Path indexFileName, JobConf job) throws IOException {
  this(indexFileName, job, new PureJavaCrc32());
}